【Scala】基於akka的多線程應用程序日誌收集服務

    Akka is a toolkit and runtime for building highly concurrent,distributed, and resilient message-driven applications on the JVM. Akka是JVM之上高併發的分佈式，可伸縮的消息驅動應用框架。下面我們將通過Akka框架實現多線程的日誌收集功能。我們把收集到的日誌數據實時存放到HDFS中，以供後續分析挖掘使用。

• 通過SSH方式，遠程登錄到客戶端 使用SSH在遠程客戶端執行SHELL命令，如最常見的tail -f實時讀取日誌文件增量

package com.changtu.service

import java.io.{BufferedWriter, OutputStreamWriter}

import akka.actor.Actor
import com.changtu.core.Host
import com.changtu.util.Logging
import com.changtu.util.hdfs.HDFSUtils
import com.changtu.util.host.{AES, Configuration, SSH}
import org.apache.hadoop.fs.Path
import org.joda.time.DateTime

import scala.util.{Failure, Success}

/**
  * Created by lubinsu on 8/22/2016.
  * 日誌收集程序，通過指定的命令收集各個客戶端的日誌，通過akka實現併發操作
  */

class CollectLogService extends Actor with Logging {

  override def receive: Receive = {
    case Host(host, port, cmd) =>
      getLogs(host, port, cmd) match {
        case 0 => logger.info("success.")
        case _ => logger.error("error.")
      }
    case _ => logger.warn("unknown operation.")
  }


  /**
    * 根據shell命令收集指定主機上的日誌
    *
    * @param host 需要收集的主機
    * @param port ssh端口號
    * @param cmd  執行命令
    * @return 返回執行後的狀態碼
    */
  private def getLogs(host: String, port: String, cmd: String): Int = {
    // 密碼解密
    val password = AES.decrypt(Configuration("passwd").getProperty(host.concat("-hadoop")), "secretKey.changtu.com") match {
      case Success(encrypted) =>
        encrypted.asInstanceOf[String]
      case Failure(e) =>
        logger.error(e.getMessage)
        ""
    }

    val ssh = (cmd: String) => SSH(host, "hadoop", port.toInt, cmd, "", password, loadToHdfs)
    ssh(cmd)
  }

  /**
    * 收集到的日誌處理方式
    * @param msg 傳入一行行記錄
    */
  private def loadToHdfs(msg: String, host: String): Unit = {
    //logger.info(msg)
    val currentTime = DateTime.now.toString("yyyyMMdd")
    val path = "/user/hadoop/bigdata/logs/rest.".concat(host).concat("-").concat(currentTime).concat(".log")
    HDFSUtils.createDirectory(path, deleteF = false)
    val fsout = HDFSUtils.getHdfs.append(new Path(path))

    val br = new BufferedWriter(new OutputStreamWriter(fsout))
    br.write(msg)
    br.newLine()
    br.close()
    fsout.close()
  }

  /**
    * 收集到的日誌處理方式
    * @param msg 傳入一行行記錄
    */
  private def loadToKafka(msg: String, host: String): Unit = {
    //logger.info(msg)
    val currentTime = DateTime.now.toString("yyyyMMdd")
    val path = "/user/hadoop/bigdata/logs/rest.".concat(host).concat("-").concat(currentTime).concat(".log")
    HDFSUtils.createDirectory(path, deleteF = false)
    val fsout = HDFSUtils.getHdfs.append(new Path(path))

    val br = new BufferedWriter(new OutputStreamWriter(fsout))
    br.write(msg)
    br.newLine()
    br.close()
    fsout.close()
  }

}

• 定義Host class

package com.changtu.util

/**
  * Created by lubinsu on 8/16/2016.
  * 配置默認參數
  */
package object host {
  /**
    * Specifies the default `charset` used to encode and decode strings.
    */
  private[host] final val DefaultCharset = "UTF-8"
}

• 命令派發

package com.changtu.service

import akka.actor.{Actor, Props}
import com.changtu.core.Host
import com.changtu.util.Logging
import com.changtu.util.host.{AES, Configuration, SSH}

import scala.util.{Failure, Success}

/**
  * Created by lubinsu on 8/23/2016.
  * 命令派發
  */
class CollectMonitor extends Actor with Logging {
  override def receive: Receive = {
    case Host(host, port, cmd) =>
      getLogFiles(host, port, cmd)
      val collector = context.actorOf(Props[CollectLogService], "collector-".concat(host))
      context.children.foreach( p => {
        println(p.path.name)
      })
      collector ! Host(host, port, cmd)
    case _ => logger.warn("unknown operation.")
  }

  private def getLogFiles(host: String, port: String, cmd: String): Int = {
    // 密碼解密
    val password = AES.decrypt(Configuration("passwd").getProperty(host.concat("-hadoop")), "secretKey.changtu.com") match {
      case Success(encrypted) =>
        encrypted.asInstanceOf[String]
      case Failure(e) =>
        logger.error(e.getMessage)
        ""
    }

    val ssh = (cmd: String) => SSH(host, "hadoop", port.toInt, cmd, "", password)
    ssh("find /appl/logs -type f")
  }

}

• 主Actor

package com.changtu.api

import akka.actor.{ActorSystem, Props}
import com.changtu.core.Host
import com.changtu.service.CollectMonitor

/**
  * Created by lubinsu on 8/23/2016.
  * 主Actor
  */
object CollectLogs extends App {

  if (args.length < 2) {
    System.err.println("Usage: CollectLogs <hosts> <command>")
    System.exit(1)
  }

  val Array(hosts, cmd) = args

  val system = ActorSystem("CollectSystem")

  val monitor = system.actorOf(Props[CollectMonitor], name = "CollectMonitor-".concat(hosts))

  hosts.split(",").foreach( p => {
    // default Actor constructor
    monitor ! Host(p.split(":")(0), p.split(":")(1), cmd)
  })

}

• 打包執行

mvn clean install

nohup java -Djava.ext.dirs=/appl/scripts/e-business/platform/target/lib:/usr/share/jdk1.8.0_77/jre/lib/ext -classpath /appl/scripts/e-business/platform/target/platform-1.1.1.jar com.changtu.api.CollectLogs "172.19.3.151:22,172.19.3.152:22,172.19.3.153:22,172.19.3.154:22,172.19.3.156:22,172.19.0.5:22,172.19.0.95:11017" "tail -f /appl/scripts/e-business/rest/*.log" >> /appl/scripts/e-business/platform/$$.log 2>&1 &