使用NLog通過Kafka實現日誌收集

使用NLog通過Kafka實現日誌收集，最終在Kibana展示

 NuGet包引用

<PackageReference Include="NLog.Kafka" Version="0.2.1" />
<PackageReference Include="NLog.Web.AspNetCore" Version="4.14.0" />

 

Logstash配置

input {
    kafka {
       bootstrap_servers => "127.0.0.1:9092"
       group_id => "logstash"
       topics => "loges"
       codec => "json"
    }
}
output{
  elasticsearch {
        hosts => ["127.0.0.1:9002"]
        index => "log_{[appname]}_%{+YYYY.MM.dd}"
  }
}

項目NLog配置 nlog.config

<?xml version="1.0" encoding="utf-8" ?>
<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      autoReload="true"
      throwConfigExceptions="true"
      internalLogLevel="info"
      internalLogFile="App_Data/logs/internal-nlog.txt">
    <!--autoReload：修改後自動加載，可能會有延遲-->
    <!--throwConfigExceptions：NLog日誌系統拋出異常-->
    <!--internalLogLevel：內部日誌的級別-->
    <!--internalLogFile：內部日誌保存路徑，日誌的內容大概就是NLog的版本信息，配置文件的地址等等-->
    <!--輸出日誌的配置，用於rules讀取-->

    <!--擴展的程序集，官方文檔：https://github.com/nlog/NLog/wiki/Extending%20NLog-->
    <extensions>
        <add assembly="NLog.Web.AspNetCore"/>
        <add assembly="WebApplication1"/>
        <add assembly="NLog.Kafka"/>
        <add assembly="Sentry.NLog"/>        
    </extensions>
    
    <!-- Layout佈局
    ${var:basePath} basePath是前面自定義的變量
    ${longdate} 日期格式 2017-01-17 16:58:03.8667
    ${shortdate}日期格式 2017-01-17
    ${date:yyyyMMddHHmmssFFF} 日期 20170117165803866
    ${message} 輸出內容
    ${guid} guid
    ${level}日誌記錄的等級
    ${logger} 配置的logger
    -->

    <!-- NLog記錄等級
    Trace - 最常見的記錄信息，一般用於普通輸出
    Debug - 同樣是記錄信息，不過出現的頻率要比Trace少一些，一般用來調試程序
    Info - 信息類型的消息
    Warn - 警告信息，一般用於比較重要的場合
    Error - 錯誤信息
    Fatal - 致命異常信息。一般來講，發生致命異常之後程序將無法繼續執行。
    自上而下，等級遞增。
    -->
    
    <variable name="consoleLayout"  value="${longdate}|${event-properties:item=EventId_Id}|${uppercase:${level}}|${logger}|${message} ${exception:format=tostring}" />
    
    <variable name="fileLayout" value="${longdate}|${event-properties:item=EventId_Id}|${uppercase:${level}}|${logger}|${message} ${exception:format=tostring}|url: ${aspnet-request-url}|action: ${aspnet-mvc-action}" />

    <targets>
        <!--控制檯日誌-->
        <target name="console" xsi:type="ColoredConsole" layout="${consoleLayout}" />
        <!--文件日誌，archive相關參數：文件拆分，archiveAboveSize="104857600"：每100M拆分一個新文件 -->
        <target name="logFile" xsi:type="File"  layout="${fileLayout}"  fileName="C:/logs/${shortdate}/${hostname}/${logName}${shortdate}.log"
                archiveEvery="Day"
                archiveFileName="C:/logs/${shortdate}/${hostname}/${logName}${shortdate}.{####}.log"
                archiveNumbering="DateAndSequence"
                archiveDateFormat="yyyy-MM-dd"
                archiveAboveSize="104857600"/>

        <target name="kafka" xsi:type="Kafka" topic="logs.mbridge.taskapi" bootstrapServers="192.168.100.100:9092">
            <property name="security.protocol" value="Plaintext" />
            <property name="sasl.mechanism" value="PLAIN" />
            <property name="acks" value="0" />

            <layout xsi:type="JsonLayout">
                <attribute name="Time" layout="${longdate}" />
                <attribute name="EventId" layout="${event-properties:item=EventId_Id}" />
                <attribute name="Level" layout="${uppercase:${level}}"/>
                <attribute name="Logger" layout="${logger}" />
                <attribute name="Hostname" layout="${hostname}" />
                <attribute name="LogName" layout="${logName}" />
                <attribute name="Message" layout="${message}" />
                <attribute name="Exception" layout="${exception:format=tostring}" />
            </layout>
        </target>
    </targets>
    
    <rules>
        <!--路由順序會對日誌打印產生影響。路由匹配邏輯爲順序匹配。-->
        
        <!--路由映射關係
        writeTo - 有writeTo纔會寫日誌，沒有此attr則忽略；writeTo的值對應<target>裏面的 name="xxx"
        final - final="true"表示路由結束
        -->
        
        <!-- NLog等級使用
        指定特定等級 如：level="Warn"
        指定多個等級 如：levels=“Warn,Debug“ 以逗號隔開
        指定等級範圍 如：minlevel="Warn" maxlevel="Error"
        -->

        <!--跳過microsoft的日誌,不記錄-->
        <logger name="Microsoft.AspNetCore.*" maxLevel="Warn" final="true" />
        <!--跳過EasyNetQ的日誌,不記錄-->
        <logger name="EasyNetQ.*" maxLevel="Warn" final="true" />

        <!--所有的日誌, 包含以Microsoft開頭的-->
        <logger name="*" maxlevel="Debug" writeTo="console" />
        <logger name="*" minlevel="Debug" writeTo="logFile" />
        <logger name="*" minlevel="Debug" writeTo="kafka" />
    </rules>
    
</nlog>

代碼

 

    public class LogUtlExHelper
    {
        private static bool _kafka = false;
        private static bool _file = false;

        /// <summary>
        /// NLog擴展的Property
        /// </summary>
        public const string NLOG_PROPERTY_LOGNAME = "logName";
        /// <summary>
        /// NLog擴展的Property
        /// </summary>
        public const string NLOG_PROPERTY_HOSTNAME = "hostName";

        public const string NLOG_TARGET_NAME_LOGFILE = "logFile";
        public const string NLOG_TARGET_NAME_CONSOLE = "console";
        public const string NLOG_TARGET_NAME_KAFKA = "kafka";

        /// <summary>
        /// 文件日誌的Logger，使用的name與NLog配置的target.name對應
        /// </summary>
        public static Logger fileLogger;
        /// <summary>
        /// 控制檯日誌的Logger，使用的name與NLog配置的target.name對應
        /// </summary>
        public static Logger consoleLogger;
        /// <summary>
        /// 控制檯日誌的Logger，使用的name與NLog配置的target.name對應
        /// </summary>
        public static Logger kafkaLogger;

        static LogUtlExHelper()
        {
            InitNLog();
        }

        public static void InitNLog()
        {
            fileLogger = LogManager.GetLogger(NLOG_TARGET_NAME_LOGFILE);
            consoleLogger = LogManager.GetLogger(NLOG_TARGET_NAME_CONSOLE);
            kafkaLogger = LogManager.GetLogger(NLOG_TARGET_NAME_KAFKA);

            SetNLogTargets();
        }

        public static void SetNLogTargets()
        {
            //Kafka
            var target = (NLog.Targets.Kafka.KafkaTarget)kafkaLogger.Factory.Configuration.FindTargetByName(NLOG_TARGET_NAME_KAFKA);
            if (target != null && !string.IsNullOrEmpty(target.BootstrapServers))
            {
                _kafka = true;
            }

            //File
            var targetFile = (NLog.Targets.FileTarget)kafkaLogger.Factory.Configuration.FindTargetByName(NLOG_TARGET_NAME_LOGFILE);
            if (targetFile != null && targetFile.FileName != null)
            {
                _file = true;
            }
        }

        #region File
        /// <summary>
        /// 添加日誌
        /// </summary>
        /// <param name="fileName">日誌文件名稱</param>
        /// <param name="message">日誌內容</param>
        /// <param name="folderName">日誌文件夾名稱</param>
        public static void Info(string fileName, string message, string folderName = "")
        {
            WriteLog(LogLevel.Info, message, fileName, folderName);
        }

        /// <summary>
        /// 添加警告日誌
        /// </summary>
        /// <param name="fileName">日誌文件名稱</param>
        /// <param name="message">日誌內容</param>
        /// <param name="folderName">日誌文件夾名稱</param>
        public static void Warn(string fileName, string message, string folderName = "")
        {
            WriteLog(LogLevel.Warn, message, fileName, folderName);
        }

        /// <summary>
        /// 添加異常日誌
        /// </summary>
        /// <param name="exception"></param>
        /// <param name="message">日誌內容</param>
        /// <param name="fileName">日誌文件名稱</param>
        /// <param name="folderName">日誌文件夾名稱</param>
        public static void Error(Exception exception, string message = "", string fileName = "", string folderName = "")
        {
            Error(message, fileName, folderName, exception);
        }

        /// <summary>
        /// 添加異常日誌
        /// </summary>
        /// <param name="message">日誌內容</param>
        /// <param name="fileName">日誌文件名稱</param>
        /// <param name="folderName">日誌文件夾名稱</param>
        /// <param name="exception"></param>
        public static void Error(string message, string fileName = "", string folderName = "", Exception exception = null)
        {
            if (string.IsNullOrEmpty(folderName))
            {
                folderName = "Error";
            }

            WriteLog(LogLevel.Error, message, fileName, folderName, exception);
        }

        /// <summary>
        /// 添加日誌
        /// </summary>
        /// <param name="logLevel">日誌等級</param>
        /// <param name="message">日誌內容</param>
        /// <param name="fileName">日誌文件名稱</param>
        /// <param name="folderName">日誌文件夾名稱</param>
        /// <param name="exception"></param>
        public static void WriteLog(LogLevel logLevel, string message, string fileName = "", string folderName = "", Exception exception = null)
        {
            string logName = string.Empty;
            string loggerName = string.Empty;

            if (_kafka)
            {
                //Kafka
                loggerName = kafkaLogger.Name;
                if (!string.IsNullOrEmpty(folderName) && !string.IsNullOrEmpty(fileName))
                {
                    logName = $"{folderName}/{fileName}";
                }
                else if (!string.IsNullOrEmpty(folderName))
                {
                    logName = folderName ?? string.Empty;
                }
                else
                {
                    logName = fileName ?? string.Empty;
                }

                LogEventInfo theEvent = new LogEventInfo(logLevel, loggerName, message);
                theEvent.Properties[NLOG_PROPERTY_LOGNAME] = logName;
                theEvent.Properties[NLOG_PROPERTY_HOSTNAME] = Environment.MachineName;
                theEvent.Exception = exception;

                kafkaLogger.Log(theEvent);
            }

            if (_file)
            {
                //本地文件日誌
                loggerName = fileLogger.Name;
                if (!string.IsNullOrEmpty(fileName))
                {
                    fileName = $"{fileName}_";
                }
                if (!string.IsNullOrEmpty(folderName))
                {
                    logName = $"{folderName}/{fileName}";
                }
                else
                {
                    logName = fileName ?? string.Empty;
                }

                LogEventInfo theEvent = new LogEventInfo(logLevel, loggerName, message);
                theEvent.Properties[NLOG_PROPERTY_LOGNAME] = logName;
                theEvent.Properties[NLOG_PROPERTY_HOSTNAME] = Environment.MachineName;
                theEvent.Exception = exception;

                fileLogger.Log(theEvent);
            }
        }
        #endregion

        #region Console
        /// <summary>
        /// 控制檯日誌
        /// </summary>
        /// <param name="message"></param>
        /// <param name="exception"></param>
        public static void ConsoleLog(string message, Exception exception = null)
        {
            ConsoleLog(LogLevel.Debug, message, exception);
        }
        /// <summary>
        /// 控制檯日誌
        /// </summary>
        /// <param name="logLevel"></param>
        /// <param name="message"></param>
        /// <param name="exception"></param>
        public static void ConsoleLog(LogLevel logLevel, string message, Exception exception = null)
        {
            LogEventInfo theEvent = new LogEventInfo(logLevel, consoleLogger.Name, message);
            //theEvent.Properties["EventId"] = new Microsoft.Extensions.Logging.EventId(2, "eventId2");
            theEvent.Exception = exception;
            consoleLogger.Log(theEvent);
        }
        #endregion
    }

/// <summary>
/// custom layout renderer
/// </summary>
[LayoutRenderer(LogUtlExHelper.NLOG_PROPERTY_LOGNAME)]
[ThreadSafe]
public class NLogLayoutRender : LayoutRenderer
{
    protected override void Append(StringBuilder builder, LogEventInfo logEvent)
    {
        if (!logEvent.Properties.ContainsKey(LogUtlExHelper.NLOG_PROPERTY_LOGNAME))
        {
            return;
        }
        string logName = logEvent.Properties[LogUtlExHelper.NLOG_PROPERTY_LOGNAME] as string;
        if (!string.IsNullOrEmpty(logName))
        {
            builder.Append(logName);
        }
    }
}

/// <summary>
/// custom layout renderer
/// </summary>
[LayoutRenderer(LogUtlExHelper.NLOG_PROPERTY_HOSTNAME)]
[ThreadSafe]
public class NLogLayoutRender2 : LayoutRenderer
{
    protected override void Append(StringBuilder builder, LogEventInfo logEvent)
    {
        if (!logEvent.Properties.ContainsKey(LogUtlExHelper.NLOG_PROPERTY_HOSTNAME))
        {
            return;
        }
        string hostName = logEvent.Properties[LogUtlExHelper.NLOG_PROPERTY_HOSTNAME] as string;
        if (!string.IsNullOrEmpty(hostName))
        {
            builder.Append(hostName);
        }
    }
}

 Kibana配置