nginx日志通过rsyslog传入到日志服务器指定目录

V5配置语法：

推送端设置

[root@nginx01 ~]# cat /etc/rsyslog.conf                                                                             

# rsyslog v5 configuration file

 

# For more information see/usr/share/doc/rsyslog-*/rsyslog_conf.html

# If you experience problems, seehttp://www.rsyslog.com/doc/troubleshoot.html

 

#### MODULES ####

 

$ModLoad imuxsock # provides support for localsystem logging (e.g. via logger command)

$ModLoad imklog  # provides kernel logging support (previously done by rklogd)

#$ModLoad immark # provides --MARK-- message capability

$ModLoadimfile   ##装载imfile模块

 

# Provides UDP syslog reception

#$ModLoad imudp

#$UDPServerRun 514

 

# Provides TCP syslog reception

#$ModLoad imtcp

#$InputTCPServerRun 514

 

#### GLOBAL DIRECTIVES ####

 

# Use default timestamp format

$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

 

# File syncing capability is disabled by default.This feature is usually not required,

# not useful and an extreme performance hit

#$ActionFileEnableSync on

 

# Include all config files in /etc/rsyslog.d/

$IncludeConfig /etc/rsyslog.d/*.conf

 

 

#### RULES ####

 

# Log all kernel messages to the console.

# Logging much else clutters up the screen.

#kern.*                                                /dev/console

 

# Log anything (except mail) of level info orhigher.

# Don't log private authentication messages!

*.info;mail.none;authpriv.none;cron.none;local5.none                /var/log/messages  ##不记录local5的日志

 

# The authpriv file has restricted access.

authpriv.*                                             /var/log/secure

 

# Log all the mail messages in one place.

mail.*                                                 -/var/log/maillog

 

 

# Log cron stuff

#cron.*                                                 /var/log/cron

cron.*                                                 @192.168.10.50

# Everybody gets emergency messages

*.emerg                                                *

 

# Save news errors of level crit and higher in aspecial file.

uucp,news.crit                                         /var/log/spooler

 

# Save boot messages also to boot.log

local7.*                                                /var/log/boot.log

 

 

# ### begin forwarding rule ###

# The statement between the begin ... end define aSINGLE forwarding

# rule. They belong together, do NOT split them. Ifyou create multiple

# forwarding rules, duplicate the whole block!

# Remote Logging (we use TCP for reliable delivery)

#

# An on-disk queue is created for this action. Ifthe remote host is

# down, messages are spooled to disk and sent whenit is up again.

#$WorkDirectory /var/lib/rsyslog # where to place spoolfiles

#$ActionQueueFileName fwdRule1 # unique name prefixfor spool files

#$ActionQueueMaxDiskSpace 1g   # 1gb space limit (use as much as possible)

#$ActionQueueSaveOnShutdown on # save messages todisk on shutdown

#$ActionQueueType LinkedList   # run asynchronously

#$ActionResumeRetryCount -1    # infinite retries if host is down

# remote host is: name/ip:port, e.g.192.168.0.1:514, port optional

#*.* @@remote-host:514

# ### end of the forwarding rule ###

$InputFileName/application/nginx/logs/access.log    ##读取日志文件（要监控的日志文件）

$InputFileTagweb_access     ##日志写入日志附加标签字符串

$InputFileSeverityinfo   ##日志等级

$InputFileStateFile/etc/rsyslog.d/stat-access    ##记录日志点等信息。（相当于msyql的master.info）文件名变了，

                                                                        这个StateFile标志必须变，否则无法传输。

$InputFileFacilitylocal5   ##设施类别

$InputFilePollInterval1    ##检查日志文件间隔（秒）

$InputFilePersistStateInterval1   ##回写偏移量数据到文件间隔时间（秒）

$InputRunFileMonitor   ##激活读取，可以设置多组日志读取，每组结束时设置本参数。以示生效。

local5.*  @192.168.10.51   ##代表local5设施的所有级别通过udp协议传送到192.168.10.51

接收端配置

[root@lb01 ~]# cat /etc/rsyslog.conf

# rsyslog v5 configuration file

 

# For more information see/usr/share/doc/rsyslog-*/rsyslog_conf.html

# If you experience problems, seehttp://www.rsyslog.com/doc/troubleshoot.html

 

#### MODULES ####

 

$ModLoad imuxsock # provides support for localsystem logging (e.g. via logger command)

$ModLoad imklog  # provides kernel logging support (previously done by rklogd)

#$ModLoad immark # provides --MARK-- message capability

 

# Provides UDP syslog reception

$ModLoadimudp      ##载入imudp模块

$UDPServerRun514   ##开启udp接收并制定端口号

 

# Provides TCP syslog reception

#$ModLoad imtcp

#$InputTCPServerRun 514

 

 

#### GLOBAL DIRECTIVES ####

 

# Use default timestamp format

$ActionFileDefaultTemplateRSYSLOG_TraditionalFileFormat

 

# File syncing capability is disabled by default.This feature is usually not required,

# not useful and an extreme performance hit

#$ActionFileEnableSync on

 

# Include all config files in /etc/rsyslog.d/

$IncludeConfig /etc/rsyslog.d/*.conf

 

 

#### RULES ####

 

# Log all kernel messages to the console.

# Logging much else clutters up the screen.

#kern.*                                                /dev/console

 

# Log anything (except mail) of level info orhigher.

# Don't log private authentication messages!

*.info;mail.none;authpriv.none;cron.none;local5.none                /var/log/messages  ##不记录local5设施的日志

 

# The authpriv file has restricted access.

authpriv.*                                             /var/log/secure

 

# Log all the mail messages in one place.

mail.*                                                 -/var/log/maillog

 

 

# Log cron stuff

cron.*                                                 /var/log/cron

 

# Everybody gets emergency messages

*.emerg                                                *

 

# Save news errors of level crit and higher in aspecial file.

uucp,news.crit                                         /var/log/spooler

 

# Save boot messages also to boot.log

local7.*                                               /var/log/boot.log

 

 

# ### begin forwarding rule ###

# The statement between the begin ... end define aSINGLE forwarding

# rule. They belong together, do NOT split them. Ifyou create multiple

# forwarding rules, duplicate the whole block!

# Remote Logging (we use TCP for reliable delivery)

#

# An on-disk queue is created for this action. Ifthe remote host is

# down, messages are spooled to disk and sent whenit is up again.

#$WorkDirectory /var/lib/rsyslog # where to placespool files

#$ActionQueueFileName fwdRule1 # unique name prefixfor spool files

#$ActionQueueMaxDiskSpace 1g   # 1gb space limit (use as much as possible)

#$ActionQueueSaveOnShutdown on # save messages todisk on shutdown

#$ActionQueueType LinkedList   # run asynchronously

#$ActionResumeRetryCount -1    # infinite retries if host is down

# remote host is: name/ip:port, e.g.192.168.0.1:514, port optional

#*.* @@remote-host:514

# ### end of the forwarding rule ###

#定义一个模板用来指定接收的日志消息的格式（默认会在记录的日志前加几个字段）

$templateSpiceTmpl,"%msg%\n"      ##%msg:2:$%为去掉日志开头的空格

#定义一个模板用来指定接收的日志文件的存放路径%……%之间的是定义日志按照年-月-日命名

$templateDynaFile,"/data/www/wifiin/logs/remote/nginx/%$YEAR%-%$MONTH%-%$DAY%.log"

#接收客户端local5设施传送来的日志并存放到指定位置（位置可用定义的模板。?代表使用动态的模板）

local5.*         ?DynaFile;SpiceTmpl

编辑/etc/sysconfig/rsyslog中SYSLOGD_OPTIONS=””开启远程日志接收功能

[root@lb01 ~]# cat /etc/sysconfig/rsyslog

# Options for rsyslogd

# Syslogd options are deprecated since rsyslog v3.

# If you want to use them, switch to compatibilitymode 2 by "-c 2"

# See rsyslogd(8) for more details

SYSLOGD_OPTIONS=""

 

或者

vi /etc/sysconfig/syslog

#######################################

把SYSLOGD_OPTIONS="-m0"

改成SYSLOGD_OPTIONS="-r -m 0"

#######################################

相关解释说明:

-r:打开接受外来日志消息的功能,其监控514 UDP端口;

-x:关闭自动解析对方日志服务器的FQDN信息,这能避免DNS不完整所带来的麻烦;

-m:修改syslog的内部mark消息写入间隔时间(0为关闭),例如240为每隔240分钟写入一次"--MARK--"信息;

-h:默认情况下,syslog不会发送从远端接受过来的消息到其他主机,而使用该选项,则把该开关打开,所有

接受到的信息都可根据syslog.conf中定义的@主机转发过去

V8配置语法：

v8 queue语法官方文档：

http://www.rsyslog.com/doc/v8-stable/rainerscript/queue_parameters.html?highlight=queue

v8模块官方文档

http://www.rsyslog.com/doc/v8-stable/configuration/modules/idx_output.html

发送端配置：

[root@wifiin-server-speedcloud-131 ~]# cat/etc/rsyslog.conf

# rsyslog configuration file

# note that most of this config file uses old-styleformat,

# because it is well-known AND quite suitable forsimple cases

# like we have with the default config. For moreadvanced

# things, RainerScript configuration is suggested.

 

# For more information see/usr/share/doc/rsyslog-*/rsyslog_conf.html

# If you experience problems, seehttp://www.rsyslog.com/doc/troubleshoot.html

 

$MaxMessageSize 1M

 

##定义规则##

##定义使用的模块类型、目标主机、端口号、使用的协议、压缩模式、压缩级别、队列类型、队列大小等。

###RULESET####

ruleset(name="remote"){

 action(type="omfwd"Target="124.193.131.2" Port="514" Protocol="tcp"compression.mode="single"compression.stream.flushOnTXEnd="off" ZipLevel="9"queue.filename="remoteQueue_124_193_131_2"queue.spoolDirectory="/var/log" queue.type="LinkedList"queue.saveonshutdown="on" queue.size="100000"queue.maxdiskspace="1000M" queue.maxfilesize="1000M" action.resumeretrycount="-1"queue.dequeuebatchsize="500" )

   

stop

}

#### MODULES ####

 

module(load="imuxsock") # providessupport for local system logging (e.g. via logger command)

module(load="imklog")   # provides kernel logging support(previously done by rklogd)

#module(load"immark")  # provides --MARK-- message capability

module(load="imfile")     ##载入imfile模块

 

# Provides UDP syslog reception

# for parameters seehttp://www.rsyslog.com/doc/imudp.html

#module(load="imudp") # needs to be donejust once

#input(type="imudp" port="514")

 

# Provides TCP syslog reception

# for parameters seehttp://www.rsyslog.com/doc/imtcp.html

module(load="imtcp")# needs to be done just once    ##载入imtcp模块

input(type="imtcp"port="514")   ##开启tcp接收日志

 

 

 

#### GLOBAL DIRECTIVES ####

 

# Use default timestamp format

$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

 

# File syncing capability is disabled by default.This feature is usually not required,

# not useful and an extreme performance hit

#$ActionFileEnableSync on

 

# Include all config files in /etc/rsyslog.d/

$IncludeConfig /etc/rsyslog.d/*.conf

 

 

#### RULES ####

 

# Log all kernel messages to the console.

# Logging much else clutters up the screen.

#kern.*                                                /dev/console

 

# Log anything (except mail) of level info orhigher.

# Don't log private authentication messages!

*.info;mail.none;authpriv.none;cron.none;local1.none                /var/log/messages   ##不记录local1设施的日志

 

# The authpriv file has restricted access.

authpriv.*                                             /var/log/secure

 

# Log all the mail messages in one place.

mail.*                                                 /var/log/maillog

 

 

# Log cron stuff

cron.*                                                 /var/log/cron

 

# Everybody gets emergency messages

*.emerg                                                :omusrmsg:*

 

# Save news errors of level crit and higher in aspecial file.

uucp,news.crit                                         /var/log/spooler

 

# Save boot messages also to boot.log

local7.*                                                /var/log/boot.log

 

# ### begin forwarding rule ###

# The statement between the begin ... end define aSINGLE forwarding

# rule. They belong together, do NOT split them. Ifyou create multiple

# forwarding rules, duplicate the whole block!

# Remote Logging (we use TCP for reliable delivery)

#

# An on-disk queue is created for this action. Ifthe remote host is

# down, messages are spooled to disk and sent whenit is up again.

#$WorkDirectory /var/lib/rsyslog # where to placespool files

#$ActionQueueFileName fwdRule1 # unique name prefixfor spool files

#$ActionQueueMaxDiskSpace 1g   # 1gb space limit (use as much as possible)

#$ActionQueueSaveOnShutdown on # save messages todisk on shutdown

#$ActionQueueType LinkedList   # run asynchronously

#$ActionResumeRetryCount -1    # infinite retries if host is down

# remote host is: name/ip:port, e.g.192.168.0.1:514, port optional

:rawmsg, contains, "gdnsd" stop

#*.* @@(z9)124.193.131.2:514

# ### end of the forwarding rule ###

 

# sdk.wifiin.cnnginx logs push

input(type="imfile"    ##输入类型为imfile

    StateFile="/etc/rsyslog.d/access_sdk_nginx"    ##记录日志点等信息。（相当于msyql的master.info）

文件名变了，这个StateFile标志必须变，否则无法传输。

   File="/data/www/wifiin/logs/nginx/access_sdk.wifiin.cn.log"   ##读取日志文件（要监控的日志文件）

    Facility="local1"    ##设施类别

    Severity="info"    ##日志等级

    Tag="sdk_nginx_access"    ##日志附加标签字符串（接收端可以通过过滤这个字符串来选择接收日志）

    PersistStateInterval="1"     ##回写偏移量数据到文件间隔时间（秒）

    Ruleset="remote")     ##使用remote规则

input(type="imfile"

    StateFile="/etc/rsyslog.d/error_sdk_nginx"

   File="/data/www/wifiin/logs/nginx/error_sdk.wifiin.cn.log"

    Facility="local1"

    Severity="info"

    Tag="sdk_nginx_error"

    PersistStateInterval="1"

    Ruleset="remote")

接收端配置：

[root@lb02 sdk_nginx]# cat /etc/rsyslog.conf

# rsyslog configuration file

# note that most of this config file uses old-styleformat,

# because it is well-known AND quite suitable forsimple cases

# like we have with the default config. For moreadvanced

# things, RainerScript configuration is suggested.

 

# For more information see/usr/share/doc/rsyslog-*/rsyslog_conf.html

# If you experience problems, seehttp://www.rsyslog.com/doc/troubleshoot.html

 

#### MODULES ####

 

module(load="imuxsock") # providessupport for local system logging (e.g. via logger command)

module(load="imklog")   # provides kernel logging support(previously done by rklogd)

#module(load"immark")  # provides --MARK-- message capability

 

# Provides UDP syslog reception

# for parameters see http://www.rsyslog.com/doc/imudp.html

#module(load="imudp") # needs to be donejust once

#input(type="imudp" port="514")

 

# Provides TCP syslog reception

# for parameters seehttp://www.rsyslog.com/doc/imtcp.html

module(load="imtcp")# needs to be done just once   ##载入imtcp模块

input(type="imtcp"port="514")     ##开启tcp接收日志

 

 

##定义记录日志信息模板

$template  SpiceTmpl,"%HOSTNAME% %msg:2:$%\n"

##定义日志存放位置及日志名格式

$template  NginxAccessDynaFile,"/data/www/wifiin/logs/remote/sdk_nginx/access-%$year%-%$month%-%$day%.log"

$template  NginxErrorDynaFile,"/data/www/wifiin/logs/remote/sdk_nginx/error-%$year%-%$month%-%$day%.log"

 

 

#### GLOBAL DIRECTIVES ####

 

# Use default timestamp format

$ActionFileDefaultTemplateRSYSLOG_TraditionalFileFormat

 

# File syncing capability is disabled by default.This feature is usually not required,

# not useful and an extreme performance hit

#$ActionFileEnableSync on

 

# Include all config files in /etc/rsyslog.d/

$IncludeConfig /etc/rsyslog.d/*.conf

 

 

#### RULES ####

 

# Log all kernel messages to the console.

# Logging much else clutters up the screen.

#kern.*                                                /dev/console

 

# Log anything (except mail) of level info orhigher.

# Don't log private authentication messages!

*.info;mail.none;authpriv.none;cron.none;local1.none                /var/log/messages  ##不记录local1的日志信息

 

# The authpriv file has restricted access.

authpriv.*                                             /var/log/secure

 

# Log all the mail messages in one place.

mail.*                                                  /var/log/maillog

 

 

# Log cron stuff

cron.*                                                 /var/log/cron

 

# Everybody gets emergency messages

*.emerg                                                :omusrmsg:*

 

# Save news errors of level crit and higher in aspecial file.

uucp,news.crit                                         /var/log/spooler

 

# Save boot messages also to boot.log

local7.*                                               /var/log/boot.log

 

##解决local0-local7不够用的问题

##过滤Tag中包含sdk_nginx_access的日志并应用?NginxAccessDynaFile;SpiceTmpl模板

:rawmsg,contains,"sdk_nginx_access" ?NginxAccessDynaFile;SpiceTmpl

##过滤Tag中包含sdk_nginx_error的日志并应用?NginxErrorDynaFile;SpiceTmpl模板

:rawmsg,contains,"sdk_nginx_error" ?NginxErrorDynaFile;SpiceTmpl

 

# ### begin forwarding rule ###

# The statement between the begin ... end define aSINGLE forwarding

# rule. They belong together, do NOT split them. Ifyou create multiple

# forwarding rules, duplicate the whole block!

# Remote Logging (we use TCP for reliable delivery)

#

# An on-disk queue is created for this action. Ifthe remote host is

# down, messages are spooled to disk and sent whenit is up again.

#$WorkDirectory /var/lib/rsyslog # where to placespool files

#$ActionQueueFileName fwdRule1 # unique name prefixfor spool files

#$ActionQueueMaxDiskSpace 1g   # 1gb space limit (use as much as possible)

#$ActionQueueSaveOnShutdown on # save messages todisk on shutdown

#$ActionQueueType LinkedList   # run asynchronously

#$ActionResumeRetryCount -1    # infinite retries if host is down

# remote host is: name/ip:port, e.g.192.168.0.1:514, port optional

#*.* @@remote-host:514

# ### end of the forwarding rule ###