logo
tt's Note
  • 运维
    • 数据库
    • linux
    • vpn
    • 日志
    • 中间件
    • 服务
    • 监控
    • shell
    • windows
    • 语言
    • 云服务
    • 其他
  • 开发
    • 工具
  • 软件
    • 浏览器
    • 多端
    • win
    • mac
    • 网站
  • 项目
    • 效率工具
    • 兴趣
  • 脚本
    • jenkins
    日志概览
    ELK日志系统
    clickhouse日志系统
    clickvisual
    filebeat
    loki
    rsyslog
    轻量化收集与消费
    上一页loki下一页轻量化收集与消费

    #rsyslog

    #单机版

    需要安装docker-ce才能用,用yum安装的不能安装!!!

    #配置syslog

    vi /etc/rsyslog.conf
    
    #把tcp两行注释去掉
    $ModLoad imtcp
    $InputTCPServerRun 514
    
    systemctl restart rsyslog
    systemctl status rsyslog

    #给docker配置rsyslog

    cat /etc/docker/daemon.json
    {
      "data-root": "/data/data-docker",
      "log-driver": "syslog",
      "log-opts": {
        "syslog-address": "tcp://127.0.0.1:514",
        "tag": "docker/{{.Name}},"
       }
    }
    data-root  #指定镜像容器存放的位置
    log-driver #创建一个日志处理的服务
    log-opts   #日志服务的参数
      syslog-address  #tcp 公网ip
      tag             #日志存放的位置,具体路径在模板处设置
                        
    systemctl restart docker
    systemctl status docker

    #创建一个rsyslog模板

    cd /etc/rsyslog.d
    #新建模板
    vi rule.conf
    #最开始要添加这些
    # #011替换成tab
    $EscapeControlCharactersOnReceive off
    # 删除日志首位空格,只保留原日志
    $template CleanMsgFormat,"%msg:2:$%\n"
    
    
    #### bind ####
    # 定义 bind 日志模版
    ### 拦截全部docker日志
    $template docker,"data/logs/docker/%syslogtag:F,44:1%/%$YEAR%-%$MONTH%-%$DAY%.log"
    if $syslogtag contains 'docker' then ?docker;CleanMsgFormat
    & ~
    #& stop 类似于java 的break
    
    #### nginx ####
    $template nginx,"/data/logs/nginx/%programname%-%timereported:0:10:date-rfc3339%-%HOSTNAME%.log"
    if $programname startswith 'nginx' then ?nginx;CleanMsgFormat
    & stop
    systemctl restart rsyslog
    systemctl status rsyslog
    $template bind,"/data/logs/......"
     # 日志的绝对路径 和 daemon.json中的 tag参数 拼接 的路径为 日志具体存放地址

    将模板放到docker的配置上

    #日志自动压缩+自动删除脚本

    #!/bin/bash
    
    left_day=180
    
    find /data/logs/ -name *$(date -d "1 days ago" +%Y-%m-%d)*.log -type f -exec gzip {} \;
    find /data/logs -name *$(date -d "${left_day} days ago" +%Y-%m-%d)*.gz -exec rm -rf {} \;
    find /data/logs -name *$(date -d "${left_day} days ago" +%Y-%m-%d)*.log -exec rm -rf {} \;

    #部署日志服务器

    #目录

    .
    ├── rsyslog.conf
    ├── rsyslog.d
    │   └── app.conf
    └── start.sh

    #启动脚本

    #!/bin/bash
    
    SYSLOGD_OPTIONS=""
    
    docker kill rsyslog
    docker rm rsyslog
    docker run -d --name rsyslog \
    --restart=always \
    --network=host \
    -v /etc/timezone:/etc/timezone:ro \
    -v /etc/localtime:/etc/localtime:ro \
    -v $(pwd)/rsyslog.conf:/etc/rsyslog.conf \
    -v $(pwd)/rsyslog.d/:/etc/rsyslog.d/ \
    -v /data/logs/:/data/logs/ \
    --log-driver=json-file \
    --log-opt max-size=10m \
    --log-opt max-file=3 \
    buyfakett/rsyslog \
    /usr/sbin/rsyslogd -n $SYSLOGD_OPTIONS
    
    sleep 3
    
    # 旧镜像 registry.cn-hangzhou.aliyuncs.com/buyfakett/rsyslog_base_centos7:latest
    
    docker logs --tail=100 rsyslog
    

    #配置

    内置配置rsyslog.conf
    自定义配置app.conf
    # rsyslog configuration file
    # #011替换成tab
    $EscapeControlCharactersOnReceive off
    # 删除日志首位空格,只保留原日志
    $template CleanMsgFormat,"%msg:2:$%\n"
    
    # 设置新创建的日志文件的权限为 644
    $FileCreateMode 0644
    # 设置新创建的日志文件夹的权限为 755
    $DirCreateMode 0755
    
    # For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
    # If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
    
    #### MODULES ####
    
    # The imjournal module bellow is now used as a message source instead of imuxsock.
    $ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
    $ModLoad imjournal # provides access to the systemd journal
    #$ModLoad imklog # reads kernel messages (the same are read from journald)
    #$ModLoad immark  # provides --MARK-- message capability
    
    $MaxMessageSize 32k
    
    # Provides UDP syslog reception
    #$ModLoad imudp
    #$UDPServerRun 514
    $ModLoad imudp
    $UDPServerRun 10514
    
    # Provides TCP syslog reception
    $ModLoad imtcp
    $InputTCPServerRun 10514
    
    
    #### GLOBAL DIRECTIVES ####
    
    # Where to place auxiliary files
    $WorkDirectory /var/lib/rsyslog
    
    # Use default timestamp format
    $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
    
    # File syncing capability is disabled by default. This feature is usually not required,
    # not useful and an extreme performance hit
    #$ActionFileEnableSync on
    
    # Include all config files in /etc/rsyslog.d/
    $IncludeConfig /etc/rsyslog.d/*.conf
    
    # Turn off message reception via local log socket;
    # local messages are retrieved through imjournal now.
    $OmitLocalLogging on
    
    # File to store the position in the journal
    $IMJournalStateFile imjournal.state
    
    
    #### RULES ####
    
    # Log all kernel messages to the console.
    # Logging much else clutters up the screen.
    #kern.*                                                 /dev/console
    
    # Log anything (except mail) of level info or higher.
    # Don't log private authentication messages!
    *.info;mail.none;authpriv.none;cron.none                /data/logs/messages
    
    # The authpriv file has restricted access.
    authpriv.*                                              /var/log/secure
    
    # Log all the mail messages in one place.
    mail.*                                                  -/var/log/maillog
    
    
    # Log cron stuff
    cron.*                                                  /var/log/cron
    
    # Everybody gets emergency messages
    *.emerg                                                 :omusrmsg:*
    
    # Save news errors of level crit and higher in a special file.
    uucp,news.crit                                          /var/log/spooler
    
    # Save boot messages also to boot.log
    local7.*                                                /var/log/boot.log
    
    
    # ### begin forwarding rule ###
    # The statement between the begin ... end define a SINGLE forwarding
    # rule. They belong together, do NOT split them. If you create multiple
    # forwarding rules, duplicate the whole block!
    # Remote Logging (we use TCP for reliable delivery)
    #
    # An on-disk queue is created for this action. If the remote host is
    # down, messages are spooled to disk and sent when it is up again.
    #$ActionQueueFileName fwdRule1 # unique name prefix for spool files
    #$ActionQueueMaxDiskSpace 1g   # 1gb space limit (use as much as possible)
    #$ActionQueueSaveOnShutdown on # save messages to disk on shutdown
    #$ActionQueueType LinkedList   # run asynchronously
    #$ActionResumeRetryCount -1    # infinite retries if host is down
    # remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional
    #*.* @@remote-host:514
    # ### end of the forwarding rule ###
    
    # 定义 logv2 日志模版
    $ModLoad omkafka
    
    global(parser.permitSlashInProgramName="on")
    
    # 文件输出模板
    template(name="CleanMsgBlank" type="list") {            #定义模板名称和类型
                property(name="msg" position.from="2" droplastlf="on")  # 删除 msg 前面的空格,同模板 CleanMsgFormat 功能
                constant(value="\n")
                }
    
    # 日志文件动态命名模板
    # template(name="LogV2DynFile" type="string" string="/data/logs/%programname%-%timereported:0:10:date-rfc3339%.log")
    # v2
    template(name="LogDynFile" type="string" string="/data/logs/%programname%-%timereported:0:10:date-rfc3339%.log")
    # nginx
    template(name="LogDynFileNginx" type="string" string="/data/logs/middleware/nginx/syslog/%HOSTNAME%/%timereported:0:10:date-rfc3339%.log")
    # v1
    template(name="LogV1DynFile" type="string" string="/data/logs/logv1/%programname:F,44:1%-%timereported:0:10:date-rfc3339%.log")
    
    # 日志转 json 格式模板
    template(name="logv2json" type="list" option.jsonf="on") {            #定义模板名称和类型
                property(outname="object" name="programname" field.delimiter="47" field.number="2" format="jsonf")  # field.delimiter指定分隔符 用 ascii 10进制 47 代表"/"
                property(outname="appName" name="programname" field.delimiter="47" field.number="3" format="jsonf")  # field.delimiter指定分隔符 用 ascii 10进制 47 代表"/"
                property(outname="hostname" name="programname" field.delimiter="47" field.number="4" format="jsonf")  # field.delimiter指定分隔符 用 ascii 10进制 47 代表"/"
                property(outname="containerName" name="programname" field.delimiter="47" field.number="5" format="jsonf")  # field.delimiter指定分隔符 用 ascii 10进制 47 代表"/"
                property(outname="message" name="msg" position.from="2" droplastlf="on" format="jsonf")  # 删除 msg 前面的空格,同模板 CleanMsgFormat 功能
                constant(outname="logver" value="logv2" format="jsonf")
                constant(outname="appType" value="docker" format="jsonf")
    }
    
    # 日志转 json 格式模板
    template(name="logv1json" type="list" option.jsonf="on") {            #定义模板名称和类型
                property(outname="hostname" name="programname" field.delimiter="47" field.number="2" format="jsonf")  # field.delimiter指定分隔符 用 ascii 10进制 47 代表"/"
                property(outname="containerName" name="programname" field.delimiter="47" field.number="3" format="jsonf")  # field.delimiter指定分隔符 用 ascii 10进制 47 代表"/"
                property(outname="message" name="msg" position.from="2" droplastlf="on" format="jsonf")  # 删除 msg 前面的空格,同模板 CleanMsgFormat 功能
                constant(outname="logver" value="logv1" format="jsonf")
                constant(outname="appType" value="docker" format="jsonf")
    }
    
    # json 日志发送到 kafka
    action(type="omkafka" broker=["127.0.0.1:9092"] topic="logv2" confParam=["compression.codec=snappy"] template="logv2json")
    # action(type="omkafka" broker=["127.0.0.1:9092"] topic="logv1" confParam=["compression.codec=snappy"] template="logv1json")
    
    # 服务端日志写入文件
    if $syslogtag startswith 'logv2' then ?LogDynFile;CleanMsgBlank
    
    # 终止其它文件输出日志
    & stop
    
    # 中间件日志写入文件
    if $syslogtag startswith 'middleware' then ?LogDynFile;CleanMsgBlank
    
    # 终止其它文件输出日志
    & stop
    
    # 中间件日志写入文件
    if $syslogtag startswith 'nginx' then ?LogDynFileNginx;CleanMsgBlank
    
    # 终止其它文件输出日志
    & stop
    
    # 'docker'为设docker启动时设置的tag名称
    if $syslogtag startswith 'docker' then ?LogV1DynFile;CleanMsgBlank
    # 终止其它文件输出日志
    & stop

    docker配置syslog

    --log-driver=syslog \
    --log-opt syslog-address=tcp://127.0.0.1:514 \
    --log-opt tag="log/${OBJECT}/${APPNAME}/$(hostname)/{{.Name}}" \
    Tip

    配置的时候,tag的参数就代表了日志服务器的目录

    #高可用

    由于docker直接填写日志服务器的地址,当网络不通的时候,日志将会丢失

    故需要部署高可用,逻辑:

    dokcer发送日志到本地rsyslog,当远程rsyslog可用时发送到远程,当不可用时写本地文件

    Tip

    rsyslog默认转发的tag会截断到32位,需覆盖默认配置

    template (name="ForwardFormat" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME%%syslogtag:1:256%%msg:::sp-if-no-1st-sp%%msg%")
    action(type="omfwd" target="192.168.1.1" port="10514" protocol="tcp" template="ForwardFormat")

    #转发配置

    global(parser.permitSlashInProgramName="on")
    
    # 文件输出模板
    template(name="CleanMsgBlank" type="list") {            #定义模板名称和类型
                property(name="msg" position.from="2" droplastlf="on")  # 删除 msg 前面的空格,同模板 CleanMsgFormat 功能
                constant(value="\n")
                }
    template(name="ForwardFormat" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME%%syslogtag:1:256%%msg:::sp-if-no-1st-sp%%msg%")
    template(name="LogDynFile" type="string" string="/data/logs/%programname%-%timereported:0:10:date-rfc3339%.log")
    # 定义一个 ruleset
    ruleset(name="agent" queue.type="linkedList" queue.filename="fwdq") {
        # 服务端日志写入文件
        if $syslogtag startswith 'logv2' then {
            action(type="omfwd" target="172.168.3.103" port="10514" protocol="tcp" template="ForwardFormat" RebindInterval="4096")
    
            action(type="omfile" dynaFile="LogDynFile" template="CleanMsgBlank" action.execOnlyWhenPreviousIsSuspended="on")
            stop
        }
    }
    
    # 定义一个 input,将所有 514 端口收到的日志都应用 agent 这个 ruleset
    input(type="imtcp" port="514" ruleset="agent")
    Tip

    如果远程关闭立马写到本地文件

    如果远程恢复30s后发远程