基於openResty + KafkaMQ + SpringBoot+Mysql打造業務數據安全審計平臺

設計思路：

1、通過openresty-1.15.8.1 引入lua-resty-kafka 插件（自帶lua腳步支持），定義採集客戶端URL返回的response_body具體信息（前後端分離項目，只採集ajax請求即可）的.conf文件

 

2、將採集信息已日誌形式輸出，同時生產消息至KafkaMQ

3、SpringBoot 開發KafkaMQ消費端，消費消息（獲取自定義的規則信息，進行匹配。將匹配結果存儲至Mysql）

 

具體實現nginx.conf

#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

daemon off;
events {
    worker_connections  1024;
}

http {

    log_format main  escape=json '{ "@timestamp": "$time_local", '
                         '"x_requested_with":"$http_x_requested_with", '
                         '"remote_addr": "$remote_addr", '
                         '"upstream_addr": "$upstream_addr",'
                         '"remote_user": "$remote_user", '
                         '"body_bytes_sent": "$body_bytes_sent", '
                         '"request_time": "$request_time", '
                         '"status": "$status", '
                         '"request": "$request", '
                         '"request_method": "$request_method", '
                         '"http_referrer": "$http_referer", '
                         '"body_bytes_sent":"$body_bytes_sent", '
                         '"http_x_forwarded_for": "$http_x_forwarded_for", '
                         '"host":""$host",'
                         '"remote_addr":""$remote_addr",'
                         '"http_user_agent": "$http_user_agent",'
                         '"http_uri": "$uri",'
                         '"http_host":"$http_host",'
                         '"req_body":"$resp_body" }';
                         
    include       mime.types;
    default_type  application/octet-stream;

    charset  utf-8;
  server_names_hash_bucket_size 128;
  client_header_buffer_size 32k;
  large_client_header_buffers 4 64k;
  client_max_body_size 80m;
  sendfile on;
  tcp_nopush     on;
  keepalive_timeout 65;
  server_tokens off;
  fastcgi_connect_timeout 600;
  fastcgi_send_timeout 600;
  fastcgi_read_timeout 600;
  fastcgi_buffer_size 16k;
  fastcgi_buffers 16 16k;
  fastcgi_busy_buffers_size 16k;
  fastcgi_temp_file_write_size 16k;
  tcp_nodelay on;
  gzip on;
  gzip_min_length  1k;
  gzip_buffers     4 16k;
  gzip_http_version 1.0;
  gzip_comp_level 2;
  gzip_types   text/plain text/javascript application/x-javascript text/css application/xml  image/jpeg image/gif image/png;
  gzip_vary on;
  proxy_ignore_client_abort on;

  # 配置lua依賴庫地址 
  lua_package_path "F:/cg-test/openresty-1.15.8.1-win64/lualib/kafka/?.lua;;";

  include ../conf.d/*.conf;
  
    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    

    
}
 

base.conf

server {
        listen       80;
        server_name  localhost;
        #access_log  logs/access.log main ;
        error_log   logs/error.log;

        # lua代碼
        set $resp_body "";
        lua_need_request_body on;

        location / {
            #root   /usr/local/openresty/nginx/html/;
            alias F:/chengangGit/fintech-DP/JY-pangu/pangu-html/;
            index  index.html index.htm;
            error_page 405 =200 http://$host$request_uri;
        }

        location ^~ /api/{
           log_escape_non_ascii off;
        

           proxy_pass  http://127.0.0.1:8092/api/;
           
           access_log  logs/bizaccess.log main ;
           #local resp_body = string.sub(ngx.arg[1], 1, 1000)
           body_filter_by_lua '
               local resp_body = ngx.arg[1]
               ngx.ctx.buffered = (ngx.ctx.buffered or "") .. resp_body
               if ngx.arg[2] and "XMLHttpRequest" == ngx.var.http_x_requested_with then
                  ngx.var.resp_body = ngx.ctx.buffered
               end
               ';
           # 使用log_by_lua 包含lua代碼,因爲log_by_lua指令運行在請求最後且不影響proxy_pass機制
           
           log_by_lua '  
            -- 引入lua所有api  
            local cjson = require "cjson"  
            local producer = require "resty.kafka.producer"  
            -- 定義kafka broker地址，ip需要和kafka的host.name配置一致  
            local broker_list = {  
                { host = "172.18.101.128", port = 9092 },  
            }  
            -- 定義json便於日誌數據整理收集  
            local log_json = {}  
            log_json["uri"]=ngx.var.uri  
            log_json["args"]=ngx.var.args  
            log_json["host"]=ngx.var.host  
            log_json["request_body"]=ngx.var.request_body  
            log_json["remote_addr"] = ngx.var.remote_addr  
            log_json["remote_user"] = ngx.var.remote_user  
            log_json["time_local"] = ngx.var.time_local  
            log_json["status"] = ngx.var.status  
            log_json["body_bytes_sent"] = ngx.var.body_bytes_sent  
            log_json["http_referer"] = ngx.var.http_referer  
            log_json["http_user_agent"] = ngx.var.http_user_agent  
            log_json["http_x_forwarded_for"] = ngx.var.http_x_forwarded_for  
            log_json["upstream_response_time"] = ngx.var.upstream_response_time  
            log_json["request_time"] = ngx.var.request_time  
            log_json["resp_body"] = ngx.var.resp_body
            -- 轉換json爲字符串  
            local message = cjson.encode(log_json);  
            -- 定義kafka異步生產者  
            local bp = producer:new(broker_list, { producer_type = "async" })  
            -- 發送日誌消息,send第二個參數key,用於kafka路由控制:  
            -- key爲nill(空)時，一段時間向同一partition寫入數據  
            -- 指定key，按照key的hash寫入到對應的partition 僅記錄ajax請求
            if "XMLHttpRequest" == ngx.var.http_x_requested_with then
                local ok, err = bp:send("openresty-log", nil, message)  

                if not ok then  
                    ngx.log(ngx.ERR, "kafka send err:", err)  
                    return  
                end
                if ok then
                    ngx.log(ngx.INFO, "kafka send ok:", err)  
                    return
                end
            end
            
          ';  
        
                   
        }

        location /healthz {
                        access_log off;
                        return 200;
                }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }

 

SpringBoot MQ消費端代碼參考：

KafkaConsumer.java

/**
 * <p>className: KafkaConsumer</p>
 * <p>description: MQ消費者 設置項目啓動時就開始消費</p>
 *
 * @author CG
 * @version 1.0.0
 * @date 2019-08-27 10:00
 */

@Lazy(false)
@Component("com.mq.kafka.KafkaConsumer")
public class KafkaConsumer {
    
    private static Logger logger = LoggerFactory.getLogger(KafkaConsumer.class);

    @Value("${kakfa.mq.topic:openresty-log}")
    private String mqTopic ;
    @Autowired
    private ISysDictDataService dictDataService;
    @Autowired
    private IBizLogMqService bizLogMqService;

    @KafkaListener(topics= {"${kakfa.mq.topic:openresty-log}"})
    public void consumerMsg(ConsumerRecord<?,?> record){
        logger.info("------------------------------------------consumerMsg---------------------------------");
        SysDictData dictData = new SysDictData();
        dictData.setDictType("my_rule");
        List<SysDictData> rules = dictDataService.selectDictDataList(dictData);

        Optional<?> kafkaMsg = Optional.ofNullable(record.value());
        if(kafkaMsg.isPresent()){
            Object msg = kafkaMsg.get();

            logger.info("-------record:{}",record);
            logger.info("-------msg:{}",msg);
            String msgStr = msg.toString();
            JSONObject msgObj = JSONObject.parseObject(msgStr);
            //TODO 1 通過消息獲取 cookie獲取用戶信息
            //2 將消息內容寫入 DB 支持可視化查詢 消息內容
            BizLogMq bizLogMq = new BizLogMq();
            bizLogMq.setStatus("1");

            //消息內容大小
            bizLogMq.setExt1(String.valueOf(record.serializedValueSize()));

            if(record.serializedValueSize() >0){
                bizLogMq.setUri(msgObj.getString("http_referer"));
                bizLogMq.setRemoteUser(msgObj.getString("remote_user"));
                bizLogMq.setArgs(msgObj.getString("request_body"));
                bizLogMq.setCreateBy("0");
                bizLogMq.setMqTitle(String.valueOf(msgObj.getString("host")));
                bizLogMq.setExt2(msgObj.getString("remote_addr"));
            }
            //3 支持規則匹配 並完成消息自定義的標識
            StringBuffer ruleRemark = new StringBuffer(String.valueOf(record.offset()));
            if(rules != null && rules.size() > 0){
                for (int i =0;i < rules.size(); i++){
                    SysDictData rule = rules.get(i);
                    String dictVal = rule.getDictValue();
                    if(StringUtils.isNotEmpty(dictVal) ){
                        ruleRemark.append(",").append(this.matchRexgex(dictVal,msgStr));
                    }
                }
            }

            //內配符合規則
            if(ruleRemark.toString().contains(",1")){
                bizLogMq.setMqType("1");
            }

            bizLogMq.setRemark(ruleRemark.toString());
            if(msgStr.length() >= 4000){
                msgStr = msgStr.substring(0, 3800);
            }
            bizLogMq.setMqContent(msgStr);


            bizLogMqService.insertBizLogMq(bizLogMq);
            //4 支持可視化規則配置，並可以自動歷史匹配並完成數據標識
        }
    }

    public int matchRexgex(String rex,String src){
        boolean exists = false;
        exists = Pattern.matches(rex, src);
        if(exists){
            return 1;
        }else{
            return 0;
        }
    }
    public boolean  matchRexgex(String src){
        boolean exists = false;
        String rex = "";
        exists = Pattern.matches(rex, src);

        return exists;
    }

    public static void main(String [] args) {
//驗證手機號 .*((17[0-9])|(14[0-9])|(13[0-9])|(15[^4,\D])|(18[0,5-9]))\d{8}.*
//驗證郵箱 .*([a-z0-9A-Z]+[-|\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\.)+[a-zA-Z]{2,}.*
//驗證身份證 .*(\d{17}).*
//客戶姓名：TODO
        String rex = ".*([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}.*";
        //".*(\\d{17}).*";//".*((17[0-9])|(14[0-9])|(13[0-9])|(15[^4,\\D])|(18[0,5-9]))\\d{8}.*";//".*[0-9]{11}.*";
        String src = "{\"host\":\"127.0.0.1\",\"time_local\":\"10\\/Sep\\/2019:14:22:19 +0800\",\"resp_body\":\"{\\r\\n  \\\"retCode\\\" : \\\"0000\\\",\\r\\n  \\\"retDesc\\\" : \\\"成功\\\",\\r\\n  \\\"responseBody\\\" : {\\r\\n    \\\"pageParameter\\\" : {\\r\\n      \\\"pageSize\\\" : 10,\\r\\n      \\\"currentPage\\\" : 1,\\r\\n      \\\"totalPage\\\" : 1,\\r\\n      \\\"totalCount\\\" : 1\\r\\n    },\\r\\n    \\\"result\\\" : [ {\\r\\n      \\\"id\\\" : 4,\\r\\n      \\\"configName\\\" : \\\"1\\\",\\r\\n      \\\"configCode\\\" : \\\"1\\\",\\r\\n      \\\"configValue\\\" : \\\"[email protected]\\\",\\r\\n      \\\"configType\\\" : \\\"1\\\",\\r\\n      \\\"validateState\\\" : \\\"1\\\"\\r\\n    } ]\\r\\n  }\\r\\n}\",\"http_referer\":\"http:\\/\\/127.0.0.1\\/pangu\\/platform\\/sysConfig\\/querySysConfig.html\",\"status\":\"200\",\"request_time\":\"0.026\",\"remote_addr\":\"127.0.0.1\",\"request_body\":\"{\\\"pageParameter\\\":{\\\"currentPage\\\":\\\"1\\\",\\\"pageSize\\\":\\\"10\\\",\\\"selectRowIndex\\\":\\\"undefined\\\"},\\\"searchParams\\\":{\\\"dto\\\":{\\\"configName\\\":\\\"\\\",\\\"configCode\\\":\\\"\\\",\\\"configValue\\\":\\\"[email protected]\\\",\\\"configType\\\":\\\"\\\"}}}\",\"uri\":\"\\/api\\/sysConfig\\/searchByPage\\/v1\",\"args\":\"reqMsgPage\",\"http_user_agent\":\"Mozilla\\/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/75.0.3770.100 Safari\\/537.36\",\"body_bytes_sent\":\"421\",\"upstream_response_time\":\"0.026\"}";

        KafkaConsumer test = new KafkaConsumer();
        System.out.println("==========="+test.matchRexgex(rex,src));

    }