nginx整合Kafka

nginx整合Kafka

需求: 將網站產生的用戶日誌使用通過nginx寫入Kafka中,不通過log文件和flume採集

• 前端測試代碼: 使用ajax發送用戶數據

  /**
   * 生命週期函數--監聽頁面初次渲染完成
   */
  onReady: function () {
    //在這個事件中，記錄用戶的行爲，然後發送到後臺服務器
    //獲取當前位置
    wx.getLocation({
      success: function (res) {
        //緯度
        var lat = res.latitude;
        //經度
        var log = res.longitude;
        //從本地存儲中取出唯一身份標識
        var openid = wx.getStorageSync('openid')
        console.info(openid)
        //發送request向mongo中添加數據（添加一條文檔（json））
        wx.request({
          //用POST方式請求es可以只指定index和type，不用指定id
          // 向nginx服務器發送消息,將消息寫入kafka中
          url: "http://node01/kafka/user",
          data: {
            time: new Date(),
            openid: openid,
            lat: lat,
            log: log
          },
          method: "POST"
        })
      },
  })

• 下載安裝nginx,這裏需要使用kafka依賴和kafka插件,具體參照
  • https://github.com/brg-liuwei/ngx_kafka_module
  • 由於nginx是由c語言編寫,所以下載kafka插件也需要進行編譯安裝

# 先安裝git
yum install git
# 安裝nginx

# kafka依賴
git clone https://github.com/edenhill/librdkafka
cd librdkafka
# 安裝編譯依賴
yum install -y gcc gcc-c++ pcre-devel zlib-devel
./configure
make
sudo make install

# kafka整合nginx所需插件安裝編譯
git clone https://github.com/brg-liuwei/ngx_kafka_module
# cd /path/to/nginx
./configure --add-module=/path/to/ngx_kafka_module
make
sudo make install
# or, use `sudo make upgrade` instead of `sudo make install`

# 啓動nginx會出現如下錯誤
[root@node01 nginx]# sbin/nginx -t
sbin/nginx: error while loading shared libraries: librdkafka.so.1: cannot open shared object file: No such file or directory
# 需要加載so庫
echo "/usr/local/lib" >> /etc/ld.so.conf
ldconfig

# 配置nginx.conf
#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;
#pid        logs/nginx.pid;
events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';
    #access_log  logs/access.log  main;
    sendfile        on;
    #tcp_nopush     on;
    #keepalive_timeout  0;
    keepalive_timeout  65;
    #gzip  on;
    
    kafka;
    kafka_broker_list node01:9092 node02:9092 node03:9092; 	
    
    server {
        listen       80;
        server_name  node01;
        #charset koi8-r;
        #access_log  logs/host.access.log  main;
		
		# 配置該路徑會將發送到nginx的數據寫入kafka中的track主題
    	location = /kafka/track {
                kafka_topic track;
        }

    	location = /kafka/user {
                kafka_topic user;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

# 啓動kafka
# 後臺啓動
nohup bin/kafka-server-start.sh config/server.properties 2>&1 &
# 停止命令
bin/kafka-server-stop.sh
# 創建Topic
bin/kafka-topics.sh --create --zookeeper node01:2181 --replication-factor 2 --partitions 3 --topic track
# 消費數據
bin/kafka-console-consumer.sh --from-beginning --topic track  --zookeeper node01:2181,node02:2181,node03:2181

# 測試發送請求
curl localhost/kafka/track -d "message send to kafka topic"

[root@node01 nginx]# curl localhost/kafka/track -d "message send to kafka topic"

# 我們可以發現消費者可以消費到
[root@node02 kafka_2.11-0.10.0.0]# bin/kafka-console-consumer.sh --from-beginning --topic track  --zookeeper node01:2181,node02:2181,node03:2181
message send to kafka topic

• 啓動web項目發送log請求發現可以消費到,對應ajax中url: “http://node01/kafka/user”