openresty+redis實現 redis存儲upstream

任務描述

1. upstream的server實例存儲在redis中，爲openresty設置一個location，來觸發加載server的動作
2. 訪問upstream時，輪詢內部的server
3. 不包括健康檢測，我認爲既然你的server都存儲在redis裏面了，健康檢測也應該獨立於openresty，發現壞節點，重新加載
   

技術描述

ngx.balancer      ngx.shared.DICT    resty.redis 

部署環境描述

1. http://10.161.4.61:80/  後端實例
2. http://10.161.4.63:80/  後端實例
3. http://10.161.4.63:8000/  openresty
4. 10.161.4.63 6379   redis
   

步驟

部署redis

啓動一個redis實例，沒有特殊要求，能訪問就行

設置openresty

修改配置文件

worker_processes  3;
error_log logs/error.log;
events {
    worker_connections  30;
}
http {
	lua_shared_dict  servers 10m;#全局變量，存儲upstream，根據你的後端節點數量，設置大小
	lua_shared_dict  now_server 1m;#全局變量，用於輪詢，記錄現在是哪個節點服務
	lua_shared_dict  lens 1m;#全局變量，記錄後端節點的數量
	upstream  tomcat17 {#這個upstream用於對比測試
    		server   10.161.4.63:80 weight=1;
    		server   10.161.4.61:80 weight=1;
    		keepalive 10;
        }
    upstream test{#動態加載的upstream
    	server   192.168.0.18:80 weight=1;#此處必須有一個server，否則會啓動失敗，可以胡亂寫一個
    	balancer_by_lua_block {
    		local set_peer = require "set_peer"
			set_peer.run_peer()
     	}
     	keepalive 10;
    }
    #init_worker_by_lua_block {
	#		local set_peer = require "set_peer"
	#		set_peer.insert_server()   
	#}
	server{
		listen 8000;
		location ~ ^/api/([-_a-zA-Z0-9/]+) {
			access_by_lua_file work/lua/check_access.lua;
			content_by_lua_file work/lua/$1.lua;
		}
		location = /tomcat17 {#用於對比測試的location
		     proxy_pass http://tomcat17/;
		}
		location = /insert {#用於加載server的location
			content_by_lua_block{
				local set_peer = require "set_peer"
				set_peer.insert_server()
			}
		}
		location = /test_run {#用於測試的upstream
			proxy_pass http://test/;
		        
		}

		location = /status {
			stub_status;
		}
	}
}

lualib或者你自己的lua目錄下創建，set_peer.lua 腳本（剛學lua沒多久，寫的有點水）

local balancer = require "ngx.balancer"
local now_server = ngx.shared.now_server;
local servers = ngx.shared.servers;
local server_len = ngx.shared.lens;
local redis = require "resty.redis"

local _M = { _VERSION = '1.0' }

local function con_redis()
	local red = redis:new()
	red:set_timeout(1000)
	local ok, err = red:connect("127.0.0.1", 6379)--注意此處
    if not ok then
        ngx.log(ngx.ERR,"failed to connect: ", err)
        return ngx.exit(500)
    else
    	return red
    end
end

local function set_peer(host,port)
	if not host and not port then
	    ngx.log(ngx.ERR, " host and port can not be nil")
        return ngx.exit(500)
	end

	local ok, err = balancer.set_current_peer(host, port)
    if not ok then
        ngx.log(ngx.ERR, "failed to set the current peer: ", err)
        return ngx.exit(500)
    end
end

local function split( str,reps)
    local resultStrList = {}
    string.gsub(str,'[^'..reps..']+',function ( w )
        table.insert(resultStrList,w)
    end)
    return resultStrList
end

local function test_server()
	 local value, err = servers:get(1)
	 if  not value then
	 	ngx.log(ngx.ERR, "no server found: ", err)
	 	ngx.status=ngx.HTTP_INTERNAL_SERVER_ERROR 
	 	return ngx.say("no server found")
	 else
	 	return true
	 end
end

local function fail_say( fail_str )
	-- body
	ngx.log(ngx.ERR, fail_str)
	ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR 
	return ngx.say(fail_str)
end

local function get_lens()
	-- body
	local len, flags = server_len:get("len")
	if not len then
		local fail_str = "failed to get len "..flags
		fail_say(fail_str)
	end
	return len
end

local function get_nums()
	local value = now_server:get("nums")
	if not value then
		local fail_str = 'reset  nums  to 1'
		ngx.log(ngx.ERR, fail_str)
		local ok, err = now_server:set("nums",1)
		if not ok then
			fail_str = 'failed to set nums '..err
			fail_say(fail_str)
		else
			return -1
		end
	end
	return value
end

local function round_server()
	-- body
	local value = get_nums()
	if not value then
		return
	end
	--ngx.log(ngx.ERR, '---->','ago',value)
	local len = get_lens()
	if not len then
		return
	end
	if value+1 >= len then
		local ok, err = now_server:set("nums",1)
		if not ok then
			fail_str = 'failed to set nums '..err
			fail_say(fail_str)
		else
			return 1
		end
	else
		local ok, err = now_server:set( "nums", 2+value )
		if not ok then
			fail_str = 'failed to set nums '..err
			fail_say(fail_str)
		else
			return 2+value
		end
	end 
end

local function  handle_redis(red)
	if not red then
		return 
	end
    local res, err = red:smembers("servers")
    if not res then
        ngx.log(ngx.ERR,"failed to smembers servers: ", err)
        return ngx.exit(500)
    else
    	return res
    end

end

function _M.insert_server()
	-- body
	local red = con_redis()
	if not red then
		return
	end
	local red_result = handle_redis(red)
	--ngx.log(ngx.ERR,'-->',red_result[1])
	if not red_result then
		red:close()
		return
	end
	local nums = 0
	for  k,v in pairs(red_result) do
		local resultStrList = split(v,':')
		--ngx.log(ngx.ERR,'-->', resultStrList[1], resultStrList[2])
		if  resultStrList[1] and  resultStrList[2] then
			local res, err = servers:set(2*k-1, resultStrList[1])
			if not res then
        		ngx.log(ngx.ERR,"failed to set servers first: ", err)
        		red:close()
        		return ngx.exit(500)
        	end
        	--ngx.log(ngx.ERR,'-->',2*k-1,'--->',resultStrList[1])
        	--ngx.log(ngx.ERR,'-->',2*k,'--->',resultStrList[2])
        	local res1, err1 = servers:set(2*k, resultStrList[2])
        	if not res1 then
        		ngx.log(ngx.ERR,"failed to set servers second: ", err1)
        		local res2, err2 = servers:rpop(k)
        		if not res2 then
        			ngx.log(ngx.ERR,"failed to rpop servers first: ", err2)
        		end
        		red:close()
        		return ngx.exit(500)
        	else
        		nums = nums+1
        		--ngx.log(ngx.ERR,'-->', nums)
        	end
		end
	end
	local res, err = server_len:set('len', nums*2)
	if not res then
        ngx.log(ngx.ERR,"failed to set servers first: ", err)
        return ngx.exit(500)
    else
    	ngx.say('ok,all is ', nums)
    	return true
    end
end

function _M.run_peer()
	local ok = test_server()
	if not ok then
		return		
	end
	local len = round_server()
	if not len then
		return		
	end
	ngx.log(ngx.ERR,'--->', len)--可以去掉，調試時使用
	local host ,err = servers:get(len)
	if not host then
		ngx.log(ngx.ERR,"failed to get server-host: ", err)
        return ngx.exit(500)
	end
	--ngx.log(ngx.ERR,"get server: ", host, '----', len+1)
	local port ,err = servers:get(len+1)
	if not port then
		ngx.log(ngx.ERR,"failed to get server-port: ", err)
        return ngx.exit(500)
	end
	ngx.log(ngx.ERR, "set peer ", host, ':', port) --可以去掉，調試時使用
	set_peer(host, port)
end

return _M

重啓openresty

重啓一下openresty，啓動時監控一下errlog,確保啓動成功

在redis內插入upstream的server

SADD servers '10.161.4.63:80' '10.161.4.61:80'

動態加載

curl -X GET http://10.161.4.63:8000/insert

正常的時候會得到 OK，all is 2 的句子

訪問測試

連續兩次執行，curl -X GET http://10.161.4.63:8000/test_run，查看結果是不是輪詢

簡單的壓力測試

簡單測試了一下，覺得性能差不多，僅供參考呀

----------------------------------------------------------------------------------------------------------------------------------

就寫到這了，如果有問題，歡迎來qq羣 630300475 一起聊聊