流量採集任務分配算法

任務分配算法，主要是輪詢然後是 MAP利用數據結構， KEY裏封裝對象 再追加list  如 

Map<String, List<String>> old_map = new HashMap();

old_map.get(same.getIsoCode()).add(nodeSame.getCountryIp());

對於任務分配 非常有幫助

/*
 * Zenlayer.com Inc.
 * Copyright (c) 2014-2019 All Rights Reserved.
 */
package com.zenlayer.ad.nodetool;

import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.record.City;
import com.maxmind.geoip2.record.Continent;
import com.maxmind.geoip2.record.Country;
import com.maxmind.geoip2.record.Location;
import com.maxmind.geoip2.record.Postal;
import com.maxmind.geoip2.record.Subdivision;
import com.zenlayer.ad.jedis.RedisClient;
import com.zenlayer.ad.mapper.SnmpMapper;
import com.zenlayer.util.DataTransformation;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @author jack.li
 * @date 2019-10-30 18:07:30
 * @version $ Id: NodeToolService.java, v 0.1  admin Exp $
 *<1> 採用新的分配節點任務，按區域劃分，通過IP自動獲取所在地區 (BMC 交換機的分配方式)
 *
 *<2> 按着IP，平均分配到所有存活的節點上
 */
@Component("nodeToolService")
public class NodeToolService {

    @Resource
    RedisClient redisclient;
    @Autowired
    private SnmpMapper snmpMapper;
    @Autowired
    NodeIdcService nodeIdcService;

    //默認洛杉機的節點，備份採集所有失敗的交換機
    public static String laxIp = "68.8.8.194";

    public boolean getSnmpAll() throws IOException, GeoIp2Exception {

        //1 所有內網IP 不進行地址解析, 2 另外分配所有失敗交換機的備選節點
        List<Map<String, Object>> innerAllIpList = new ArrayList<>();

        //所有公網IP，需要解析地區位置，分配採集任務
        List<Map<String, Object>> publicIp = new ArrayList<>();

        //採集服務註冊的IP，往這裏分配交換機IP
        List<CountryModel> nodelist = new ArrayList<>();

        //中國的IP，記錄下來，跳過此IP去分配內網IP,用LIST防止有多箇中國服務器
        List<String> cn_ip_list = new ArrayList();

        Map<String, List<Map<String, Object>>> snmpAll = new HashMap<>();

        Map<String, List<Map<String, Object>>> bmclist = null;
        Map<String, List<Map<String, Object>>> idclist = null;
        try {
            AllIplistTask(publicIp, innerAllIpList, nodelist, cn_ip_list);
            if (publicIp.size() > 0 && nodelist.size() > 0) {
                bmclist = nodeIPtask(publicIp, innerAllIpList, nodelist, cn_ip_list);
            }
            if (nodelist.size() > 0) {
                idclist = nodeIdcService.getIdcNodeService(nodelist, innerAllIpList);
                nodelist.stream().forEach(countryModel -> {
                    snmpAll.put(countryModel.getCountryIp(), new ArrayList<>());
                });
                bmclist.forEach((k, v) -> {
                    snmpAll.get(k).addAll(v);
                });
                idclist.forEach((k, v) -> {
                    snmpAll.get(k).addAll(v);
                });
                snmpAll.forEach((s, maps) -> {
                    // JSONArray arraylist = new JSONArray(Collections.singletonList(maps));
                    redisclient.set(s, DataTransformation.parseListForMapsToJsonArrayStr(maps).toJSONString(), 0);
                    //System.out.println(DataTransformation.parseListForMapsToJsonArrayStr(maps).size());
                });
            }
            //內網地直 直接分配到洛杉機，採集專用所有
            if (innerAllIpList.size() > 0) {
                redisclient.set(laxIp, DataTransformation.parseListForMapsToJsonArrayStr(innerAllIpList).toJSONString(), 0);
            }
            System.out.println("====================================:SNMP任務調度結束:====================================：");
        } catch (IOException e) {
            e.printStackTrace();
            return false;
        }
        return true;

    }

    //封裝所有IP區域 過濾的方法
    public void AllIplistTask(List publicIps, List innerAllIpList, List nodelists, List cn_ip_lists) throws IOException {

        File database = getFileGeo();

        DatabaseReader reader = new DatabaseReader.Builder(database).build();
        List<Map<String, Object>> countryList = new ArrayList<>();

        List<Set<Map<String, Object>>> listset = new ArrayList(snmpMapper.getSnmpList());

        List<Map<String, Object>> list = new ArrayList(listset);

        //過濾掉所有爲空的IP
        List<Map<String, Object>> filtelistIP = null;
        filtelistIP = list.stream().filter(smap -> null != smap.get("ip") && !"".equals(smap.get("ip"))).collect(Collectors.toList());

        //所有公網IP，需要解析地區位置，分配採集任務
        List<Map<String, Object>> publicIp = publicIps;

        filtelistIP.stream().forEach(filterIP -> {
            if (internalIp(filterIP.get("ip").toString())) {
                //  innerAllIpList.add(filterIP);// 暫時去掉內網地址
            } else {
                publicIp.add(filterIP);
            }
        });

        publicIp.stream().forEach(imp -> {
            try {
                InetAddress ipAddress = InetAddress.getByName(imp.get("ip").toString());
                CityResponse response = reader.city(ipAddress);
                Continent continent = response.getContinent();
                Country country = response.getCountry();

                imp.put("countryName", country.getNames().get("zh-CN"));
                imp.put("isoCode", country.getIsoCode());
                imp.put("countryIp", imp.get("ip"));
                imp.put("code", continent.getCode());
                countryList.add(imp);
            } catch (IOException e) {
                e.printStackTrace();
            } catch (GeoIp2Exception e) {
                e.printStackTrace();
            }
        });
        //打印局域網IP
        System.out.println("innerAllIpList: " + innerAllIpList.size());
        System.out.println("publicIp: " + publicIp.size());
        System.out.println("===================打印需要分配給採集服務器IP地理位置=====================");
        //採集服務註冊的IP，往這裏分配交換機IP
        List<CountryModel> nodelist = nodelists;
        List<String> list_ip = redisclient.smembers("collet:node", 0);
        //刪除洛杉機節點，不分配任務，只作備選節點
        list_ip.remove(laxIp);

        //中國的IP記錄下來
        List<String> cn_ip_list = cn_ip_lists;
        if (null == list_ip) {
            return;
        }
        list_ip.stream().filter(s -> !StringUtils.isBlank(s)).forEach(nodeip -> {
            try {
                InetAddress ipAddress = InetAddress.getByName(nodeip);
                CityResponse response = reader.city(ipAddress);
                Continent continent = response.getContinent();

                Country country = response.getCountry();
                if (null != response) {
                    CountryModel model = new CountryModel();
                    model.setName(country.getNames().get("zh-CN"));
                    model.setIsoCode(country.getIsoCode());
                    model.setCountryIp(nodeip);
                    model.setCode(continent.getCode());
                    nodelist.add(model);
                    if (country.getIsoCode().equals("CN")) {
                        cn_ip_list.add(nodeip);
                    }
                    System.out.println(model.getName() + ", EN: " + model.getIsoCode() + " IP: " + model.getCountryIp());
                } else {
                    System.out.println("************************* response is null: " + response + " IP: " + nodeip);
                }
            } catch (IOException e) {
                e.printStackTrace();
            } catch (GeoIp2Exception e) {
                e.printStackTrace();
            }
        });
    }

    //分配給採集服務器的IP
    public Map<String, List<Map<String, Object>>> nodeIPtask(List<Map<String, Object>> publicIp, List<Map<String, Object>> innerIp,
                                                             List<CountryModel> nodelist, List<String> cn_ip_list) {

        //處理 沒有區域交換機的IP數據，所有日本，臺灣等IP
        List<Map<String, Object>> publicTaskList = new ArrayList<>();

        Map<String, List<Map<String, Object>>> listAllMap = new HashMap<String, List<Map<String, Object>>>();
        List<CountryModel> strlist = new ArrayList<>();

        //中國一個LIST  國外一個LIST，區分開來
        nodelist.stream().forEach(countryModel -> {

            listAllMap.put(countryModel.getCountryIp(), new ArrayList<>());

            //單獨給中國一個LIST
            List<Map<String, Object>> publicTaskCnmap = new ArrayList<>();
            // 除了中國之外的IP節點,並且有服務器區域的
            List<Map<String, Object>> publicTaskAll = new ArrayList<>();

            strlist.add(countryModel);
            for (int s = 0; s < publicIp.size(); s++) {
                if (countryModel.getIsoCode().equals(publicIp.get(s).get("isoCode")) && countryModel.getIsoCode().equals("CN")) {
                    publicTaskCnmap.add(publicIp.get(s));
                } else if (countryModel.getIsoCode().equals(publicIp.get(s).get("isoCode"))) {
                    publicTaskAll.add(publicIp.get(s));
                }
            }
            if (publicTaskCnmap.size() > 0) {
                listAllMap.put(countryModel.getCountryIp(), publicTaskCnmap);
            }
            if (publicTaskAll.size() > 0) {
                listAllMap.put(countryModel.getCountryIp(), publicTaskAll);
            }
        });

        //採集服務器重複的IP記錄，再分配,  isocode相同，但是IP不相同，說明不是同一個節點， 多個服務的時候，old_ip和oldNodeip有重複，因爲後面用KEY處理，重複的會覆蓋
        List<String> old_ip = new ArrayList<>();
        Map<String, List<String>> old_map = new HashMap();
        // 是否同一國家
        nodelist.stream().forEach(same -> {
            //同一個國家兩個節點以上，分配任務，多個節點輪詢分配
            if (!old_map.containsKey(same.getIsoCode())) {
                old_map.put(same.getIsoCode(), new ArrayList<>());
            }
            nodelist.stream().forEach(nodeSame -> {
                if (same.getIsoCode().equals(nodeSame.getIsoCode()) && !same.getCountryIp().equals(nodeSame.getCountryIp())) {
                    old_ip.add(nodeSame.getCountryIp());
                    old_map.get(same.getIsoCode()).add(nodeSame.getCountryIp());
                }
            });
        });

        //相同國家的的IP，重新再分配，先刪除重複的IP KEY健值
        Map<String, List<Map<String, Object>>> sameListIsocode = new HashMap<>();
        // 用重複的ID去 刪除KEY值
        listAllMap.forEach((s, maps) -> {
            old_ip.stream().forEach(s1 -> {
                if (s.equals(s1)) {
                    sameListIsocode.put(s, maps);
                    listAllMap.put(s, new ArrayList<>());
                }
            });
        });
        //平均分配兩個,一個區域CN 下面有多個IP，得到一個LIST值， 輪詢分到重複的IP上
        old_map.forEach((code, maps) -> {
            //有兩個節點相同的國家
            if (maps.size() > 0) {
                List<Map<String, Object>> ziNode = new ArrayList<>();
                ziNode = sameListIsocode.get(maps.get(0));
                int f = 0;
                for (int j = 0; j < ziNode.size(); j++) {
                    if (f >= maps.size()) {
                        f = 0;
                    }
                    listAllMap.get(maps.get(f)).add(ziNode.get(j));
                    f++;
                }
            }

        });

        //======================================== 這是處理沒有區域服務器IP的數據，比如日本臺灣等
        publicIp.stream().forEach(smp -> {
            //如果採集服務器有這個IP區域，就不進行分配，標記
            boolean tagst = false;
            for (int s = 0; s < strlist.size(); s++) {
                if (smp.get("isoCode").equals(strlist.get(s).getIsoCode())) {
                    tagst = true;
                }
            }
            if (!tagst) {
                publicTaskList.add(smp);
            }
        });

// 按大洲重新分配 
//沒有洲的IP
        List<Map<String, Object>> no_zhou_list = new ArrayList<>();//
        //按國家分配後，再按大洲分配，有大洲的ip
        List<Map<String, Object>> have_zhou_list = new ArrayList<>();//
        publicTaskList.stream().forEach(smp -> {
            //如果採集服務器有這個IP區域，就不進行分配，標記
            boolean tagst = false;
            for (int s = 0; s < strlist.size(); s++) {
                if (smp.get("code").equals(strlist.get(s).getCode())) {
                    tagst = true;
                }
            }
            if (!tagst) {
                no_zhou_list.add(smp);
            } else {
                have_zhou_list.add(smp);
            }
        });

        //按大洲分配，亞洲，歐洲等
        Map<String, List<String>> zhouPlan = new HashMap<>();
        nodelist.stream().forEach(ms -> {
            if (!zhouPlan.containsKey(ms.getCode())) {
                zhouPlan.put(ms.getCode(), new ArrayList<>());
            }
            zhouPlan.get(ms.getCode()).add(ms.getCountryIp());
        });
        //封裝洲的MAP,先封裝 後輪詢分配
        Map<String, List<Map<String, Object>>> zall = new HashMap<>();
        zhouPlan.forEach((k, v) -> {
            if (!zall.containsKey(k)) {
                zall.put(k, new ArrayList<>());
            }
            have_zhou_list.stream().forEach(zo -> {
                if (k.equals(zo.get("code"))) {
                    zall.get(k).add(zo);
                }
            });
            int zs = 0;
            List<Map<String, Object>> se = zall.get(k);
            for (int i = 0; i < se.size(); i++) {
                if (zs >= v.size()) {
                    zs = 0;
                }
                listAllMap.get(v.get(zs)).add(se.get(i));
                zs++;
            }
        });
        System.out.println("no_zhou_list-==========: " + no_zhou_list.size());
        //隨機分配任務到各洲， 之前是輪詢分配到國家
        int ns = 0;
        for (int y = 0; y < no_zhou_list.size(); y++) {
            if (ns >= nodelist.size()) {
                ns = 0;
            }
            listAllMap.get(nodelist.get(ns).getCountryIp()).add(no_zhou_list.get(y));
            ns++;
        }

/*    
        int ns = 0;
        for (int y = 0; y < publicTaskList.size(); y++) {
            if (ns >= nodelist.size()) {
                ns = 0;
            }
            listAllMap.get(nodelist.get(ns).getCountryIp()).add(publicTaskList.get(y));
            ns++;
        }
        //內網分配到國外的採集服務器上,除了中國服務器,避免國內兩臺以上服務器採集, 順序分配或輪詢分配
        int x = 0;
        for (int t = 0; t < innerIp.size(); t++) {
            boolean tabCn = false;  //國內IP 不參與分配內網
            if (x >= strlist.size()) {
                x = 0;
            }
            for (int e = 0; e < cn_ip_list.size(); e++) {
                if (listAllMap.get(strlist.get(x).getCountryIp()).equals(cn_ip_list.get(e))) {
                    tabCn = true;
                }
            }
            if (!tabCn) {
                listAllMap.get(strlist.get(x).getCountryIp()).add(innerIp.get(t));
            }
            x++;
        }*/

        return listAllMap;
    }

    //GeoIP2-City 數據庫文件,測試用於IP查詢位置
    public static void getLiteIPinit() throws IOException, GeoIp2Exception {
        File database = getFileGeo();
        // this.getClass().getClassLoader().getResource("").getPath();  得到的是 ClassPath的絕對URI路徑。

        // 創建 DatabaseReader對象
        DatabaseReader reader = new DatabaseReader.Builder(database).build();

        InetAddress ipAddress = InetAddress.getByName("18.8.6.22");

        CityResponse response = reader.city(ipAddress);

        Continent continent = response.getContinent();
        System.out.println(continent.getCode());            // 'US'
        System.out.println(continent.getName());               // 'United States'
        System.out.println(continent.getNames().get("zh-CN")); // '

        Country country = response.getCountry();
        System.out.println(country.getIsoCode());            // 'US'
        System.out.println(country.getName());               // 'United States'
        System.out.println(country.getNames().get("zh-CN")); // '美國'

        Subdivision subdivision = response.getMostSpecificSubdivision();
        System.out.println(subdivision.getName());    // 'Minnesota'
        System.out.println(subdivision.getIsoCode()); // 'MN'

        City city = response.getCity();
        System.out.println(city.getName()); // 'Minneapolis'

        Postal postal = response.getPostal();
        System.out.println(postal.getCode()); // '55455'

        Location location = response.getLocation();
        System.out.println(location.getLatitude());  // 44.9733
        System.out.println(location.getLongitude()); // -93.2323

    }

    // 判斷IP是否是區域網
    public static boolean internalIp(String ip) {

        byte[] addr = new byte[0];
        try {
            //將ip字符串轉爲byte數組,注意:ip不可以是域名,否則會進行域名解析
            addr = InetAddress.getByName(ip).getAddress();
        } catch (UnknownHostException e) {
            e.printStackTrace();
        }
        final byte b0 = addr[0];
        final byte b1 = addr[1];
        //10.x.x.x/8
        final byte SECTION_1 = 0x0A;
        //172.16.x.x/12
        final byte SECTION_2 = (byte) 0xAC;
        final byte SECTION_3 = (byte) 0x10;
        final byte SECTION_4 = (byte) 0x1F;
        //192.168.x.x/16
        final byte SECTION_5 = (byte) 0xC0;
        final byte SECTION_6 = (byte) 0xA8;
        switch (b0) {
            case SECTION_1:
                return true;
            case SECTION_2:
                if (b1 >= SECTION_3 && b1 <= SECTION_4) {
                    return true;
                }
            case SECTION_5:
                switch (b1) {
                    case SECTION_6:
                        return true;
                }
            default:
                return false;
        }
    }

    // 第二種算法，平均分配IP到所有存活節點服務上
    public void avgIptask() throws IOException {

        //所有內網IP 不進行地址解析
        List<Map<String, Object>> innerAllIpList = new ArrayList<>();

        //所有公網IP，需要解析地區位置，分配採集任務
        List<Map<String, Object>> publicIp = new ArrayList<>();

        //採集服務註冊的IP，往這裏分配交換機IP
        List<CountryModel> nodelist = new ArrayList<>();

        //中國的IP，記錄下來，跳過此IP去分配內網IP,用LIST防止有多箇中國服務器
        List<String> cn_ip_list = new ArrayList();

        AllIplistTask(publicIp, innerAllIpList, nodelist, cn_ip_list);

        Map<String, List<Map<String, Object>>> listAllMap = new HashMap<String, List<Map<String, Object>>>();
        nodelist.stream().forEach(countryModel -> {
            listAllMap.put(countryModel.getCountryIp(), new ArrayList<>());
        });

        // 全並兩個 IP 的LIST 數據
        List<Map<String, Object>> allIplist = new ArrayList<>();
        allIplist.addAll(publicIp);
        allIplist.addAll(innerAllIpList);

        int x = 0;
        int nodeSize = nodelist.size();
        for (int t = 0; t < allIplist.size(); t++) {
            boolean tabCn = false;  //國內IP 不參與分配內網
            if (x >= nodeSize) {
                x = 0;
            }
            listAllMap.get(nodelist.get(x).getCountryIp()).add(allIplist.get(t));
            x++;
        }
        //把分配好任務的LIST保存到採集服務的IP上   放到redis 的節點上
        listAllMap.forEach((s, maps) -> {
            // JSONArray arraylist = new JSONArray(Collections.singletonList(maps));
            // System.out.println(arraylist.get(0));
            // redisclient.set(s, DataTransformation.parseListForMapsToJsonArrayStr(maps).toJSONString(), 0);
            System.out.println(DataTransformation.parseListForMapsToJsonArrayStr(maps).size());
        });

    }

    public static File getFileGeo() {
        // 容器裏運行的時候，找不到文件路徑  本地運行用下面路徑
        File database = new File(Thread.currentThread().getContextClassLoader().getResource("GeoLite2-City.mmdb").getPath());
        //打包上線，用下面路徑
        //File database = new File(System.getProperty("user.dir") + "/GeoLite2-City.mmdb");
        return database;
    }

    public static void main(String[] args) throws IOException {
        //這裏因爲是去讀取本地的純真庫，所以有一個IO異常拋出
    /*    QQWry wry = new QQWry();
        NodeToolService nodeTool = new NodeToolService();
        List<Map<String, Object>> list = nodeTool.getSnmpAll();
        List<IPZone> listipzone = new ArrayList<>();
        list.stream().forEach(ipmap -> {
            listipzone.add(wry.findIP(ipmap.get("ip").toString()));
        });
        //  IPZone zone = wry.findIP("123.123.123.123");
        listipzone.stream().forEach(ipZone -> System.out.println(ipZone.getMainInfo()));
        //  System.out.println(zone.getMainInfo());
        // System.out.println(zone.getSubInfo());

        測試IP  加到LIST測試區域
        中國, EN: CN IP: 13
        南非, EN: ZA IP: 126.60
        美國, EN: US IP: 107.60
        德國, EN: DE IP: 107.40
        新加坡, EN: SG IP: 12.60
        俄羅斯, EN: RU IP: 128.60
        美國, EN: US IP: 107.49

        */
        try {
            getLiteIPinit();
        } catch (GeoIp2Exception e) {
            e.printStackTrace();
        }
        //180.168.61.243
        //boolean ipbyte = internalIp("127.0.0.1");
        //if (ipbyte) {
        //    System.out.println("10.68.1.243");
        //} else {
        //    System.out.println("判斷局域網IP:" + "127.0.0.1");
        //}
    }

}