springboot整合hbase

前言

	因公司業務場景，原本使用phoenix數據庫（hbase的一個sql層）的查詢服務需要改爲直連hbase，因爲phoenix建表
	映射到hbase這個過程需要的時間太長了，大數據組的同事是這麼說的，也許是我們服務器資源不夠吧。原本寫的很
	nice的sql要全部重構了！

java連接hbase注意事項

1、對hbase不熟悉的建議先大致瞭解一下它的基本原理及重要組成部分，它是一個非關係型的分佈式存儲系統
2、選擇正確版本的jar，根據hbase的版本選擇，實在不知道的話可以去hbase的安裝目錄裏找hbase-client.jar
3、注意數據入庫時的編碼與查詢的時候解碼的格式是否一致，否則會出現亂碼現象
4、查詢時注意檢查和確定幾個關鍵：表名、列族、列名（區分大小寫），列名的大小寫弄錯將導致過濾器失效！
5、本文使用的是原生的java的api，除此之外可以使用與spring整合的hbaseTemplate操作hbase，有過一層小小的封裝

一、導入相應的jar包

        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>1.2.0</version>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>log4j</groupId>
                    <artifactId>log4j</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>javax.servlet</groupId>
                    <artifactId>servlet-api</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

注：因爲我項目中有引用phoenix-core.jar，裏面包含了hbase相關的包所以不需要引用這個包了

二、hbase相關配置文件搞起來（application.properties）

hbase.zookeeper.quorum=hadoop-02,hadoop-03,hadoop-04
hbase.zookeeper.property.clientPort=2181
hbase.rootdir=/hbase

三、添加讀取配置文件的類

@Component
public class HbaseConfig {

    public static String quorum;
    public static String zkPort;
    public static String rootdir;

    public static final String COLUMN_FAMILY = "info";
    public static final String TABLE_NAME_RESUME_LIST = "ODS_RESUME_LIST_PHOENIX";
    public static final String TABLE_NAME_CV_RESUME_INFO = "ODS_RESUME_LIST_PHOENIX";

    @Value("${hbase.zookeeper.quorum}")
    public void setQuorum(String quorum) {
        this.quorum = quorum;
    }

    @Value("${hbase.zookeeper.property.clientPort}")
    public void setZkPort(String zkPort) {
        this.zkPort = zkPort;
    }

    @Value("${hbase.rootdir}")
    public void setRootdir(String rootdir) {
        this.rootdir = rootdir;
    }
}

四、編寫連接hbase工具類

@Component
public class HbaseUtils {

    private Logger LOGGER = LoggerFactory.getLogger(this.getClass());

    private static Configuration conf = HBaseConfiguration.create();
    private static ExecutorService pool = Executors.newScheduledThreadPool(20); // 設置連接池
    private static Connection connection = null;
    private static HbaseUtils instance = null;
    private static Admin admin = null;

    private HbaseUtils() {
        if (connection == null) {
            try {
                conf.set("hbase.zookeeper.quorum", HbaseConfig.quorum);
                conf.set("hbase.zookeeper.property.clientPort", HbaseConfig.zkPort);
                conf.set("hbase.rootdir", HbaseConfig.rootdir);
                connection = ConnectionFactory.createConnection(conf, pool);
                admin = connection.getAdmin();
            } catch (IOException e) {
                LOGGER.error("HbaseUtils實例初始化失敗！錯誤信息爲：" + e.getMessage(), e);
            }
        }
    }

    // 簡單單例方法，如果autowired自動注入就不需要此方法
    public static synchronized HbaseUtils getInstance() {
        if (instance == null) {
            instance = new HbaseUtils();
        }
        return instance;
    }

    public static Connection getConn() {
        return connection;
    }

    public int countByFilter(String tableName, FilterList filterList) {
        if (StringUtils.isBlank(tableName)) {
            return 0;
        }
        int count = 0;
        try {
            Table table = connection.getTable(TableName.valueOf(tableName));
            Scan scan = new Scan();
            scan.setFilter(filterList);
            ResultScanner scanner = table.getScanner(scan);
            for (Result result : scanner) {
                if (result.size() > 0) {
                    count++;
                }
            }
            return count;
        } catch (IOException e) {
            LOGGER.error("query hbase error.tableName={}", tableName, e);
        }
        return count;

    }

    public List<Map<String, String>> listByFilter(String tableName, FilterList filterList) {
        if (StringUtils.isBlank(tableName)) {
            return Collections.emptyList();
        }
        try {
            Table table = connection.getTable(TableName.valueOf(tableName));
            Scan scan = new Scan();
            scan.setFilter(filterList);
            ResultScanner scanner = table.getScanner(scan);
            List<Map<String, String>> list = new ArrayList<>();
            for (Result result : scanner) {
                Map<String, String> obj = new HashMap<>();
                for (Cell cell : result.listCells()) {
                    obj.put(Bytes.toString(CellUtil.cloneQualifier(cell)),
                            Bytes.toString(CellUtil.cloneValue(cell)).trim());
                }
                list.add(obj);
            }
            return list;
        } catch (IOException e) {
            LOGGER.error("query hbase error.tableName={}", tableName, e);
        }
        return Collections.emptyList();

    }

    public static SingleColumnValueFilter builEqualValueFilter(String column, String value) {
        SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(
                Bytes.toBytes(HbaseConfig.COLUMN_FAMILY), Bytes.toBytes(column), CompareFilter.CompareOp.EQUAL,
                Bytes.toBytes(value));
        singleColumnValueFilter.setFilterIfMissing(true);
        return singleColumnValueFilter;
    }
}

五、單元測試

    @Test
    public void testHBase1() {
        // System.out.println(JSON.toJSONString(hbaseResumeListService.list("華爲", "")));
        // 獲取所有表名
        try {
            Admin admin = HbaseUtils.getInstance().getConn().getAdmin();
            TableName[] tableNames = admin.listTableNames();
            for (TableName tableName : tableNames) {
                System.out.println(tableName.getNameAsString());
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    @Test
    public void testHBase5() throws IOException {
    	//建表
        Admin admin = HbaseUtils.getInstance().getConn().getAdmin();
        HTableDescriptor hTableDescriptor = new 		
        HTableDescriptor(TableName.valueOf("T_XP_TEST"));
		//設置列族名
        hTableDescriptor.addFamily(new HColumnDescriptor(Bytes.toBytes("base")));
        admin.createTable(hTableDescriptor);
        System.out.println("表是否建成功？=" + 
        admin.tableExists(TableName.valueOf("T_XP_TEST")));
    }

    @Test
    public void testHBaseAdd() throws IOException {
    	//插入數據
        Table table = 
        HbaseUtils.getInstance().getConn().getTable(TableName.valueOf("T_XP_TEST"));
        Put put = new Put(Bytes.toBytes("r1"));
        put.addColumn(Bytes.toBytes("base"), Bytes.toBytes("name"), Bytes.toBytes("xp"));
        put.addColumn(Bytes.toBytes("base"), Bytes.toBytes("age"), Bytes.toBytes("18"));
        put.addColumn(Bytes.toBytes("base"), Bytes.toBytes("gender"), Bytes.toBytes("1"));
        put.addColumn(Bytes.toBytes("base"), Bytes.toBytes("phone"), 
        Bytes.toBytes("13262831974"));
        put.addColumn(Bytes.toBytes("base"), Bytes.toBytes("team"), Bytes.toBytes("湖人"));
        table.put(put);

    }

    @Test
    public void testHBaseAll() throws IOException {
    	//掃描全表數據
        Table table = 
        HbaseUtils.getInstance().getConn().getTable(TableName.valueOf("T_XP_TEST"));
        ResultScanner scanner = table.getScanner(new Scan());
        List<Map<String, String>> list = new ArrayList<>();
        for (Result result : scanner) {
            Map<String, String> obj = new HashMap<>();
            for (Cell cell : result.listCells()) {
                obj.put(Bytes.toString(CellUtil.cloneQualifier(cell)), 
                Bytes.toString(CellUtil.cloneValue(cell)));
            }
            list.add(obj);
        }
        System.out.println(JSON.toJSONString(list));

    }

    @Test
    public void testHBaseFilter() throws IOException {
        Table table = 
        HbaseUtils.getInstance().getConn().getTable(TableName.valueOf("T_XP_TEST"));
        Scan scan = new Scan();
        // 設置過濾器，設置多個時用scan.setFilter(filterList); filterList.add....
        scan.setFilter(new SingleColumnValueFilter(Bytes.toBytes("base"), 
        Bytes.toBytes("team"),
                CompareFilter.CompareOp.EQUAL, Bytes.toBytes("湖人")));
        ResultScanner scanner = table.getScanner(scan);
        List<Map<String, String>> list = new ArrayList<>();
        for (Result result : scanner) {
            Map<String, String> obj = new HashMap<>();
            for (Cell cell : result.listCells()) {
                obj.put(Bytes.toString(CellUtil.cloneQualifier(cell)), 
                Bytes.toString(CellUtil.cloneValue(cell)));
            }
            list.add(obj);
        }
        System.out.println(JSON.toJSONString(list));

    }