springBoot直接集成Hbase2.1

序
上次分享了基於spring的starter集成hbase的，說過要分享直接對接的。
一、pom引包

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>3.0.0</version>
    <exclusions>
        <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
        </exclusion>
        <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
        </exclusion>
        <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
        </exclusion>
    </exclusions>
</dependency>
<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-shaded-client</artifactId>
    <version>2.1.0</version>
</dependency>
<dependency>
   <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>3.0.0</version>
</dependency>

這裏要說的有3點
1、hbase-shaded-client這個包比原生的hbase-client包好處是回將hbase-client包對應的版本整合一起
2、hadoop-common這個包要引入，不然可能會在表操作時報錯
3、注意要在環境變量配置HADOOP_HOME，建一個hadoop目錄，目錄建一個bin，裏面放Hadoop-common（網上下載）離對應版本的文件

二、配置類
HBaseConfig

import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.io.IOException;
import java.util.Map;
import java.util.Set;

/**
 * @author zhengwen
 **/
@Configuration
@EnableConfigurationProperties(HBaseProperties.class)
public class HBaseConfig {
    private final HBaseProperties properties;

    public HBaseConfig(HBaseProperties properties) {
        this.properties = properties;
    }

    @Bean
    public Connection hbaseConnection() {
        Connection conn = null;
        try {
            conn = ConnectionFactory.createConnection(configuration());
        } catch (IOException e) {
            System.out.println(e.getMessage());
        }
        return conn;
    }

    public org.apache.hadoop.conf.Configuration configuration() {
        org.apache.hadoop.conf.Configuration configuration = HBaseConfiguration.create();
        Map<String, String> config = properties.getConfig();
        Set<String> keySet = config.keySet();
        for (String key : keySet) {
            configuration.set(key, config.get(key));
        }
        return configuration;
    }
}

HBaseProperties

import org.springframework.boot.context.properties.ConfigurationProperties;

import java.util.Map;

/**
 * @author zhengwen
 **/
@ConfigurationProperties(prefix = "hbase")
public class HBaseProperties {
    private Map<String, String> config;

    public Map<String, String> getConfig() {
        return config;
    }

    public void setConfig(Map<String, String> config) {
        this.config = config;
    }
}

HbaseUtil


import com.google.common.io.Resources;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;

import java.io.File;
import java.io.IOException;

/**
 * @author zhengwen
 **/
@Slf4j
public class HbaseUtil {
    /**
     * 判斷表是否存在
     * @param admin hbase連接admin
     * @param tableName 表對象
     * @return 布爾值，true是，false否
     */
    public static boolean checkTableExist(Admin admin, TableName tableName) throws IOException {
        if (admin == null || tableName == null){
            return false;
        }
        return admin.tableExists(tableName);
    }

    /**
     * 初始化連接
     * @param hbaseConf
     * @param hbaseConnection
     */
    public static Admin initConn(Configuration hbaseConf, Connection hbaseConnection) {
        //這裏可以通過xml補充配置，這個hbase-site.xml在hbase的安裝目錄去找
        Admin admin = null;
        hbaseConf.addResource(Resources.getResource("hbase-site.xml"));
        File workaround = new File(".");
        System.getProperties().put("hadoop.home.dir",
                workaround.getAbsolutePath());
        new File("./bin").mkdirs();
        try {
            new File("./bin/winutils.exe").createNewFile();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
            log.error("----初始hbase連接異常：{}",e.getMessage(),e);
        }
        try {
            hbaseConnection = ConnectionFactory.createConnection(hbaseConf);
            admin = hbaseConnection.getAdmin();
        } catch (IOException e) {
            e.printStackTrace();
            log.error("---獲取hbase admin異常:{}",e.getMessage(),e);
        }
        return admin;
    }

    /**
     * 關閉連接
     * @param admin
     * @param hbaseConnection
     */
    public static void closeConn(Admin admin,Connection hbaseConnection) {
        try {
            if (null != admin) {
                admin.close();
            }
            /* 一定注意，這裏這個連接不能關閉，否則後面就都沒有了，讓spring自己管理，
            我就唄這裏坑了半天
            if (null != hbaseConnection) {
                hbaseConnection.close();
            }*/
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

三、使用


import com.alibaba.fastjson.JSON;
import com.fillersmart.g20.dc.collect.store.constant.Constant;
import com.fillersmart.g20.dc.collect.store.model.HbaseTable;
import com.fillersmart.g20.dc.collect.store.service.HbaseStoreService;
import com.fillersmart.g20.dc.collect.store.util.HbaseDataUtil;
import com.fillersmart.g20.dc.collect.store.util.HbaseUtil;
import com.fillersmart.g20.dc.core.data.HbaseSaveData;
import com.fillersmart.g20.dc.core.result.Result;
import com.fillersmart.g20.dc.core.result.ResultGenerator;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import javax.annotation.Resource;
import java.util.*;

/**
 * @author zhengwen
 **/
@Slf4j
@Service
public class HbaseStoreServiceImpl implements HbaseStoreService {

    @Autowired
    private Connection hbaseConnection;

    @Override
    public Result<?> createTable(HbaseTable hbaseTable) {

        String tableNameStr = hbaseTable.getTableName();
        Admin admin = null;
        try {
            Configuration hbaseConf = hbaseConnection.getConfiguration();
            admin = HbaseUtil.initConn(hbaseConf, hbaseConnection);

            TableName tableName = TableName.valueOf(tableNameStr);
            List<Map<String, String>> columnMpList = hbaseTable.getColumnMapList();

            boolean isExist = HbaseUtil.checkTableExist(admin, tableName);

            if (isExist) {
                log.info("----表：{}已經存在", tableNameStr);
                return ResultGenerator.genFailResult(tableName + "表已經存在");
            } else {
                //入參轉列簇信息
                List<ColumnFamilyDescriptor> columnFamilyDescriptorList = new ArrayList<>();
                for (Map<String, String> mp : columnMpList) {
                    Set<String> keySet = mp.keySet();
                    Iterator<String> it = keySet.iterator();
                    while (it.hasNext()) {
                        String key = it.next();
                        if ("colEn".equals(key)) {
                            String colEn = mp.get(key);

                            ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(colEn.getBytes()).build();
                            columnFamilyDescriptorList.add(columnFamilyDescriptor);

                        }
                    }
                }
                TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName).setColumnFamilies(columnFamilyDescriptorList).build();
                admin.createTable(tableDescriptor);
            }

        } catch (Exception e) {
            e.printStackTrace();
            log.error("---創建表:{}異常，原因：{}", tableNameStr, e.getCause().getMessage(), e);
            return ResultGenerator.genFailResult(e.getCause().getMessage());
        } finally {
            HbaseUtil.closeConn(admin, hbaseConnection);
        }

        return ResultGenerator.genSuccessResult();
    }

    @Override
    public Result<?> saveData(HbaseSaveData hbaseSaveData) {
        String tableNameStr = hbaseSaveData.getTableName();
        List<Map<String, String>> dataMpList = hbaseSaveData.getDataList();
        Admin admin = null;
        try {
            Configuration hbaseConf = hbaseConnection.getConfiguration();
            admin = HbaseUtil.initConn(hbaseConf, hbaseConnection);

            //record
            String recordTableNameStr = tableNameStr + "_record";
            TableName recordTableName = TableName.valueOf(recordTableNameStr);
            boolean tableExist = HbaseUtil.checkTableExist(admin, recordTableName);
            if (tableExist) {
                Table table = hbaseConnection.getTable(recordTableName);

                if (CollectionUtils.isEmpty(dataMpList)) {
                    return ResultGenerator.genFailResult("保存數據爲空");
                }
                List<Put> putList = new ArrayList<>();
                for (Map<String, String> dataMp : dataMpList) {
                    putList = HbaseDataUtil.initDataList(dataMp, Constant.RECORD_SAVE);
                }
                if (CollectionUtils.isEmpty(putList)) {
                    log.error("---記錄數據存儲數據初始異常:{}", JSON.toJSONString(putList));
                } else {
                    table.put(putList);
                }
                table.close();
                log.info("----表：{}數據存儲HBase成功",recordTableName.getNameAsString());
            } else {
                log.error("---表：{}不存在，無法保存", recordTableNameStr);
                return ResultGenerator.genFailResult(recordTableNameStr + "表不存在，無法保存");
            }

            //最新記錄
            String shadowTableNameStr = tableNameStr + "_shadow";
            TableName shadowTableName = TableName.valueOf(shadowTableNameStr);
            tableExist = HbaseUtil.checkTableExist(admin, shadowTableName);
            if (tableExist) {
                Table table = hbaseConnection.getTable(shadowTableName);

                if (CollectionUtils.isEmpty(dataMpList)) {
                    return ResultGenerator.genFailResult("保存數據爲空");
                }
                List<Put> putList = new ArrayList<>();
                for (Map<String, String> dataMp : dataMpList) {
                    putList = HbaseDataUtil.initDataList(dataMp, Constant.SHADOW_SAVE);
                }
                if (CollectionUtils.isEmpty(putList)) {
                    log.error("---shadow最新數據存儲數據初始異常:{}", JSON.toJSONString(putList));
                } else {
                    table.put(putList);
                }
                table.close();
                log.info("----表：{}數據存儲HBase成功",shadowTableName.getNameAsString());
            } else {
                log.error("---表：{}不存在，無法保存", shadowTableNameStr);
                return ResultGenerator.genFailResult(shadowTableNameStr + "表不存在，無法保存");
            }
        } catch (Exception e) {
            e.printStackTrace();
            log.error("---Hbase數據表:{}保存數據異常，原因：{}", tableNameStr, e.getMessage(), e);
            return ResultGenerator.genFailResult(tableNameStr + "數據存儲失敗");
        } finally {
            HbaseUtil.closeConn(admin, hbaseConnection);
        }

        return ResultGenerator.genSuccessResult();
    }

    @Override
    public Result<?> restCreateTable(HbaseTable hbaseTable) {
        String tableNameStr = hbaseTable.getTableName();

        List<Map<String, String>> columnMpList = hbaseTable.getColumnMapList();

        if (StringUtils.isBlank(tableNameStr)) {
            return ResultGenerator.genFailResult("表名不能爲空");
        }
        //TODO 怎麼判斷表是否存在？
        //表參數轉


        return null;
    }
}

這裏就直接貼一個service的實現類大家參考，裏面也就寫了建表與數據保存的。這裏要說的關鍵點有3點：
1、hbase的建表時不需要考慮列，只需要先設置好列簇就行
2、hbase的數據保存類似jdbc，素有數據都可以轉爲map，key是字段名，value是值，批量插入
3、注意存儲對象轉map不能有嵌套對象，當然如果真有也不是不可以，也就是字段離存的json字符串而已
四、補充
hbase-site.xml,這個其實可以不要，如果你不要就是默認hbase的安裝離得默認配置，要就是可以使用這裏面得配置覆蓋安裝離的配置

<?xml version="1.0" encoding="UTF-8"?>

<!--Autogenerated by Cloudera Manager-->
<configuration>
  <property>
    <name>hbase.rootdir</name>
    <value>hdfs://bigdata01:8020/hbase</value>
  </property>
  <property>
    <name>hbase.client.write.buffer</name>
    <value>2097152</value>
  </property>
  <property>
    <name>hbase.client.pause</name>
    <value>100</value>
  </property>
  <property>
    <name>hbase.client.retries.number</name>
    <value>10</value>
  </property>
  <property>
    <name>hbase.client.scanner.caching</name>
    <value>100</value>
  </property>
  <property>
    <name>hbase.client.keyvalue.maxsize</name>
    <value>10485760</value>
  </property>
  <property>
    <name>hbase.ipc.client.allowsInterrupt</name>
    <value>true</value>
  </property>
  <property>
    <name>hbase.client.primaryCallTimeout.get</name>
    <value>10</value>
  </property>
  <property>
    <name>hbase.client.primaryCallTimeout.multiget</name>
    <value>10</value>
  </property>
  <property>
    <name>hbase.client.scanner.timeout.period</name>
    <value>60000</value>
  </property>
  <property>
    <name>hbase.coprocessor.region.classes</name>
    <value>org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint</value>
  </property>
  <property>
    <name>hbase.regionserver.thrift.http</name>
    <value>false</value>
  </property>
  <property>
    <name>hbase.thrift.support.proxyuser</name>
    <value>false</value>
  </property>
  <property>
    <name>hbase.rpc.timeout</name>
    <value>60000</value>
  </property>
  <property>
    <name>hbase.snapshot.enabled</name>
    <value>true</value>
  </property>
  <property>
    <name>hbase.snapshot.region.timeout</name>
    <value>300000</value>
  </property>
  <property>
    <name>hbase.snapshot.master.timeout.millis</name>
    <value>300000</value>
  </property>
  <property>
    <name>hbase.security.authentication</name>
    <value>simple</value>
  </property>
  <property>
    <name>hbase.rpc.protection</name>
    <value>authentication</value>
  </property>
  <property>
    <name>zookeeper.session.timeout</name>
    <value>60000</value>
  </property>
  <property>
    <name>zookeeper.znode.parent</name>
    <value>/hbase</value>
  </property>
  <property>
    <name>zookeeper.znode.rootserver</name>
    <value>root-region-server</value>
  </property>
  <property>
    <name>hbase.zookeeper.quorum</name>
    <value>bigdata03,bigdata01,bigdata02</value>
  </property>
  <property>
    <name>hbase.zookeeper.property.clientPort</name>
    <value>2181</value>
  </property>
  <property>
    <name>hbase.rest.ssl.enabled</name>
    <value>false</value>
  </property>
</configuration>

最後這裏說下，我們都是使用的是別名，所以要配置host，不然這些個bigdata01、bigdata02都不知道是什麼地址的。晚上都推薦用SwitchHosts，但是我在win10上改來改去狠容易把原host文件丟失，還是使用早些年在卓爾電商做支付，同事們都用的一個小程序方便。在win10上依然可用，不錯。
然後呢我這裏是沒有加一些封裝好的接口的，下次分享hbaseApi，順便把相關的封裝的接口分享學習下，下次的集成方式也將又不一樣哦。
這裏沒有用最新版本是因爲項目上使用的就是3.0 + 2.1。同學們可以自己更換版本玩。

springBoot直接集成Hbase2.1

.NET有哪些好用的定時任務調度框架

Python 將PDF轉爲PDF/A、PDF/X，以及PDF/A轉回PDF

elk3

Kafka存儲機制

aws語音呼叫調用，告警電話

深度學習框架火焰圖pprof和CUDA Nsys配置指南

爬蟲兩種繞過5s盾的方法

【轉】[C#] WebAPI 防止併發調用二（冥等性）

【轉】[SQL Server]關掉 SSMS 的 IntelliSense

號稱能打敗MLP的KAN到底行不行？數學核心原理全面解析

中臺真的好可以解決一切問題？

瀏覽器WebSocket連接apollo的MQTT實現消息推送（WEB系統微信掃碼登錄）

springCloud配置中心（非Git版）

OSS存儲工具類分享（阿里）

springboot集成json web token

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結