flinkx同步es2hive

一.elasticsearch環境準備

1. elasticsearch 6.4.3
2. es-head 

elasticsearch的部署基於docker進行部署

首先我們來看一下目錄結構

cd docker
mkdir -p es
cd es
mkdir -p conf
mkdir -p data
mkdir -p logs

下面看一下docker-compose.yml

version: '3'
services:
  elasticsearch:
    image:  elasticsearch:6.4.3
    container_name: elasticsearch
    volumes:
      - $PWD/data:/usr/share/elasticsearch/data
      - $PWD/logs:/user/share/elasticsearch/logs
    ports:
        - '9200:9200'
        - '9300:9300'
    environment:
        - discovery.type=single-node
        - http.port=9200
        - http.cors.enabled=true
        - http.cors.allow-origin=*
        - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
        - http.cors.allow-credentials=false
        - bootstrap.memory_lock=true
        - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
  es-head:
    image: tobias74/elasticsearch-head:6
    container_name: es-head
    ports:
      - '9100:9100'
    links:
      - elasticsearch  

驗證

http://localhost:9200/?pretty

{
  "name" : "Yy55wm7",
  "cluster_name" : "docker-cluster",
  "cluster_uuid" : "_3n-nj84QhKKpHZ71t0OKQ",
  "version" : {
    "number" : "6.4.3",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "fe40335",
    "build_date" : "2018-10-30T23:17:19.084789Z",
    "build_snapshot" : false,
    "lucene_version" : "7.4.0",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

flinkx支持es2hive

es2hive.json

{
  "job" : {
    "content" : [ {
      "reader": {
        "name": "esreader",
        "parameter": {
          "address": "localhost:9200",
          "index": "manage",
          "type": "user",
          "column": [
            {
              "name": "user_id",
              "type": "bigint"
            },{
              "name": "user_name",
              "type": "varchar"
            },{
              "name": "user_phone",
              "type": "varchar"
            }
          ]
        }
      },
      "writer": {
          "name" : "hivewriter",
          "parameter" : {
            "jdbcUrl" : "jdbc:hive2://localhost:10000/es",
            "username" : "wangkai",
            "password" : "wangkai",
            "fileType" : "text",
            "writeMode" : "overwrite",
            "compress" : "",
            "schema" : "es",
            "charsetName" : "UTF-8",
            "maxFileSize" : 1073741824,
            "tablesColumn" : "{\"demonstrate_users\": [{\"key\": \"user_id\",\"type\": \"BIGINT\"}, {\"key\": \"user_name\",\"type\": \"string\"}, {\"key\": \"user_phone\",\"type\": \"string\"}]}",
            "defaultFS" : "hdfs://localhost:9000"
          }
      }
    }
    ],
    "setting" : {
      "restore" : {
        "maxRowNumForCheckpoint" : 0,
        "isRestore" : false,
        "restoreColumnName" : "",
        "restoreColumnIndex" : 0
      },
      "errorLimit" : {
        "record" : 0,
        "percentage" : 0
      },
      "speed" : {
        "bytes" : 1048576,
        "channel" : 1
      }
    }
  }
}

運行命令

bin/flinkx \
  -mode local \
  -job /Users/wangkai/apps/install/flinkx/es2hive.json \
  -pluginRoot syncplugins \
  -flinkconf /Users/wangkai/apps/install/flink-1.13.1/conf