刪除druid中的數據可參考 https://blog.csdn.net/qq_34864753/article/details/102861322
1. 下載imply ，解壓
https://imply.io/get-started
tar -xzf imply-3.0.12.tar.gz
impy quickstart鏈接地址（供參考）
https://docs.imply.io/on-prem/quickstart
2. 修改 _common 目錄下的 common.runtime.properties 文件
mysql 庫的創建可以參考下面，需要先創建庫，這邊使用了imply_druid用戶 （注意數據庫 需要是utf8，否則會報錯）
Exception creating table
org.skife.jdbi.v2.exceptions.CallbackFailedException: org.apache.druid.java.util.common.ISE: Druid requires its MySQL database to be created with an UTF8 charset, found `latin1`. The recommended charset is `utf8mb4`.
create database imply_druid;
CREATE USER 'imply_druid'@'%' IDENTIFIED BY 'imply_druid';
grant all privileges on imply_druid.* to 'imply_druid'@'%' identified by 'imply_druid';
3. 將hadoop的配置文件 core-site.xml，hdfs-site.xml，mapred-site.xml，yarn-site.xml拷貝到 _common目錄下
4. 啓動集羣 master節點/data節點/query節點
###master節點
/usr/local/bigdata/imply-3.0.12/bin/supervise -c /usr/local/bigdata/imply-3.0.12/conf/supervise/master-no-zk.conf -daemon
###data節點
/usr/local/bigdata/imply-3.0.12/bin/supervise -c /usr/local/bigdata/imply-3.0.12/conf/supervise/data.conf -daemon
###query節點
/usr/local/bigdata/imply-3.0.12/bin/supervise -c /usr/local/bigdata/imply-3.0.12/conf/supervise/query.conf -daemon
###重啓服務
/usr/local/bigdata/imply-3.0.12/bin/service -restart （broker,router,imply-ui,middleManager,historical,coordinator,overlord)
eg：/usr/local/bigdata/imply-2.8.6/bin/service -restart router 一般一個節點上有兩個服務，要一個一個restart
###下掉服務
/usr/local/bigdata/imply-3.0.12/bin/service --down
5. 端口及界面
主要有 8081（master節點），8090（master節點），8088（query節點），9095（query節點）四個端口
#8081端口界面
#imply.io界面，對使用方很友好
#8088端口
#8088端口
6. 使用案例：數據從hdfs到druid (表的列太多，刪除一部分)
數據，需先要上傳到hdfs
{"cc_transform_rate_history": "0.05", "cc_sex": "1", "user_id": "1000888550", "cc_call_timevstimes_latest_month": "249", "cc_entry_days": "605", "cc_transform_rate_latest_month": "0.19", "cc_transform_numbers": "174", "cc_grade": "0", "row_key": "0558880001:1", "cc_work_city_code": "310100", "cc_call_timevstimes_history": "270", "cc_connecting_numbers_valid": "3281"}
{"cc_sex": "3", "user_id": "1018039550", "cc_origin_place_code": "420000", "cc_call_timevstimes_latest_month": "41", "cc_entry_days": "138", "cc_grade": "6", "row_key": "0559308101:1", "cc_work_city_code": "420100", "cc_call_timevstimes_history": "82", "cc_connecting_numbers_valid": "2043"}
{"cc_sex": "0", "user_id": "1028090650", "cc_origin_place_code": "410000", "cc_call_timevstimes_latest_month": "230", "cc_entry_days": "32", "cc_grade": "6", "row_key": "0560908201:1", "cc_work_city_code": "310100", "cc_call_timevstimes_history": "230", "cc_connecting_numbers_valid": "348"}
{"cc_sex": "0", "user_id": "1015562650", "cc_origin_place_code": "420000", "cc_call_timevstimes_latest_month": "34", "cc_entry_days": "178", "cc_grade": "6", "row_key": "0562655101:1", "cc_work_city_code": "420100", "cc_call_timevstimes_history": "61", "cc_connecting_numbers_valid": "1890"}
{"cc_transform_rate_history": "-1.00", "cc_sex": "3", "user_id": "1003846650", "cc_entry_days": "500", "cc_transform_rate_latest_month": "-1.00", "cc_grade": "6", "row_key": "0566483001:1", "cc_call_timevstimes_history": "-1"}
{"cc_sex": "0", "user_id": "1029658650", "cc_origin_place_code": "360000", "cc_call_timevstimes_latest_month": "175", "cc_entry_days": "9", "cc_grade": "6", "row_key": "0568569201:1", "cc_work_city_code": "310100", "cc_call_timevstimes_history": "175", "cc_connecting_numbers_valid": "49"}
{"cc_transform_rate_history": "-1.00", "cc_sex": "3", "user_id": "1006958650", "cc_entry_days": "388", "cc_transform_rate_latest_month": "-1.00", "cc_grade": "6", "row_key": "0568596001:1", "cc_call_timevstimes_history": "-1"}
{"cc_sex": "3", "user_id": "1016289650", "cc_entry_days": "168", "cc_grade": "6", "row_key": "0569826101:1", "cc_call_timevstimes_history": "36", "cc_connecting_numbers_valid": "1407"}
{"cc_transform_rate_history": "0.75", "cc_sex": "1", "user_id": "40750", "cc_call_timevstimes_latest_month": "629", "cc_entry_days": "1464", "cc_transform_rate_latest_month": "0.00", "cc_transform_numbers": "141", "cc_grade": "5", "row_key": "05704:1", "cc_work_city_code": "310100", "cc_call_timevstimes_history": "337", "cc_connecting_numbers_valid": "189"}
{"cc_transform_rate_history": "-1.00", "cc_sex": "0", "user_id": "1008884750", "cc_origin_place_code": "410000", "cc_entry_days": "270", "cc_transform_rate_latest_month": "-1.00", "cc_grade": "6", "row_key": "0574888001:1", "cc_work_city_code": "310100", "cc_call_timevstimes_history": "-1"}
{
"type": "index_hadoop",
"id": "index_hadoop_user_profile_cc_2019-08-16T07:28:53.194Z",
"spec": {
"dataSchema": {
"dataSource": "user_profile_cc",
"parser": {
"parseSpec": {
"timestampSpec": {
"column": "timestamp",
"missingValue": "2019-08-15",
"format": "auto"
},
"dimensionsSpec": {
"dimensions": [
"row_key",
"user_id",
"cc_work_city_code",
"cc_call_timevstimes_latest_month",
"cc_entry_days",
"cc_origin_place_code",
"cc_transform_numbers",
"cc_transform_rate_latest_month",
"cc_call_timevstimes_history",
"cc_connecting_numbers_valid",
"cc_grade",
"cc_sex",
"cc_transform_rate_history"
]
},
"format": "json"
},
"type": "hadoopyString"
},
"metricsSpec": [],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {
"type": "none"
},
"rollup": false,
"intervals": [
"2019-08-15T00:00:00.000Z/2019-08-16T00:00:00.000Z"
]
},
"transformSpec": {
"filter": null,
"transforms": []
}
},
"ioConfig": {
"type": "hadoop",
"inputSpec": {
"paths": "hdfs://nameservice1/tmp/zjf_druid/user_profile_cc",
"type": "static"
},
"metadataUpdateSpec": null,
"segmentOutputPath": null
},
"tuningConfig": {
"type": "hadoop",
"workingPath": null,
"version": "2019-08-16T07:28:53.194Z",
"partitionsSpec": {
"type": "hashed",
"targetPartitionSize": 2000000,
"maxPartitionSize": 3000000,
"assumeGrouped": false,
"numShards": -1,
"partitionDimensions": []
},
"shardSpecs": {},
"indexSpec": {
"bitmap": {
"type": "concise"
},
"dimensionCompression": "lz4",
"metricCompression": "lz4",
"longEncoding": "longs"
},
"maxRowsInMemory": 1000000,
"maxBytesInMemory": 0,
"leaveIntermediate": false,
"cleanupOnFailure": true,
"overwriteFiles": false,
"ignoreInvalidRows": false,
"jobProperties": {
"mapreduce.job.queuename":"root.zm_yarn_pool.production",
"mapreduce.job.user.classpath.first": "true",
"mapreduce.job.classloader": "true",
"mapreduce.job.classloader.system.classes": "-javax.validation.,java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop.,org.xerial.snappy."
},
"combineText": false,
"useCombiner": false,
"buildV9Directly": true,
"numBackgroundPersistThreads": 0,
"forceExtendableShardSpecs": true,
"useExplicitVersion": false,
"allowedHadoopPrefix": [],
"logParseExceptions": true,
"maxParseExceptions": 1000
},
"uniqueId": "83b32f38334d48ab99598d9c725c885b"
},
"hadoopDependencyCoordinates": null,
"classpathPrefix": null,
"context": {},
"groupId": "index_hadoop_user_profile_cc_2019-08-16T07:28:53.194Z",
"dataSource": "user_profile_cc",
"resource": {
"availabilityGroup": "index_hadoop_user_profile_cc_2019-08-16T07:28:53.194Z",
"requiredCapacity": 1
}
}
7.任務提交可以用 imply.io那個界面，也可以用 8081那個界面的Tasks / Submit task 來提交
8.注意事項
1. history目錄的runtime.properties
mi
2. middleManager目錄下的runtime.properties
