vertica 實時消費kafka 例子

1.提前在vertica種建立好相關對接kafka json數據的表

CREATE TABLE public.res_time_space
(
    record_id varchar(255) NOT NULL,
    object_id varchar(255),
    object_type int NOT NULL,
    snap_time int,
    channel_id varchar(80),
    device_id varchar(80),
    device_name varchar(80),
    tollgate_id varchar(80),
    longitude float,
    latitude float,
    image_url varchar(255),
    close_image_url varchar(255),
    distant_image_url varchar(255),
    direction int,
    speed float,
    lane int,
    event_id varchar(128),
    event_type varchar(64),
    ods_time int,
    address varchar(255),
    date_time date NOT NULL,
    s2_13 int,
    s2_15 int,
    s2_17 int,
    s2_19 int,
    type_date int NOT NULL,
    hour_time int
)
PARTITION BY (res_time_space.type_date);

res_time_space_kafka_vertica.sh

# 安裝kafka插件（初始化）
#su - dbadmin
#cd /opt/vertica/packages/kafka/ddl
#vsql -U dbadmin -w dbadmin -d viid -f install.sql
#-----------------------------------------------------------------
config_schema=res_time_space
dbhost=192.168.108.64
dbusername=dbadmin
dbpassword=dbadmin
frame_duration='00:00:10'
eof_timeout_ms=3000
operator=dbadmin
kafka_hosts=192.168.108.126:9092
kafka_topic=res_time_space
kakfa_partitions=1
target_schema=public
target_table=res_time_space
# Vertica 8.1.0+
kafka_config=" --config-schema ${config_schema} --dbhost ${dbhost} --username ${dbusername} --password ${dbpassword}"
 
# shutdown instance
/opt/vertica/packages/kafka/bin/vkconfig shutdown --instance-name ${config_schema}_instance ${kafka_config}
echo "Shutdown Instance Complete!"
# truncate table
#$VSQL <<- EOF
#drop schema ${config_schema}_instance cascade;
#truncate table public.stranger_event;
#EOF
 
# Create and Configure Scheduler
/opt/vertica/packages/kafka/bin/vkconfig scheduler --create --add ${kafka_config} --frame-duration ${frame_duration} --eof-timeout-ms ${eof_timeout_ms} --operator ${operator}
echo "Create and Configure Scheduler Complete!"
 
# Create a Cluster
/opt/vertica/packages/kafka/bin/vkconfig cluster --create --cluster ${config_schema}_cluster --hosts ${kafka_hosts} ${kafka_config}
echo "Create Cluster Complete!"
 
# Create a Data Table
 
# Create a Source
/opt/vertica/packages/kafka/bin/vkconfig source --create --source ${kafka_topic}  --cluster ${config_schema}_cluster --partitions ${kakfa_partitions} ${kafka_config}
echo "Create Kafka Source Complete!"
 
# Create a Target
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema ${target_schema} --target-table ${target_table} ${kafka_config}
echo "Create Target Complete!"
 
# Create a Load-Spec
/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec load_spec_${config_schema} --parser KafkaJSONParser --parser-parameters flatten_arrays=False,flatten_maps=False ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec load_date_dimension_spec --parser KafkaJSONParser --filters "FILTER KafkaInsertDelimiters(delimiter=E'\n')" ${kafka_config}
 
echo "Create Load-Spec Complete!"
 
# Create a Microbatch
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch ${kafka_topic} --target-schema ${target_schema} --target-table ${target_table} --rejection-schema ${target_schema} --rejection-table ${target_table}_rej --load-spec load_spec_${config_schema} --add-source ${kafka_topic} --add-source-cluster ${config_schema}_cluster ${kafka_config}
echo "Create Microbatch Complete!"
 
# Launch the Scheduler
/opt/vertica/packages/kafka/bin/vkconfig launch --instance-name load_spec_${config_schema} ${kafka_config} &
echo "Launch the Scheduler Complete!"
echo "Done!"

-- 資源池操作
CREATE RESOURCE POOL custom_pool MEMORYSIZE '10%' PLANNEDCONCURRENCY 1 QUEUETIMEOUT 0;

SELECT name, memorysize, maxmemorysize FROM V_CATALOG.RESOURCE_POOLS;
SELECT name, memorysize, maxmemorysize, priority, maxconcurrency
    FROM V_CATALOG.RESOURCE_POOLS where is_internal ='t';
   --  PRIORITY值越大優先級越高，可以存着負值
ALTER RESOURCE POOL custom_pool PRIORITY 5;
  
DROP RESOURCE POOL custom_pool;

# 安裝kafka插件（初始化）
#su - dbadmin
#cd /opt/vertica/packages/kafka/ddl
#vsql -U dbadmin -w dbadmin -d viid -f install.sql
#-----------------------------------------------------------------
scheduler_name=res_time_space
dbhost=192.168.108.64
dbusername=dbadmin
dbpassword=dbadmin
frame_duration='00:00:10'
eof_timeout_ms=3000
operator=dbadmin
kafka_hosts=192.168.108.126:9092
kafka_topic=res_time_space
kakfa_partitions=1
target_schema=public
target_table=res_time_space
# Vertica 8.1.0
kafka_config=" --config-schema ${scheduler_name} --dbhost ${dbhost} --username ${dbusername} --password ${dbpassword}"
 
# shutdown instance
/opt/vertica/packages/kafka/bin/vkconfig shutdown --instance-name ${scheduler_name}_instance ${kafka_config}
echo "Shutdown Instance Complete!"
# truncate table
#$VSQL <<- EOF
#drop schema ${scheduler_name}_instance cascade;
#truncate table public.stranger_event;
#EOF
# 創建專用資源池
/opt/vertica/bin/vsql -U dbadmin -w dbadmin -d viid -c "CREATE RESOURCE POOL res_time_space_pool MEMORYSIZE '10%' PLANNEDCONCURRENCY 1 QUEUETIMEOUT 0;"
# Create and Configure Scheduler 調度程序本身
/opt/vertica/packages/kafka/bin/vkconfig scheduler --create --add ${kafka_config} --frame-duration ${frame_duration} --eof-timeout-ms ${eof_timeout_ms} --operator ${operator} --resource-pool res_time_space_pool
echo "Create and Configure Scheduler Complete!"
 
# Create a Cluster 集羣
/opt/vertica/packages/kafka/bin/vkconfig cluster --create --cluster ${scheduler_name}_cluster --hosts ${kafka_hosts} ${kafka_config}
echo "Create Cluster Complete!"
 
# Create a Data Table
 
# Create a Source 來源
/opt/vertica/packages/kafka/bin/vkconfig source --create --source ${kafka_topic}  --cluster ${scheduler_name}_cluster --partitions ${kakfa_partitions} ${kafka_config}
echo "Create Kafka Source Complete!"
 
# Create a Target 目標
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema ${target_schema} --target-table ${target_table} ${kafka_config}
echo "Create Target Complete!"
 
# Create a Load-Spec 加載規範
/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec load_spec_${scheduler_name} --parser KafkaJSONParser --parser-parameters flatten_arrays=False,flatten_maps=False ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec load_date_dimension_spec --parser KafkaJSONParser --filters "FILTER KafkaInsertDelimiters(delimiter=E'\n')" ${kafka_config}
 
echo "Create Load-Spec Complete!"
 
# Create a Microbatch 微批處理
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch ${kafka_topic} --target-schema ${target_schema} --target-table ${target_table} --rejection-schema ${target_schema} --rejection-table ${target_table}_rej --load-spec load_spec_${scheduler_name} --add-source ${kafka_topic} --add-source-cluster ${scheduler_name}_cluster ${kafka_config}
echo "Create Microbatch Complete!"
 
# Launch the Scheduler 微批調度
/opt/vertica/packages/kafka/bin/vkconfig launch --instance-name load_spec_${scheduler_name} ${kafka_config} &
echo "Launch the Scheduler Complete!"
echo "Done!"

vertica 實時消費kafka 例子

DAPPER 事務 TRANSACTION

vertica-CUSTOM DESIGNER、 PROJECTION、procedure、空間對象

flink1.10-docker 安裝

pg,gp創建分區表、計算經緯度距離，轉換漂移經緯度

vertica-java-自定義函數類(UDX)的部署

MPP update優化一

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結