sqoop job命令自動生成

批量生成

#!/usr/bin/env python
import MySQLdb
import datetime
import time
import os

#file path
FILEPATH='/tmp'

#source database
HOST='11.143.68.29'
PORT=3306
USER=admin'
PASSWD='admin'
DB='db_1'

#sqoop
FIELDS_TERMINATED_BY='\\t'     #如果需要臨時表合併hive小文件，需使用默認分隔符
CHECK_COLUMN='__#alibaba_rds_row_id#__'

conn=MySQLdb.connect(host=HOST,port=PORT,user=USER,passwd=PASSWD,db =DB,)
cur=conn.cursor()
ret=cur.execute("select table_name from information_schema.tables where TABLE_SCHEMA='%s'"%(DB))
ret=cur.fetchall()

creFile=open(FILEPATH+'/sjob_cre.sh','w')
delFile=open(FILEPATH+'/sjob_del.sh','w')
execFile=open(FILEPATH+'/sjob_exec.sh','w')


for line in ret:
    tableName=line[0
    try:
        sjob="sudo -u hdfs sqoop job --create %s_%s -- import --connect  jdbc:mysql://%s:%s/%s  --table %s \
         --username %s --password %s -m 1  --hive-import  --fields-terminated-by '%s' --hive-database %s \
         --incremental append --check-column %s --last-value 0;"%(DB,tableName,HOST,PORT,DB,tableName,USER,PASSWD,FIELDS_TERMINATED_BY,DB,CHECK_COLUMN)
        creFile.write(sjob+'\n')
        delFile.write("sudo -u hdfs sqoop job --delete %s_%s"%(DB,tableName)+'\n')
        execFile.write("sudo -u hdfs sqoop job --exec %s_%s"%(DB,tableName)+'\n')
    except:
        print "*************************************************"
        print "Error,%s failed!"%(tableName)
        print "*************************************************"
cur.close()
conn.close()

生成一個表的job：

#!/bin/bash
set -e
#set -x

#file path
EXECFILE_PATH='/data/sqoop/sjob_exec.sh'

#source database
HOST='192.168.2.229'
PORT=3306
USER='root'
PASSWD='root'
DB='db_1'op

FIELDS_TERMINATED_BY='\t'
CHECK_COLUMN='__#alibaba_rds_row_id#__'
WORKER_NUM=1
tableName=$1

sjob="sudo -u hdfs sqoop job --create ${DB}_${tableName} -- import --connect  jdbc:mysql://${HOST}:${PORT}/${DB}  --table ${tableName} \
    --username ${USER} --password ${PASSWD} -m ${WORKER_NUM}  --hive-import  --fields-terminated-by '${FIELDS_TERMINATED_BY}' --hive-database ${DB} \
    --incremental append --check-column ${CHECK_COLUMN} --last-value 0;"

echo $sjob

echo "Add to sqoop?(y or n):"
read ret

if [ $ret == 'y' ];
then
        $($sjob)
        sjob="sqoop job --exec ${DB}_${tableName}"
        sed -i '/impala/i\'"$sjob"'' $EXECFILE_PATH
        echo "Add Completed!"
fi