java調用sqoop有兩種用法,一是使用sqoop java api,二是使用runtime調用sqoop命令,但runtime比較難用,不推薦,以下是兩種使用sqoop Java api的方法。
方法一、使用String[]傳參(不推薦)
代碼如下,但經過本人測試,同時調用多個sqoop任務會出現傳參不能識別報錯
public static void main(String[] args) throws IOException {
System.setProperty("HADOOP_USER_NAME","xxx");
System.out.println(" begin test sqoop");
String[] argument = new String[] {
"--username","xxx",
"--password","xxx",
"--num-mappers","1",
"--query","select * from (select rownum as rn,r.* from RP_SCALE r ) where rn>=1 and rn<=1000 and $CONDITIONS",
"--fields-terminated-by","\001",
"--target-dir","/data/hive/warehouse/ods_cmis.db/ods_rp_scale_view",
"--delete-target-dir",
"--connect","jdbc:oracle:thin:@xxx:1521:coresit",
};
SqoopTool sqoopTool= SqoopTool.getTool("import");
Configuration conf= new Configuration();
conf.set("fs.defaultFS","hdfs://xxx:8020");
Sqoop sqoop = new Sqoop(sqoopTool,SqoopTool.loadPlugins(conf));
Sqoop.runSqoop(sqoop,argument);
}
方法二、使用SqoopOptions對象傳參(推薦)
SqoopTool sqoopTool = SqoopTool.getTool("import");
SqoopOptions sqoopOptions = new SqoopOptions();
sqoopOptions.setConnectString("xxx");
sqoopOptions.setUsername("xxx");
sqoopOptions.setPassword("xxx");
sqoopOptions.setNumMappers(1);
sqoopOptions.setNullStringValue("\\\\N");
sqoopOptions.setNullNonStringValue("\\\\N");
sqoopOptions.setFieldsTerminatedBy('\001');
sqoopOptions.setTargetDir("/data/hive/warehouse/ods_cmis.db/ods_" + hiveTableName.toLowerCase());
sqoopOptions.setCodeOutputDir("sqoopjavafile");
sqoopOptions.setJarOutputDir("sqoopcompilefile/" + CommonUtil.getUUID() + "/");
sqoopOptions.setHiveDropDelims(true);
sqoopOptions.setSqlQuery(querySql);
sqoopOptions.setAppendMode(true);
sqoopOptions.setClassName(hiveTableName + CommonUtil.getUUID());
sqoopOptions.setSqlQuery(querySql);
sqoopOptions.setAppendMode(true);
sqoopOptions.setClassName(hiveTableName + CommonUtil.getUUID());
Configuration conf= new Configuration();
conf.set("fs.defaultFS","hdfs://xxx:8020");
Sqoop sqoop = new Sqoop(sqoopTool, SqoopTool.loadPlugins(conf), sqoopOptions);
Sqoop.runSqoop(sqoop, new String[]{});