java调用sqoop有两种用法,一是使用sqoop java api,二是使用runtime调用sqoop命令,但runtime比较难用,不推荐,以下是两种使用sqoop Java api的方法。
方法一、使用String[]传参(不推荐)
代码如下,但经过本人测试,同时调用多个sqoop任务会出现传参不能识别报错
public static void main(String[] args) throws IOException {
System.setProperty("HADOOP_USER_NAME","xxx");
System.out.println(" begin test sqoop");
String[] argument = new String[] {
"--username","xxx",
"--password","xxx",
"--num-mappers","1",
"--query","select * from (select rownum as rn,r.* from RP_SCALE r ) where rn>=1 and rn<=1000 and $CONDITIONS",
"--fields-terminated-by","\001",
"--target-dir","/data/hive/warehouse/ods_cmis.db/ods_rp_scale_view",
"--delete-target-dir",
"--connect","jdbc:oracle:thin:@xxx:1521:coresit",
};
SqoopTool sqoopTool= SqoopTool.getTool("import");
Configuration conf= new Configuration();
conf.set("fs.defaultFS","hdfs://xxx:8020");
Sqoop sqoop = new Sqoop(sqoopTool,SqoopTool.loadPlugins(conf));
Sqoop.runSqoop(sqoop,argument);
}
方法二、使用SqoopOptions对象传参(推荐)
SqoopTool sqoopTool = SqoopTool.getTool("import");
SqoopOptions sqoopOptions = new SqoopOptions();
sqoopOptions.setConnectString("xxx");
sqoopOptions.setUsername("xxx");
sqoopOptions.setPassword("xxx");
sqoopOptions.setNumMappers(1);
sqoopOptions.setNullStringValue("\\\\N");
sqoopOptions.setNullNonStringValue("\\\\N");
sqoopOptions.setFieldsTerminatedBy('\001');
sqoopOptions.setTargetDir("/data/hive/warehouse/ods_cmis.db/ods_" + hiveTableName.toLowerCase());
sqoopOptions.setCodeOutputDir("sqoopjavafile");
sqoopOptions.setJarOutputDir("sqoopcompilefile/" + CommonUtil.getUUID() + "/");
sqoopOptions.setHiveDropDelims(true);
sqoopOptions.setSqlQuery(querySql);
sqoopOptions.setAppendMode(true);
sqoopOptions.setClassName(hiveTableName + CommonUtil.getUUID());
sqoopOptions.setSqlQuery(querySql);
sqoopOptions.setAppendMode(true);
sqoopOptions.setClassName(hiveTableName + CommonUtil.getUUID());
Configuration conf= new Configuration();
conf.set("fs.defaultFS","hdfs://xxx:8020");
Sqoop sqoop = new Sqoop(sqoopTool, SqoopTool.loadPlugins(conf), sqoopOptions);
Sqoop.runSqoop(sqoop, new String[]{});