Web提交參數到Spark集羣執行任務

提供一個API供使用者調用，大抵需求如下：

輸入某用戶的位置（經緯度），提交到Web服務器，再把任務（找出該用戶附近一公里內的商城推薦）提交到Spark集羣上執行，返回計算結果後再存入到redis數據庫中，供後臺調用返回結果給使用方。

網上關於這方面的資料大抵是基於spark-assembly-1.4.1-hadoop2.6.0.jar，而我們這邊的環境是spark2.0-Hadoop2.6.0，版本的不同意味着教程可能不適用。經過測試證明，spark1.4版的jar包與平臺環境不兼容，會出現任務提交失敗的異常。所以建議把平臺環境的jar包拷入項目。
另一方面，由於集羣和發佈項目的web服務器不在同一臺機器，所以需要把spark相關的jar包（187M）拷貝一份到web服務器。

數據分析人員開發了一個jar（spark-java-2.xx-SNAPSHOT.jar）包，需要調用StationIdForUser類：傳入經緯度，然後它會上傳任務到集羣，但它依賴於spark集羣的衆多jar包。

package com.sibat.spark.action;
import org.apache.hadoop.conf.Configuration;
import org.apache.spark.SparkConf;
import org.apache.spark.deploy.yarn.Client;
import org.apache.spark.deploy.yarn.ClientArguments;
//參考網上教程
public class Action {
    public static void main(String[] args) {
        //String tmp = Thread.currentThread().getContextClassLoader().getResource("").getPath();
        //tmp = tmp.substring(0, tmp.length() - 8);
        String[] arg0 = new String[] { "--class", "cn.sibat.spark.StationIdForUser",
                //
                "--jar", ".../apache-tomcat-7.0.70/webapps/spark-task_lib/spark-java-2.4-SNAPSHOT.jar",
                //
                "--arg", "22.1956491323113,113.555733036396" // 經緯度
        };

        // StationIdForUser.main(arg0);//該方法記得註釋，不然會提交錯誤

        //以下方法是爲了方便在本地環境做測試
        Configuration conf = new Configuration();
        String os = System.getProperty("os.name");
        boolean cross_platform = false;
        if (os.contains("Windows")) {
            cross_platform = true;
        }
        conf.setBoolean("mapreduce.app-submission.cross-platform", cross_platform);// 配置使用跨平臺提交任務
        conf.set("fs.defaultFS", "hdfs://192.168.2.9:9000");// 指定namenode
        conf.set("mapreduce.framework.name", "yarn"); // 指定使用yarn框架
        conf.set("yarn.resourcemanager.address", "192.168.2.9:8032");// 指定resourcemanager
        conf.set("yarn.resourcemanager.scheduler.address", "192.168.2.9:8030");// 指定資源分配器
        conf.set("mapreduce.jobhistory.address", "192.168.2.9:10020");
        System.setProperty("SPARK_YARN_MODE", "true");

        SparkConf sparkConf = new SparkConf().setMaster("yarn").setAppName("JavaSparkPi");//appname可以隨便取，master有多種提交方式，請自行選擇
        // SparkConf sparkConf = new SparkConf();
        ClientArguments cArgs = new ClientArguments(arg0);
        new Client(cArgs, conf, sparkConf).run();
    }
}

把jar發佈到web服務器，執行java -jar jar包名 參數 參數
無法正常提交，猜測是jar包衝突。

爲了驗證這一點，寫Java程序驗證。思路大抵是：把本地的jar包存入一個List中，然後再遍歷服務器的jar文件，若有某個jar在本地沒有匹配到，就寫入2.txt（多餘的jar，可以刪除）。

package com.zh.zsr;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

/**
 * 對比服務器文件目錄和Windows文件目錄中，哪個文件是多餘的
 * 
 * @author nanphonfy
 * @time 2016年8月31日 上午9:54:19
 */
public class FilePath {
    public static void main(String[] args) throws IOException {
        String path = "C:\\Users\\sibat\\Desktop\\spark-task_lib";

        FilePath fp = new FilePath();
        List<String> paths = fp.getFiles(path);//可以得到本地的所有jar包
        List<String> names = new ArrayList<>();
        for (String p : paths) {
            String[] temp = p.split("\\\\");//Linux環境爲"/"
            int len = temp.length;
            String yes = temp[len - 1];
            names.add(yes);
        }
        int len = names.size();


        String readFile = "C://Users//sibat//Desktop//1.txt";
        String writeFile = "C://Users//sibat//Desktop//2.txt";
        FileInputStream fis = new FileInputStream(readFile);
        InputStreamReader isw = new InputStreamReader(fis, "GBK");
        BufferedReader br = new BufferedReader(isw);// 把filewriter的寫法寫成FileOutputStream形式
        FileOutputStream fos = new FileOutputStream(writeFile);
        OutputStreamWriter osw = new OutputStreamWriter(fos, "GBK");
        BufferedWriter bw = new BufferedWriter(osw);// 把filewriter的寫法寫成FileOutputStream形式
        String line = "";
        /* 得到服務器的完整jar包名，並把它存入2.txt
        long a = System.currentTimeMillis();
        while ((line = br.readLine()) != null) {
            int len1 = "-rw-r--r-- 1 datum datum 69409 8月 30 17:41 ".length();
            // line.substring(len1+1);

            bw.write(line.substring(len1 + 4));
            bw.newLine();
            bw.flush();
            // System.out.println(line);
        }
        long b = System.currentTimeMillis();
        System.out.println(b - a);
        */

        //把2.txt得到的服務器完整jar包名，拷到1.txt
        long a = System.currentTimeMillis();
        while ((line = br.readLine()) != null) {
            int i;
            for (i = 0; i < len; i++) {
                if (line.equals(names.get(i)))
                    break;
            }
            if (i == len) {
                bw.write(line);
                bw.newLine();
                bw.flush();
            }
        }
        long b = System.currentTimeMillis();
        System.out.println(b - a);
    }

    private List<String> absolutePaths = new LinkedList<>();

    /*
     * 通過遞歸得到某一路徑下所有的目錄及其文件
     */
    public List<String> getFiles(String filePath) {
        File root = new File(filePath);
        File[] files = root.listFiles();
        for (File file : files) {
            if (file.isDirectory()) {
                getFiles(file.getAbsolutePath());
            } else {// 默認爲沒有目錄，只有文件
                absolutePaths.add(file.getAbsolutePath().toString());
            }
        }
        return absolutePaths;
    }
}

這種顯示方法即可切割，得到完整的jar包名。複製進1.txt，通過如上程序去掉jar包名前的字符。
最終發現多出5個jar包，在web服務器中刪除即可。

spark-task.jar ->（依賴的jar包） spark-task_lib
緊接着，發現提交成功，但是spark上的任務卻都是失敗的，記錄日誌如下：

[datum@webserver webapps]$ java -jar spark-task.jar 
log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
16/08/31 10:00:16 INFO Client: Requesting a new application from cluster with 4 NodeManagers
16/08/31 10:00:17 INFO Client: Verifying our application has not requested more than the maximum memory capability of the cluster (8192 MB per container)
16/08/31 10:00:17 INFO Client: Will allocate AM container, with 896 MB memory including 384 MB overhead
16/08/31 10:00:17 INFO Client: Setting up container launch context for our AM
16/08/31 10:00:17 INFO Client: Setting up the launch environment for our AM container
16/08/31 10:00:17 INFO Client: Preparing resources for our AM container
16/08/31 10:00:17 WARN Client: Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME.
16/08/31 10:00:19 INFO Client: Uploading resource file:/tmp/spark-367eaac7-b80e-4517-87b3-d6540fc2cc13/__spark_libs__5719539575137016651.zip -> hdfs://192.168.2.9:9000/user/datum/.sparkStaging/application_1472554805174_0017/__spark_libs__5719539575137016651.zip
16/08/31 10:00:22 INFO Client: Uploading resource file:/data/apps/apache-tomcat-7.0.70/webapps/spark-task_lib/spark-java-2.4-SNAPSHOT.jar -> hdfs://192.168.2.9:9000/user/datum/.sparkStaging/application_1472554805174_0017/spark-java-2.4-SNAPSHOT.jar
16/08/31 10:00:22 INFO Client: Uploading resource file:/tmp/spark-367eaac7-b80e-4517-87b3-d6540fc2cc13/__spark_conf__3363112904998123006.zip -> hdfs://192.168.2.9:9000/user/datum/.sparkStaging/application_1472554805174_0017/__spark_conf__.zip
16/08/31 10:00:22 INFO SecurityManager: Changing view acls to: datum
16/08/31 10:00:22 INFO SecurityManager: Changing modify acls to: datum
16/08/31 10:00:22 INFO SecurityManager: Changing view acls groups to: 
16/08/31 10:00:22 INFO SecurityManager: Changing modify acls groups to: 
16/08/31 10:00:22 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(datum); groups with view permissions: Set(); users  with modify permissions: Set(datum); groups with modify permissions: Set()
16/08/31 10:00:22 INFO Client: Submitting application application_1472554805174_0017 to ResourceManager
16/08/31 10:00:22 INFO YarnClientImpl: Submitted application application_1472554805174_0017
16/08/31 10:00:23 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:23 INFO Client: 
     client token: N/A
     diagnostics: N/A
     ApplicationMaster host: N/A
     ApplicationMaster RPC port: -1
     queue: default
     start time: 1472608887913
     final status: UNDEFINED
     tracking URL: http://master01:8088/proxy/application_1472554805174_0017/
     user: datum
16/08/31 10:00:24 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:25 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:26 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:27 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:28 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:29 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:30 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:31 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:32 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:33 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:34 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:35 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:36 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:37 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:38 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:39 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:40 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:41 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:42 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:43 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:44 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:45 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:46 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:47 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:48 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:49 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
16/08/31 10:00:50 INFO Client: Application report for application_1472554805174_0017 (state: ACCEPTED)
^C16/08/31 10:00:51 INFO ShutdownHookManager: Shutdown hook called
16/08/31 10:00:51 INFO ShutdownHookManager: Deleting directory /tmp/spark-367eaac7-b80e-4517-87b3-d6540fc2cc13

Google了許多資料，猜測程序試圖把web本機的jar包提交到spark上（然而這是沒必要的，因爲本機的jar是從集羣上拷貝下來的）。
嘗試用SparkSubmit方法，出現錯誤。最後查看源碼才發現，原來是新版本的類做了改進，以致於舊教程不適用。代碼如下：

package com.sibat.spark.action;

import org.apache.spark.deploy.SparkSubmit;
/**
 * 
 * @author nanphonfy
 *
 */
public class Task {
    //該方法與舊教程的差別在於，舊教程需要在arg0數組中寫上--jar和--arg參數，而實際上，在新版的spark類中是不需要這麼做的。
    public static void main(String[] args) {
        String arg = null;
        if (args.length == 2) {
            arg = args[0] + "," + args[1];
            System.out.println("通過web項目執行cmd：" + arg);
            String[] arg0 = new String[] { "--master", "spark://master01:7077", //
                    "--class", "cn.sibat.spark.StationIdForUser", //
                    ".../apache-tomcat-7.0.70/webapps/spark-task_lib/spark-java-2.7-SNAPSHOT.jar", //
                    "" + arg + "" };
            SparkSubmit.main(arg0);
        } else {
            String[] arg0 = new String[] { "--master", "spark://master01:7077", //
                    "--class", "cn.sibat.spark.StationIdForUser", //
                    ".../apache-tomcat-7.0.70/webapps/spark-task_lib/spark-java-2.7-SNAPSHOT.jar", //
                    "22.1956491323113,113.555733036396" };
            SparkSubmit.main(arg0);
        }
    }
}

提交成功：

[datum@webserver webapps]$ sh exe.sh
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
16/09/01 14:47:02 INFO SparkContext: Running Spark version 2.0.0
16/09/01 14:47:02 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/09/01 14:47:02 INFO SecurityManager: Changing view acls to: datum
16/09/01 14:47:02 INFO SecurityManager: Changing modify acls to: datum
16/09/01 14:47:02 INFO SecurityManager: Changing view acls groups to: 
16/09/01 14:47:02 INFO SecurityManager: Changing modify acls groups to: 
16/09/01 14:47:02 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(datum); groups with view permissions: Set(); users  with modify permissions: Set(datum); groups with modify permissions: Set()
16/09/01 14:47:03 INFO Utils: Successfully started service 'sparkDriver' on port 55903.
16/09/01 14:47:03 INFO SparkEnv: Registering MapOutputTracker
16/09/01 14:47:03 INFO SparkEnv: Registering BlockManagerMaster
16/09/01 14:47:03 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-10614c25-2d1a-472d-8020-d37463290840
16/09/01 14:47:03 INFO MemoryStore: MemoryStore started with capacity 1106.4 MB
16/09/01 14:47:04 INFO SparkEnv: Registering OutputCommitCoordinator
16/09/01 14:47:04 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.
16/09/01 14:47:04 INFO Utils: Successfully started service 'SparkUI' on port 4041.
16/09/01 14:47:04 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://192.168.2.2:4041
16/09/01 14:47:04 INFO SparkContext: Added JAR file:/data/apps/apache-tomcat-7.0.70/webapps/spark-task_lib/spark-java-2.6-SNAPSHOT.jar at spark://192.168.2.2:55903/jars/spark-java-2.6-SNAPSHOT.jar with timestamp 1472712424481
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Connecting to master spark://master01:7077...
16/09/01 14:47:04 INFO TransportClientFactory: Successfully created connection to master01/192.168.2.9:7077 after 26 ms (0 ms spent in bootstraps)
16/09/01 14:47:04 INFO StandaloneSchedulerBackend: Connected to Spark cluster with app ID app-20160901144819-0004
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20160901144819-0004/0 on worker-20160901101340-192.168.2.13-55623 (192.168.2.13:55623) with 16 cores
16/09/01 14:47:04 INFO StandaloneSchedulerBackend: Granted executor ID app-20160901144819-0004/0 on hostPort 192.168.2.13:55623 with 16 cores, 1024.0 MB RAM
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20160901144819-0004/1 on worker-20160901101426-192.168.2.7-56600 (192.168.2.7:56600) with 8 cores
16/09/01 14:47:04 INFO StandaloneSchedulerBackend: Granted executor ID app-20160901144819-0004/1 on hostPort 192.168.2.7:56600 with 8 cores, 1024.0 MB RAM
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20160901144819-0004/2 on worker-20160901101433-192.168.2.10-52999 (192.168.2.10:52999) with 8 cores
16/09/01 14:47:04 INFO StandaloneSchedulerBackend: Granted executor ID app-20160901144819-0004/2 on hostPort 192.168.2.10:52999 with 8 cores, 1024.0 MB RAM
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20160901144819-0004/3 on worker-20160901101446-192.168.2.6-47335 (192.168.2.6:47335) with 8 cores
16/09/01 14:47:04 INFO StandaloneSchedulerBackend: Granted executor ID app-20160901144819-0004/3 on hostPort 192.168.2.6:47335 with 8 cores, 1024.0 MB RAM
16/09/01 14:47:04 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45142.
16/09/01 14:47:04 INFO NettyBlockTransferService: Server created on 192.168.2.2:45142
16/09/01 14:47:04 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.2.2, 45142)
16/09/01 14:47:04 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.2.2:45142 with 1106.4 MB RAM, BlockManagerId(driver, 192.168.2.2, 45142)
16/09/01 14:47:04 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.2.2, 45142)
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20160901144819-0004/0 is now RUNNING
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20160901144819-0004/1 is now RUNNING
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20160901144819-0004/3 is now RUNNING
16/09/01 14:47:04 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20160901144819-0004/2 is now RUNNING
16/09/01 14:47:06 INFO EventLoggingListener: Logging events to hdfs://master01:9000/historyserverforspark/app-20160901144819-0004
16/09/01 14:47:06 INFO StandaloneSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.0
16/09/01 14:47:06 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Registered executor NettyRpcEndpointRef(null) (192.168.2.13:55909) with ID 0
16/09/01 14:47:06 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Registered executor NettyRpcEndpointRef(null) (192.168.2.6:58228) with ID 3
16/09/01 14:47:06 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.2.13:48271 with 366.3 MB RAM, BlockManagerId(0, 192.168.2.13, 48271)
16/09/01 14:47:06 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.2.6:48243 with 366.3 MB RAM, BlockManagerId(3, 192.168.2.6, 48243)
16/09/01 14:47:06 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Registered executor NettyRpcEndpointRef(null) (192.168.2.10:40581) with ID 2
16/09/01 14:47:07 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.2.10:50607 with 366.3 MB RAM, BlockManagerId(2, 192.168.2.10, 50607)
16/09/01 14:47:07 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 227.3 KB, free 1106.2 MB)
16/09/01 14:47:07 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Registered executor NettyRpcEndpointRef(null) (192.168.2.7:60528) with ID 1
16/09/01 14:47:07 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 20.2 KB, free 1106.2 MB)
16/09/01 14:47:07 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.2.2:45142 (size: 20.2 KB, free: 1106.4 MB)
16/09/01 14:47:07 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.2.7:50861 with 366.3 MB RAM, BlockManagerId(1, 192.168.2.7, 50861)
16/09/01 14:47:07 INFO SparkContext: Created broadcast 0 from textFile at StationIdForUser.java:27
16/09/01 14:47:07 INFO FileInputFormat: Total input paths to process : 1
16/09/01 14:47:07 INFO SparkContext: Starting job: foreach at StationIdForUser.java:36
16/09/01 14:47:07 INFO DAGScheduler: Got job 0 (foreach at StationIdForUser.java:36) with 2 output partitions
16/09/01 14:47:07 INFO DAGScheduler: Final stage: ResultStage 0 (foreach at StationIdForUser.java:36)
16/09/01 14:47:07 INFO DAGScheduler: Parents of final stage: List()
16/09/01 14:47:07 INFO DAGScheduler: Missing parents: List()
16/09/01 14:47:07 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[2] at filter at StationIdForUser.java:36), which has no missing parents
16/09/01 14:47:07 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 1106.2 MB)
16/09/01 14:47:07 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 1106.2 MB)
16/09/01 14:47:07 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.2.2:45142 (size: 2.1 KB, free: 1106.4 MB)
16/09/01 14:47:07 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1012
16/09/01 14:47:07 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at filter at StationIdForUser.java:36)
16/09/01 14:47:07 INFO TaskSchedulerImpl: Adding task set 0.0 with 2 tasks
16/09/01 14:47:07 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.2.6, partition 0, ANY, 5509 bytes)
16/09/01 14:47:07 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 192.168.2.13, partition 1, ANY, 5509 bytes)
16/09/01 14:47:07 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Launching task 0 on executor id: 3 hostname: 192.168.2.6.
16/09/01 14:47:07 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Launching task 1 on executor id: 0 hostname: 192.168.2.13.
16/09/01 14:47:08 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.2.6:48243 (size: 2.1 KB, free: 366.3 MB)
16/09/01 14:47:08 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.2.6:48243 (size: 20.2 KB, free: 366.3 MB)
16/09/01 14:47:08 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.2.13:48271 (size: 2.1 KB, free: 366.3 MB)
16/09/01 14:47:08 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.2.13:48271 (size: 20.2 KB, free: 366.3 MB)
16/09/01 14:47:10 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 2554 ms on 192.168.2.13 (1/2)
16/09/01 14:47:10 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 2921 ms on 192.168.2.6 (2/2)
16/09/01 14:47:10 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 
16/09/01 14:47:10 INFO DAGScheduler: ResultStage 0 (foreach at StationIdForUser.java:36) finished in 2.930 s
16/09/01 14:47:10 INFO DAGScheduler: Job 0 finished: foreach at StationIdForUser.java:36, took 3.130529 s
16/09/01 14:47:10 INFO SparkUI: Stopped Spark web UI at http://192.168.2.2:4041
16/09/01 14:47:10 INFO StandaloneSchedulerBackend: Shutting down all executors
16/09/01 14:47:10 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Asking each executor to shut down
16/09/01 14:47:10 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
16/09/01 14:47:10 INFO MemoryStore: MemoryStore cleared
16/09/01 14:47:10 INFO BlockManager: BlockManager stopped
16/09/01 14:47:10 INFO BlockManagerMaster: BlockManagerMaster stopped
16/09/01 14:47:10 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
16/09/01 14:47:10 INFO SparkContext: Successfully stopped SparkContext
16/09/01 14:47:10 INFO ShutdownHookManager: Shutdown hook called
16/09/01 14:47:10 INFO ShutdownHookManager: Deleting directory /tmp/spark-5b4d875e-2252-4767-b63b-b297b9d559f9

關於提交經緯度到spark集羣：
①參考網絡資料，大抵是基於spark1.4.1-Hadoop2.6.0（而我們的環境是spark2.0.0-Hadoop2.6.0），正因爲版本的不一致，導致教程與實操出現出入；
②先測試單機版jar，本地需引入單獨開發的jar所依賴的spark包；
③然後測試集羣，需要把多餘jar包去掉（通過程序遍歷匹配）；
④提交任務，卻執行失敗（發現教程的某些參數在新版本中不需要寫前綴）；
⑤web服務器環境（需拷入spark的200多個jar包）的spark包與項目框架包存在大量衝突，所以web版嵌入調用spark程序不是一種好的解決方案；
⑥通過Java程序執行Linux命令，來執行提交spark任務的jar包，解決jar包衝突問題。
思路：把提交spark任務的程序與uhuibao項目分離，通過Java程序執行外部環境的Linux指令，以執行spark-task.jar，並把經度緯度設置爲指令參數（它依賴於spark的衆多包）。

controller層

package cn.sibat.uhuibao.api;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;

import cn.sibat.uhuibao.redis.service.RealTimeUserService;
import cn.sibat.uhuibao.redis.service.impl.RealTimeUserServiceImpl;
import cn.sibat.uhuibao.util.JsonUtil;
import cn.sibat.uhuibao.util.Status;

/**
 * 提交任務到spark集羣執行
 * 
 * @author nanphonfy
 *
 */
@Path("spark")
public class SparkApi {
    @GET
    @Path("recommend")
    @Produces(MediaType.APPLICATION_JSON)
    public String searchRealTimeRecommendation(
            @QueryParam("lng") @DefaultValue("null") String lng,
            @QueryParam("lat") @DefaultValue("null") String lat) {
        if (lng == null || lat == null || lng.isEmpty() || lat.isEmpty()) {
            return JsonUtil.getResponse(Status.PARA_ERROR).toString();
        }

        Runtime rt = Runtime.getRuntime();
        String command = "java -jar .../apache-tomcat-7.0.70/webapps/spark-task.jar "
                + lng + " " + lat;
        System.out.println(command);

        new Thread(new Runnable() {// 以讓spark有充足的時間做測試
                    @Override
                    public void run() {
                        try {
                            Thread.sleep(1000 * 10);// 睡眠10秒
                            System.out.println("等待10秒！");
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }
                });

        try {
            rt.exec(command);// 執行Linux命令
        } catch (IOException e) {
            e.printStackTrace();
            return JsonUtil.getResponse(Status.SYS_ERROR).toString();
        }

        System.out.println("執行成功");
        String key = lng + "," + lat;
        RealTimeUserService rs = RealTimeUserServiceImpl.getInstance();
        String list = rs.getRealTimeRecommendation(key);
        if (list == null) {
            return JsonUtil.getResponse(Status.NOT_FOUND).toString();
        } else {
            String[] result = list.split(",");
            int len = result.length;
            List<String> values = new ArrayList<String>();
            for (int i = 0; i < len; i++) {
                values.add(result[i].trim());
            }
            return JsonUtil.getDataResponse(Status.OK, values).toString();
        }
    }
}

該項目的控制層是用web service框架Jersey，而spark包也存在Jersey包，版本不一定與項目一致。除此之外，想嵌入到一個成型的項目，勢必會引起衝突，必須手工除之，然而沒有這個必要。

“由於調用 Runtime.exec方法所創建的子進程沒有自己的終端或控制檯，因此該子進程的標準IO（如stdin,stdou,stderr）都通過Process.getOutputStream()，Process.getInputStream()， Process.getErrorStream() 方法重定向給它的父進程了”
參考：
http://blog.csdn.net/fansy1990/article/details/48001013
http://wangbaoaiboy.blog.163.com/blog/static/52111910201111892938552/

---

作者： @nanphonfy
Email: nanphonfy (Nfzone) gmail.com 請將(Nfzone)換成@

---