spark sql with hive

spark standalone

spark-1.5.0-bin-hadoop2.6

spark-env.sh配置

export SPARK_MASTER_IP=172.16.31.11

export SPARK_MASTER_PORT=7077

export SPARK_WORKER_CORES=1

export SPARK_WORDER_INSTANCES=1

export SPARK_WORKER_MEMORY=1g

export MASTER=spark://172.16.31.11:7077

slaves

MRguan

啓動

./sbin/start-all.sh

./bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode client lib/spark-examples-1.5.0-hadoop2.6.0.jar

spark sql with hive

spark-env.sh添加

export HADOOP_CONF_DIR=/opt/beh/core/hadoop/etc/hadoop

export YARN_CONF_DIR=/opt/beh/core/hadoop/etc/hadoop

export HADOOP_HOME=/opt/beh/core/hadoop

export SPARK_JAR=/opt/beh/core/spark/lib/spark-assembly-1.5.0-hadoop2.6.0.jar

export SPARK_CLASSPATH=/opt/beh/core/spark/lib/mysql-connector-java-5.1.30-bin.jar

 將hive-site.xml和hive-log4j.properties複製到$SPARK_HOME/conf下

 將 $HIVE_HOME/lib下 mysql-connector-java-5.1.30-bin.jar複製到$SPARK_HOME/lib下

注意版本 不然報錯

17/02/28 09:12:48 WARN HiveMetaStore: Retrying creating default database after error: Error creating transactional connection factory

javax.jdo.JDOFatalInternalException: Error creating transactional connection factory

重啓spark

執行./bin/spark-sql.sh