spark 安裝實戰

 

1. wget https://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz

tar -zxvf spark-2.2.0-bin-hadoop2.7.tgz

mv spark-2.2.0-bin-hadoop2.7 spark-2.2.0

vi /etc/profile

export SPARK_HOME=/usr/local/spark-2.2.0
export PATH=$PATH:$SPARK_HOME/bin

cd /opt/spark-2.2.0/conf

cp spark-env.sh.template spark-env.sh

vi conf/spark-env.sh

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
export SCALA_HOME=/usr/scala-2.11.8
export HADOOP_HOME=/usr/local/hadoop-2.7.2
export HADOOP_CONF_DIR=/usr/local/hadoop-2.7.2/etc/hadoop
export SPARK_MASTER_IP=SparkMaster
export SPARK_WORKER_MEMORY=4g
export SPARK_WORKER_CORES=2
export SPARK_WORKER_INSTANCES=1

wget  https://www.apache.org/dist/hadoop/common/hadoop-2.8.4/hadoop-2.8.4.tar.gz

wget https://www.scala-lang.org/download/2.11.8.html/scala-2.12.8.tgz

tar -zxvf hadoop-2.8.4.tar.gz

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
export SPARK_HOME=/opt/spark-2.2.0
export HADOOP_HOME=/opt/hadoop-2.8.4
export PATH=$PATH:$SPARK_HOME/bin:$HADOOP_HOME/bin