解壓
echo 'SPARK_HOME=$HOME/program/spark' >> ~/.bash_profile
echo 'PATH=$PATH:SPARK_HOME/bin' >> ~/.bash_profile
tar -zxvf spark.tar
mv spark $SPARK_HOME
修改配置文件
- spark-env.sh
cd $SPARK_HOME/conf
cp spark-env.sh.template spark-env.sh
vi spark-env.sh
export SCALA_HOME=/usr/local/bigdata/scala
export JAVA_HOME=/usr/local/bigdata/java/jdk1.8.0_211
export HADOOP_HOME=/usr/local/bigdata/hadoop-2.7.1
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
SPARK_MASTER_IP=Master
SPARK_LOCAL_DIRS=/usr/local/bigdata/spark-2.4.3
SPARK_DRIVER_MEMORY=512M
- slaves
cp slaves.template slaves
vi slaves
slave01
slave02
啓動
cd $HADOOP_HOME/sbin/
./start-all.sh
./start-history-server.sh
cd $SPARK_HOME/sbin/
./start-all.sh
./start-history-server.sh
查看
http://lx81:8080/
命令行模式
pyspark --master spark://lx81:7077 # python
spark-shell --master spark://lx81:7077 # scala