Spark2.4.5編譯

下載spark源碼
[root@hadoop001 source]# wget https://downloads.apache.org/spark/spark-2.4.5/spark-2.4.5.tgz

解壓
[root@hadoop001 source]# tar -zxvf spark-2.4.5.tgz
編譯文檔
http://spark.apache.org/docs/latest/building-spark.html
編譯前置要求
Maven 3.6.*
Java 8+
Scala 2.11.8
安裝jdk
[root@hadoop001 ~]# mkdir /usr/java
 
[root@hadoop001 ~]# tar -zxvf /home/hadoop/app/jdk-8u45-linux-x64.gz -C /usr/java/
[root@hadoop001 java]# chown -R root:root jdk1.8.0_212/
[root@hadoop001 java]# chown -R root:root jdk1.8.0_212/*
 
[hadoop@hadoop001 zookeeper]$ vi ~/.bash_profile 
 
export JAVA_HOME=/usr/java/jdk1.8.0_45/
PATH=$JAVA_HOME/bin:$PATH:$HOME/bin
 
export PATH
安裝maven
[root@hadoop001 software]# wget http://us.mirrors.quenda.co/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
[root@hadoop001 software]# tar -zxvf apache-maven-3.6.3-bin.tar.gz -C /root/app/

配置環境變量
# .bash_profile

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
     . ~/.bashrc
fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin
JAVA_HOME=/usr/java/jdk1.8.0_212
PATH=$PATH:$JAVA_HOME/bin
MAVEN_HOME=/root/app/apache-maven-3.6.3
PATH=$PATH:$MAVEN_HOME/bin
export PATH


配置maven倉庫位置
[root@hadoop001 apache-maven-3.6.3]# vi /root/app/apache-maven-3.6.3/conf/settings.xml
<localRepository>/root/maven_repo</localRepository>
安裝scala
[root@hadoop001 software]# wget https://downloads.lightbend.com/scala/2.13.2/scala-2.13.2.tgz
[root@hadoop001 software]# tar -zxvf scala-2.13.2.tgz -C /root/app/

配置環境變量
# .bash_profile

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
     . ~/.bashrc
fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin
JAVA_HOME=/usr/java/jdk1.8.0_212
PATH=$PATH:$JAVA_HOME/bin
MAVEN_HOME=/root/app/apache-maven-3.6.3
SCALA_HOME=/root/app/scala-2.13.2
PATH=$PATH:$MAVEN_HOME/bin:$SCALA_HOME/bin
export PATH
[root@hadoop001 ~]# yum -y install git
開始編譯
[root@hadoop001 spark-2.4.5]# ./dev/make-distribution.sh --name 2.6.0-cdh5.7.0 --tgz -Phadoop-2.6 -Dhadoop.version=2.6.0-cdh5.7.0 -Phive -Phive-thriftserver -Pyarn

 

發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章