dolphinscheduler部署文檔（單機版）

下載

後端: wget https://downloads.apache.org/incubator/dolphinscheduler/1.2.0/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin.tar.gz
前端: wget https://downloads.apache.org/incubator/dolphinscheduler/1.2.0/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-front-bin.tar.gz

部署後端

安裝

mkdir -p /opt/app/dolphinscheduler    # ds的安裝目錄
mkdir -p /opt/soft/dolphinscheduler    # ds的下載目錄
cd /opt/soft/dolphinscheduler
tar -zxvf apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin.tar.gz -C /opt/app/dolphinscheduler
mv apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin  dolphinscheduler-backend

tree -L 1
.
├── bin           # 基礎服務啓動腳本
├── conf          # 項目配置文件
├── DISCLAIMER-WIP# DISCLAIMER文件
├── install.sh    # 一鍵部署腳本
├── lib           # 項目依賴jar包，包括各個模塊jar和第三方jar
├── LICENSE       # LICENSE文件
├── licenses      # 運行時license
├── NOTICE        # NOTICE文件
├── script        # 集羣啓動、停止和服務監控啓停腳本
└── sql           # 項目依賴sql文件
————————————————

創建用戶並授權

useradd dolphinscheduler    # add user dolphinscheduler

echo "dolphinscheduler" | passwd --stdin dolphinscheduler    # modify user password

# 配置sudo免密
sed -i '$adolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers（報錯可以使用root用戶直接編輯/etc/sudoers）
# 報錯需要sudo權限，進入/etc/sudoers 將 Default requiretty註釋掉

# 修改目錄權限，使得部署用戶對dolphinscheduler-backend目錄有操作權限
chown -R dolphinscheduler:dolphinscheduler dolphinscheduler-backend

# 修改安裝目錄權限，後面的一鍵部署腳本部署完成會自動啓動，不給權限會無法啓動
chown -R dolphinscheduler:dolphinscheduler /opt/app/dolphinscheduler-backend

ssh免密登錄

su dolphinscheduler

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys

數據庫初始化

• 創建數據庫

mysql -uroot -p
-- 創建數據庫
CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
-- 創建用戶及授權（可能需要修改數據庫密碼等級）
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%' IDENTIFIED BY 'dolphinscheduler';
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'localhost' IDENTIFIED BY 'dolphinscheduler';
flush privileges;

• 修改配置

vi conf/application-dao.properties
# 需要註釋掉postgresql的信息
# postgre
#spring.datasource.driver-class-name=org.postgresql.Driver
#spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
# mysql
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://master:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8     #需要修改ip，本機localhost即可
spring.datasource.username=dolphinscheduler
spring.datasource.password=dolphinscheduler

• 執行腳本初始化數據表

sh script/create-dolphinscheduler.sh

尖叫提示：執行腳本前需要下載mysql-java驅動程序包（需要選擇對應mysql版本的驅動，不是bin），https://downloads.mysql.com/archives/c-j/

修改環境變量

vi conf/env/.dolphinscheduler_env.sh    # 這是一個隱藏文件
# JAVA_HOME 和 PATH 是必須要配置的，其他沒有用到的可以忽略或者註釋掉
#export HADOOP_HOME=/opt/soft/hadoop
#export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
#export SPARK_HOME1=/opt/soft/spark1
#export SPARK_HOME2=/opt/soft/spark2
#export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/usr/java/jdk1.8.0_111
#export HIVE_HOME=/opt/soft/hive
#export FLINK_HOME=/opt/soft/flink
export PATH=$HADOOP_HOME/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$PATH

jdk軟連接

echo $JAVA_HOME    # 沒有軟連接後面在啓動腳本的時候會報錯：nohup /bin/java不存在
sudo ln -s /usr/java/jdk1.8.0_111/bin/java /usr/bin/java

修改一鍵部署腳本

vi install.sh
# for example postgresql or mysql ...
dbtype="mysql"

# db config
# db address and port
dbhost="master:3306"

# db name
dbname="dolphinscheduler"

# db username
username="dolphinscheduler"

# db passwprd
# Note: if there are special characters, please use the \ transfer character to transfer
passowrd="dolphinscheduler"

# conf/config/install_config.conf config
# Note: the installation path is not the same as the current path (pwd)
# 將應用安裝到哪個目錄下
installPath="/opt/app/dolphinscheduler"

# deployment user
# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
# 使用哪個用戶部署
deployUser="dolphinscheduler"

# zk cluster,集羣方式以逗號分隔
zkQuorum="master:2181"

# install hosts
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
# 在哪些機器上部署
ips="master"

# conf/config/run_config.conf config
# run master machine
# Note: list of hosts hostname for deploying master
masters="master"

# run worker machine
# note: list of machine hostnames for deploying workers
workers="master"

# run alert machine
# note: list of machine hostnames for deploying alert server
alertServer="master"

# run api machine
# note: list of machine hostnames for deploying api server
apiServers="master"

# 郵件服務
# alert config
# mail protocol
mailProtocol="SMTP"

# mail server host
mailServerHost="***"

# mail server port
mailServerPort="***"

# sender
mailSender="*****"

# user
mailUser="*****"

# sender password
mailPassword="*****"

# TLS mail protocol support
starttlsEnable="false"

sslTrust="*****"

# SSL mail protocol support
# note: The SSL protocol is enabled by default.
# only one of TLS and SSL can be in the true state.
sslEnable="true"

# download excel path
xlsFilePath="/tmp/xls"

# 業務用到的比如sql等資源文件上傳到哪裏，可以設置：HDFS,S3,NONE，單機如果想使用本地文件系統，請配置爲HDFS，因爲HDFS支持本地文件系統；如果不需要資源上傳功能請選擇NONE。強調一點：使用本地文件系統不需要部署hadoop
resUploadStartupType="HDFS"

# 這裏以保存到本地文件系統爲例
#注：但是如果你想上傳到HDFS的話，NameNode啓用了HA，則需要將core-site.xml和hdfs-site.xml放到conf目錄下，本例即是放到/opt/dolphinscheduler/conf下面，並配置namenode cluster名稱；如果NameNode不是HA,則修改爲具體的ip或者主機名即可
defaultFS="hdfs://hadoopcluster/dolphinscheduler"    #hdfs://{具體的ip/主機名}:8020

# 如果ResourceManager是HA，則配置爲ResourceManager節點的主備ip或者hostname,比如"192.168.xx.xx,192.168.xx.xx"，否則如果是單ResourceManager或者根本沒用到yarn,請配置yarnHaIps=""即可，我這裏沒用到yarn，配置爲""
yarnHaIps=""

# 如果是單ResourceManager，則配置爲ResourceManager節點ip或主機名，否則保持默認值即可。我這裏沒用到yarn，保持默認
singleYarnIp=""

# 由於hdfs支持本地文件系統，需要確保本地文件夾存在且有讀寫權限
hdfsPath="/data/dolphinscheduler"

安裝python的zookeeper工具kazoo

#安裝pip
sudo yum -y install python-pip;  #ubuntu請使用 sudo apt-get install python-pip
sudo pip install kazoo;

# 注意：如果yum沒找到python-pip，也可以通過下面方式安裝
sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
sudo python get-pip.py  # 如果是python3，使用sudo python3 get-pip.py
#然後
sudo pip install kazoo;

執行一鍵部署腳本

su dolphinscheduler

sh install.sh

查看進程是否啓動

jps

MasterServer         ----- master服務
WorkerServer         ----- worker服務
LoggerServer         ----- logger服務
ApiApplicationServer ----- api服務
AlertServer          ----- alert服務

日誌目錄結構

logs/
    ├── dolphinscheduler-alert-server.log
    ├── dolphinscheduler-master-server.log
    |—— dolphinscheduler-worker-server.log
    |—— dolphinscheduler-api-server.log
    |—— dolphinscheduler-logger-server.log

部署前端

解壓

tar -zxvf apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-front-bin.tar.gz -C /opt/soft/dolphinscheduler;
mv apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-front-bin dolphinscheduler-ui

部署

自動化部署

• 執行自動化部署腳本

cd dolphinscheduler-ui;
sh ./install-dolphinscheduler-ui.sh;
執行後，會在運行中請鍵入前端端口，默認端口是8888，如果選擇默認，鍵入回車，或者鍵入其他端口
然後會讓鍵入跟前端ui交互的api-server的ip
接着是讓鍵入跟前端ui交互的api-server的port
接着是操作系統選擇
等待部署完成

• 修改nginx配置

vi /etc/nginx/nginx.conf
# add param 在http方法體內添加即可
client_max_body_size 1024m;
systemctl restart nginx

手動部署

• 安裝nginx，官網下載: http://nginx.org/en/download.html 或者 yum install nginx -y
• 修改nginx配置文件

vi /etc/nginx/nginx.conf
server {
  listen       8888;# 訪問端口(自行修改)
  server_name  localhost;
  #charset koi8-r;
  #access_log  /var/log/nginx/host.access.log  main;
  location / {
      root   /opt/app/dolphinscheduler-ui/dist;      # 前端解壓的dist目錄地址(自行修改)
      index  index.html index.html;
  }
  location /dolphinscheduler {
      proxy_pass http://localhost:12345;    # api服務地址(自行修改)
      proxy_set_header Host $host;
      proxy_set_header X-Real-IP $remote_addr;
      proxy_set_header x_real_ipP $remote_addr;
      proxy_set_header remote_addr $remote_addr;
      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      proxy_http_version 1.1;
      proxy_connect_timeout 4s;
      proxy_read_timeout 30s;
      proxy_send_timeout 12s;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "upgrade";
  }
  #error_page  404              /404.html;
  # redirect server error pages to the static page /50x.html
  #
  error_page   500 502 503 504  /50x.html;
  location = /50x.html {
      root   /usr/share/nginx/html;
  }
}

初始賬號密碼

地址：192.168.xx.xx:8888 用戶名密碼：admin/dolphinscheduler123

指令

啓動：
    啓動所有進程：sh bin/start-all.sh
    啓動master-server：sh bin/dolphinscheduler-daemon.sh start master-server
    啓動worker-server：sh bin/dolphinscheduler-daemon.sh start worker-server
    啓動logger-server：sh bin/dolphinscheduler-daemon.sh start logger-server
    啓動alert-server：sh bin/dolphinscheduler-daemon.sh start alert-server
    啓動api-server：sh bin/dolphinscheduler-daemon.sh start api-server

停止：
    停止所有進程：sh bin/stop-all.sh
    停止master-server：sh bin/dolphinscheduler-daemon.sh stop master-server
    停止worker-server：sh bin/dolphinscheduler-daemon.sh stop worker-server
    停止logger-server：sh bin/dolphinscheduler-daemon.sh stop logger-server
    停止alert-server：sh bin/dolphinscheduler-daemon.sh stop alert-server
    停止api-server：sh bin/dolphinscheduler-daemon.sh stop api-server