環境
三臺主機
pxc1:192.168.7.71
pxc2:192.168.7.72
pxc3:192.168.7.73
firewalld 和 selinux都關閉，保證時間同步；如果已安裝MySQL，必須卸載！！！
OS 版本
[root@pxc1 ~]#cat /etc/redhat-release
CentOS Linux release 7.7.1908 (Core)
操作步驟
pxc1主機
1.安裝輔助工具Ansible，前提三臺主機做好key驗證
[root@pxc1 ~]#yum -y install ansible
1.1/etc/ansible/hosts配置，在最後一行後面添加以下配置
[pxcservers]
192.168.7.71
192.168.7.72
192.168.7.73
1.2/etc/ansible/ansible.cfg配置
module_name = shell #115行，將默認的模塊command該爲shell
host_key_checking = False #71行，去掉前面的註釋
2.安裝Percona XtraDB Cluster 5.7
# 1.使用清華大學的yum源，官方源太慢了
[root@pxc1 ~]#vim /etc/yum.repos.d/pxc.repo
[percona]
name=percona_repo
baseurl=https://mirrors.tuna.tsinghua.edu.cn/percona/release/$releasever/RPMS/$basearch
enabled=1
gpgcheck=0
# 2.同時將配置文件傳給pxc2和pxc3
[root@pxc1 ~]#ansible 'pxcservers:!192.168.7.71' -m copy -a 'src=/etc/yum.repos.d/pxc.repo dest=/etc/yum.repos.d'
192.168.7.73 | CHANGED => {
# 3.查看三臺主機的pxc.repo文件是否一致
[root@pxc1 ~]#ansible 'pxcservers' -a 'cat /etc/yum.repos.d/pxc.repo'
192.168.7.73 | CHANGED | rc=0 >>
[percona]
name=percona_repo
baseurl=https://mirrors.tuna.tsinghua.edu.cn/percona/release/$releasever/RPMS/$basearch
enabled=1
gpgcheck=0
192.168.7.72 | CHANGED | rc=0 >>
[percona]
name=percona_repo
baseurl=https://mirrors.tuna.tsinghua.edu.cn/percona/release/$releasever/RPMS/$basearch
enabled=1
gpgcheck=0
192.168.7.71 | CHANGED | rc=0 >>
[percona]
name=percona_repo
baseurl=https://mirrors.tuna.tsinghua.edu.cn/percona/release/$releasever/RPMS/$basearch
enabled=1
gpgcheck=0
# 4.執行以下命令，在三臺主機都安裝PXC 5.7
[root@pxc1 ~]#ansible 'pxcservers' -a 'yum install Percona-XtraDB-Cluster-57 -y'
3.修改配置文件
# /etc/my.cnf爲主配置文件，當前版本中，其餘的配置文件都放在/etc/percona-xtradb-cluster.conf.d目錄裏，包括mysqld.cnf，mysqld_safe.cnf，wsrep.cnf 三個文件
[root@pxc1 ~]#tree /etc/percona-xtradb-cluster.conf.d/
/etc/percona-xtradb-cluster.conf.d/
├── mysqld.cnf
├── mysqld_safe.cnf
└── wsrep.cnf
0 directories, 3 files
[root@pxc1 ~]#egrep -v "^#|^$" /etc/percona-xtradb-cluster.conf.d/wsrep.cnf
[mysqld]
wsrep_provider=/usr/lib64/galera3/libgalera_smm.so
wsrep_cluster_address=gcomm://192.168.7.71,192.168.7.72,192.168.7.73 #集羣中每個幾點的IP
binlog_format=ROW
default_storage_engine=InnoDB
wsrep_slave_threads= 8
wsrep_log_conflicts
innodb_autoinc_lock_mode=2
wsrep_node_address=192.168.7.71 #取消行首註釋，指定本節點的IP
wsrep_cluster_name=pxc-cluster
wsrep_node_name=pxc-cluster-node-1 #本節點在集羣中的名稱
pxc_strict_mode=ENFORCING
wsrep_sst_method=xtrabackup-v2
wsrep_sst_auth="sstuser:123.com" #取消行首註釋，並且修改密碼
3.1將配置文件copy給pxc2，pxc3兩臺主機，然後稍作修改，主要項是wsrep_node_address和wsrep_node_name
# 1.copy
[root@pxc1 ~]#ansible 'pxcservers:!192.168.7.71' -m copy -a 'src=/etc/percona-xtradb-cluster.conf.d/wsrep.cnf dest=/etc/percona-xtradb-cluster.conf.d'
# 2.修改pxc2主機的配置文件
[root@pxc1 ~]#ansible '192.168.7.72' -a 'sed -i -e s/wsrep_node_address=192.168.7.71/wsrep_node_address=192.168.7.72/ -e s/wsrep_node_name=pxc-cluster-node-1/wsrep_node_name=pxc-cluster-node-2/ /etc/percona-xtradb-cluster.conf.d/wsrep.cnf'
# 3.修改pxc3主機的配置文件
[root@pxc1 ~]#ansible '192.168.7.73' -a 'sed -i -e s/wsrep_node_address=192.168.7.71/wsrep_node_address=192.168.7.73/ -e s/wsrep_node_name=pxc-cluster-node-1/wsrep_node_name=pxc-cluster-node-3/ /etc/percona-xtradb-cluster.conf.d/wsrep.cnf'
# 4.查看三個幾點的配置文件信息
[root@pxc1 ~]#ansible 'pxcservers' -a 'egrep -v "^#|^$" /etc/percona-xtradb-cluster.conf.d/wsrep.cnf'
[root@pxc1 ~]#ansible 'pxcservers' -a 'egrep -v "^#|^$" /etc/percona-xtradb-cluster.conf.d/wsrep.cnf'
192.168.7.72 | CHANGED | rc=0 >>
[mysqld]
wsrep_provider=/usr/lib64/galera3/libgalera_smm.so
wsrep_cluster_address=gcomm://192.168.7.71,192.168.7.72,192.168.7.73
binlog_format=ROW
default_storage_engine=InnoDB
wsrep_slave_threads= 8
wsrep_log_conflicts
innodb_autoinc_lock_mode=2
wsrep_node_address=192.168.7.72
wsrep_cluster_name=pxc-cluster
wsrep_node_name=pxc-cluster-node-2
pxc_strict_mode=ENFORCING
wsrep_sst_method=xtrabackup-v2
wsrep_sst_auth="sstuser:123.com"
192.168.7.73 | CHANGED | rc=0 >>
[mysqld]
wsrep_provider=/usr/lib64/galera3/libgalera_smm.so
wsrep_cluster_address=gcomm://192.168.7.71,192.168.7.72,192.168.7.73
binlog_format=ROW
default_storage_engine=InnoDB
wsrep_slave_threads= 8
wsrep_log_conflicts
innodb_autoinc_lock_mode=2
wsrep_node_address=192.168.7.73
wsrep_cluster_name=pxc-cluster
wsrep_node_name=pxc-cluster-node-3
pxc_strict_mode=ENFORCING
wsrep_sst_method=xtrabackup-v2
wsrep_sst_auth="sstuser:123.com"
192.168.7.71 | CHANGED | rc=0 >>
[mysqld]
wsrep_provider=/usr/lib64/galera3/libgalera_smm.so
wsrep_cluster_address=gcomm://192.168.7.71,192.168.7.72,192.168.7.73
binlog_format=ROW
default_storage_engine=InnoDB
wsrep_slave_threads= 8
wsrep_log_conflicts
innodb_autoinc_lock_mode=2
wsrep_node_address=192.168.7.71
wsrep_cluster_name=pxc-cluster
wsrep_node_name=pxc-cluster-node-1
pxc_strict_mode=ENFORCING
wsrep_sst_method=xtrabackup-v2
wsrep_sst_auth="sstuser:123.com"
注：儘管Galera Cluster不再需要通過binlog的形式進行同步，但還是建議在配置文件中開啓二進制日誌功能，原因是後期如果有新節點需要加入，老節點通過SST全量傳輸的方式向新節點傳輸數據，很可能會拖垮集羣性能，所以讓新節點先通過binlog方式完成同步後再加入集羣會是一種更好的選擇
4.啓動集羣中的第一個節點
[root@pxc1 ~]#systemctl start [email protected]
[root@pxc1 ~]#systemctl enable [email protected]
# PXC最常使用的端口號如下，其中3306、4567啓動的時候就會監聽
4567：組成員之間進行溝通的端口號
3306：數據庫對外服務的端口號
4444：請求SST的端口號
4568：用於傳輸IST的端口號
5.修改初始化密碼，並且創建及授權sstuser用戶
[root@pxc1 ~]#mysql -p`awk /root@localhost/'{print $NF}' /var/log/mysqld.log`
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 22
Server version: 5.7.27-30-57-log
Copyright (c) 2009-2019 Percona LLC and/or its affiliates
Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> alter user 'root'@'localhost' identified by '123.com';
Query OK, 0 rows affected (0.01 sec)
mysql> create user 'sstuser'@'localhost' identified by '123.com';
Query OK, 0 rows affected (0.01 sec)
mysql> grant reload,lock tables,process,replication client on *.* to 'sstuser'@'localhost';
Query OK, 0 rows affected (0.00 sec)
6.查看相關的狀態變量
mysql> show status like 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 1 |
+--------------------+-------+
1 row in set (0.01 sec)
# 表示該Galera集羣中只有一個節點
mysql> show status like 'wsrep_cluster_status';
+----------------------+---------+
| Variable_name | Value |
+----------------------+---------+
| wsrep_cluster_status | Primary |
+----------------------+---------+
1 row in set (0.00 sec)
# 表示該節點在集羣中的狀態爲Primary，且已經完全連接並準備好
mysql> show status like 'wsrep_local_state%';
+---------------------------+--------------------------------------+
| Variable_name | Value |
+---------------------------+--------------------------------------+
| wsrep_local_state_uuid | b0fdc391-156e-11ea-89de-331ca2945d2c |
| wsrep_local_state | 4 |
| wsrep_local_state_comment | Synced |
+---------------------------+--------------------------------------+
3 rows in set (0.00 sec)
#狀態爲Synced(4)，表示數據已同步完成(因爲是第一個引導節點，無數據需要同步)。 如果狀態是Joiner, 意味着 SST 沒有完成. 只有所有節點狀態是Synced，纔可以加新節點
7.啓動PXC集羣中pxc2,pxc3節點
[root@pxc1 ~]#ansible 'pxcservers:!192.168.7.71' -a 'systemctl enable --now mysql'
pxc2主機
8.查看集羣狀態，驗證集羣是否成功（也可以是其它任意節點）
[root@pxc2 ~]#mysql -p123.com
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 12
Server version: 5.7.27-30-57-log Percona XtraDB Cluster (GPL), Release rel30, Revision 64987d4, WSREP version 31.39, wsrep_31.39
Copyright (c) 2009-2019 Percona LLC and/or its affiliates
Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> show variables like 'wsrep_node_name';
+-----------------+--------------------+
| Variable_name | Value |
+-----------------+--------------------+
| wsrep_node_name | pxc-cluster-node-2 |
+-----------------+--------------------+
1 row in set (0.01 sec)
mysql> show variables like 'wsrep_node_address';
+--------------------+--------------+
| Variable_name | Value |
+--------------------+--------------+
| wsrep_node_address | 192.168.7.72 |
+--------------------+--------------+
1 row in set (0.00 sec)
mysql> show variables like 'wsrep_on';
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| wsrep_on | ON |
+---------------+-------+
1 row in set (0.00 sec)
mysql> show status like 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
1 row in set (0.00 sec)
pxc3主機
9.創建數據庫（也可以是其它任意節點）
# 查看當前的數據庫
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| mysql |
| performance_schema |
| sys |
+--------------------+
4 rows in set (0.00 sec)
# 創建db1數據庫
[root@pxc1 ~]#ansible '192.168.7.73' -a 'mysql -p123.com -e "create database db1;\n"'
192.168.7.73 | CHANGED | rc=0 >>
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
# 查看集羣中的其它節點是否同步新建的db1數據庫
[root@pxc1 ~]#ansible 'pxcservers' -a 'mysql -p123.com -e "show databases\n"'
192.168.7.73 | CHANGED | rc=0 >>
PAGER set to stdout
Database
information_schema
db1
mysql
performance_schema
sysmysql: [Warning] Using a password on the command line interface can be insecure.
192.168.7.72 | CHANGED | rc=0 >>
PAGER set to stdout
Database
information_schema
db1
mysql
performance_schema
sysmysql: [Warning] Using a password on the command line interface can be insecure.
192.168.7.71 | CHANGED | rc=0 >>
PAGER set to stdout
Database
information_schema
db1
mysql
performance_schema
sysmysql: [Warning] Using a password on the command line interface can be insecure.
10.在集羣中的所有節點同時創建db2數據庫，只有一個節點會成功
[root@pxc1 ~]#ansible 'pxcservers' -a 'mysql -p123.com -e "create database db2\n"'
192.168.7.73 | FAILED | rc=1 >>
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
ERROR 1007 (HY000) at line 1: Can't create database 'db2'; database existsnon-zero return code
192.168.7.72 | CHANGED | rc=0 >>
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
192.168.7.71 | FAILED | rc=1 >>
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
ERROR 1007 (HY000) at line 1: Can't create database 'db2'; database existsnon-zero return code
在PXC集羣中加入節點
一個節點加入到Galera集羣有兩種情況：新節點加入集羣、暫時離組的成員再次加入集羣
1）節點加入Galera集羣
新節點加入集羣時，需要從當前集羣中選擇一個Donor節點來同步數據，也就是所謂的
state_snapshot_tranfer(SST)過程。SST同步數據的方式由選項wsrep_sst_method決定，一般選擇的是xtrabackup。必須注意，新節點加入Galera時，會刪除新節點上所有已有數據，再通過xtrabackup(假設使用的是該方式)從Donor處完整備份所有數據進行恢復。所以，如果數據量很大，新節點加入過程會很慢。而且，在一個新節點成爲Synced狀態之前，不要同時加入其它新節點，否則很容易將集羣壓垮。
如果是這種情況，可以考慮使用wsrep_sst_method=rsync來做增量同步，既然是增量同步，最好保證新節點上已經有一部分數據基礎，否則和全量同步沒什麼區別，且這樣會對Donor節點加上全局read only鎖。
2）舊節點加入Galera集羣
如果舊節點加入Galera集羣，說明這個節點在之前已經在Galera集羣中呆過，有一部分數據基礎，缺少的只是它離開集羣時的數據。這時加入集羣時，會採用IST(incremental snapshot transfer)傳輸機制，即使用增量傳輸。
但注意，這部分增量傳輸的數據源是Donor上緩存在GCache文件中的，這個文件有大小限制，如果缺失的數據範圍超過已緩存的內容，則自動轉爲SST傳輸。如果舊節點上的數據和Donor上的數據不匹配(例如這個節點離組後人爲修改了一點數據)，則自動轉爲SST傳輸。
11.在PXC集羣中再加一臺新的主機PXC4：192.168.7.74
11.1安裝
# 先在ansible主控機的hosts文件中添加192.168.7.74，然後執行以下操作
[root@pxc1 ~]#ansible '192.168.7.74' -m copy -a 'src=/etc/yum.repos.d/pxc.repo dest=/etc/yum.repos.d'
[root@pxc1 ~]#ansible '192.168.7.74' -a 'yum -y install Percona-XtraDB-Cluster-57'
[root@pxc1 ~]#ansible '192.168.7.74' -m copy -a 'src=/etc/percona-xtradb-cluster.conf.d/wsrep.cnf dest=/etc/percona-xtradb-cluster.conf.d'
[root@pxc1 ~]#ansible '192.168.7.74' -a 'sed -i -e s/wsrep_node_address=192.168.7.71/wsrep_node_address=192.168.7.74/ -e s/wsrep_node_name=pxc-cluster-node-1/wsrep_node_name=pxc-cluster-node-4/ /etc/percona-xtradb-cluster.conf.d/wsrep.cnf'
[root@pxc1 ~]#ansible 192.168.7.74 -a "sed -i 's/^wsrep_cluster_address.*/&,192.168.7.74/' /etc/percona-xtradb-cluster.conf.d/wsrep.cnf"
[root@pxc1 ~]#ansible 192.168.7.74 -a 'egrep -v "^#|^$" /etc/percona-xtradb-cluster.conf.d/wsrep.cnf'
192.168.7.74 | CHANGED | rc=0 >>
[mysqld]
wsrep_provider=/usr/lib64/galera3/libgalera_smm.so
wsrep_cluster_address=gcomm://192.168.7.71,192.168.7.72,192.168.7.73,192.168.7.74
binlog_format=ROW
default_storage_engine=InnoDB
wsrep_slave_threads= 8
wsrep_log_conflicts
innodb_autoinc_lock_mode=2
wsrep_node_address=192.168.7.74
wsrep_cluster_name=pxc-cluster
wsrep_node_name=pxc-cluster-node-4
pxc_strict_mode=ENFORCING
wsrep_sst_method=xtrabackup-v2
wsrep_sst_auth="sstuser:123.com"
11.2啓動
[root@pxc1 ~]#ansible 192.168.7.74 -a 'systemctl enable --now mysql'
11.3在新加入的節點中查看集羣狀態
[root@pxc4 ~]#mysql -p123.com
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 11
Server version: 5.7.27-30-57-log Percona XtraDB Cluster (GPL), Release rel30, Revision 64987d4, WSREP version 31.39, wsrep_31.39
Copyright (c) 2009-2019 Percona LLC and/or its affiliates
Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> show status like 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 4 |
+--------------------+-------+
1 row in set (0.01 sec)
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| db1 |
| db2 |
| mysql |
| performance_schema |
| sys |
+--------------------+
6 rows in set (0.00 sec)
修改集羣中其它節點的配置文件，將新加入的節點添加到配置文件中
[root@pxc1 ~]#ansible 'pxcservers:!192.168.7.74' -a "sed -i 's/^wsrep_cluster_address.*/&,192.168.7.74/' /etc/percona-xtradb-cluster.conf.d/wsrep.cnf"
在PXC集羣中修復故障節點
12.停止集羣中任意節點的mysql服務，這裏停止pxc2
[root@pxc1 ~]#ansible 192.168.7.72 -a 'systemctl stop mysql'
13.在集羣中的其它任意節點（這裏在pxc3）查看wsrep_cluster_size變量少了一個節點
[root@pxc3 ~]#mysql -p123.com
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 13
Server version: 5.7.27-30-57-log Percona XtraDB Cluster (GPL), Release rel30, Revision 64987d4, WSREP version 31.39, wsrep_31.39
Copyright (c) 2009-2019 Percona LLC and/or its affiliates
Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> show status like 'wrep_cluster_size';
Empty set (0.00 sec)
mysql> show status like 'wrep_cluster_size';
Empty set (0.00 sec)
mysql> show status like 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
1 row in set (0.00 sec)
14.在集羣中的其它任意節點創建db3數據庫（這裏在pxc4），然後查看其它節點是否同步數據
[root@pxc4 ~]#mysql -p123.com
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 12
Server version: 5.7.27-30-57-log Percona XtraDB Cluster (GPL), Release rel30, Revision 64987d4, WSREP version 31.39, wsrep_31.39
Copyright (c) 2009-2019 Percona LLC and/or its affiliates
Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> create database db3;
Query OK, 1 row affected (0.01 sec)
[root@pxc1 ~]#ansible pxcservers -a 'mysql -p123.com -e "show databases;\n"'
192.168.7.73 | CHANGED | rc=0 >>
Database
information_schema
db1
db2
db3
mysql
performance_schema
sys
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
192.168.7.72 | FAILED | rc=1 >>
mysql: [Warning] Using a password on the command line interface can be insecure.
ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/lib/mysql/mysql.sock' (2)non-zero return code
192.168.7.74 | CHANGED | rc=0 >>
Database
information_schema
db1
db2
db3
mysql
performance_schema
sys
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
192.168.7.71 | CHANGED | rc=0 >>
Database
information_schema
db1
db2
db3
mysql
performance_schema
sys
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
15.恢復pxc2的mysql服務，數據同步
[root@pxc1 ~]#ansible 192.168.7.72 -a 'systemctl start mysql'
192.168.7.72 | CHANGED | rc=0 >>
[root@pxc1 ~]#ansible 192.168.7.72 -a 'mysql -p123.com -e "show databases;\n"'
192.168.7.72 | CHANGED | rc=0 >>
Database
information_schema
db1
db2
db3
mysql
performance_schema
sys
PAGER set to stdoutmysql: [Warning] Using a password on the command line interface can be insecure.
