文章目錄
一、環境準備
各節點IP分配如下表：
| 操作系統 | 主機名 | VM網段IP | NAT網段IP | 硬盤規劃 | 內存 | CPU |
|---|---|---|---|---|---|---|
| Centos 7 | ceph01 | 192.168.100.101 | 192.168.11.134 | 20G+1024G | 4G | 雙核雙線程 |
| Centos 7 | ceph02 | 192.168.100.102 | 192.168.11.135 | 20G+1024G | 4G | 雙核雙線程 |
| Centos 7 | ceph03 | 192.168.100.103 | 192.168.11.136 | 20G+1024G | 4G | 雙核雙線程 |
1、主機名
#三個節點分別配置主機名
[root@localhost ~]# hostnamectl set-hostname ceph01
[root@localhost ~]# hostnamectl set-hostname ceph02
[root@localhost ~]# hostnamectl set-hostname ceph03
2、關閉防火牆、核心防護
#三個節點均需要操作，以ceph01爲例
[root@ceph01 ~]# systemctl stop firewalld
[root@ceph01 ~]# systemctl disable firewalld
[root@ceph01 ~]# setenforce 0
[root@ceph01 ~]# sed -i '7s/enforcing/disabled/' /etc/selinux/config
3、配置hosts
#三個節點均需要操作，以ceph01爲例
[root@ceph01 ~]# vim /etc/hosts
192.168.100.101 ceph01
192.168.100.102 ceph02
192.168.100.103 ceph03
4、創建免交互
#三個節點均需要操作，以ceph01爲例
[root@ceph01 ~]# ssh-keygen
[root@ceph01 ~]# ssh-copy-id root@ceph01
[root@ceph01 ~]# ssh-copy-id root@ceph02
[root@ceph01 ~]# ssh-copy-id root@ceph03
#免交互登錄測試
5、配置YUM源
#三個節點均需要操作，以ceph01爲例
[root@ceph01 ~]# vim /etc/yum.conf
keepcache=1 //開啓緩存 三個節點都改
[root@ceph01 ~]# yum -y install wget curl net-tools bash-completion //安裝wget curl net-tools bash-completion
[root@ceph01 ~]# cd /etc/yum.repos.d/
[root@ceph01 yum.repos.d]# mkdir backup
[root@ceph01 yum.repos.d]# mv C* backup
[root@ceph01 yum.repos.d]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
[root@ceph01 yum.repos.d]# wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
[root@ceph01 yum.repos.d]# cat << EOF > /etc/yum.repos.d/ceph.repo
[ceph]
name=Ceph packages for
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarch/
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[ceph-source]
name=Ceph source packages
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/SRPMS/
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
EOF
[root@ceph03 yum.repos.d]# cat ceph.repo
[root@ceph03 yum.repos.d]# yum update -y //升級yum源
6、配置NTP時鐘服務
① 配置ceph01時間服務器
[root@ceph01 ~]# yum -y install ntpdate ntp
[root@ceph01 ~]# ntpdate ntp1.aliyun.com //同步阿里雲
[root@ceph01 ~]# clock -w //clock -w把當前系統時間寫入到CMOS中
[root@ceph01 ~]# vim /etc/ntp.conf //清空加入以下內容
driftfile /var/lib/ntp/drift
restrict defaultnomodify
restrict 127.0.0.1
restrict ::1
restrict 192.168.100.0 mask 255.255.255.0 nomodify notrap fudge 127.127.1.0 stratum 10
server 127.127.1.0
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
[root@ceph01 ~]# systemctl start ntpd
[root@ceph01 ~]# systemctl enable ntpd
② ceph02同步客戶端ceph01
[root@ceph02 ~]# yum -y install ntpdate
[root@ceph02 ~]# ntpdate ceph01
[root@ceph02 ~]# crontab -e
*/2 * * * * /usr/bin/ntpdate ceph01 >> /var/log/ntpdate.log
[root@ceph02 ~]# systemctl restart crond
[root@ceph02 ~]# crontab -l
③ ceph03同步客戶端ceph01
[root@ceph03 ~]# yum -y install ntpdate
[root@ceph03 ~]# ntpdate ceph01
[root@ceph03 ~]# crontab -e
*/2 * * * * /usr/bin/ntpdate ceph01 >> /var/log/ntpdate.log
[root@ceph03 ~]# systemctl restart crond
[root@ceph03 ~]# crontab -l
二、CEPH集羣搭建
使用線網源搭建
#登錄ceph01 ##
##mkdir /etc/ceph
##yum -y install http://download.ceph.com/rpm-mimic/el7/noarch/ceph-deploy-2.0.1-0.noarch.rpm
##yum -y install python-setuptools
##cd /etc/ceph
##ceph-deploy install ceph01 ceph02 ceph03
##scp ceph.repo epel.repo root@ceph02:/etc/yum.repos.d##
##scp ceph.repo epel.repo root@ceph03:/etc/yum.repos.d##
##yum -y install ceph ceph-redosgw三臺主機###
1、三個節點均創建ceph目錄並安裝ceph
[root@ceph01 ~]# mkdir /etc/ceph
[root@ceph01 ~]# yum -y install ceph
2、在ceph01安裝批量安裝工具
[root@ceph01 ~]# yum -y install python-setuptools
[root@ceph01 ~]# yum -y install ceph-deploy
3、ceph01創建mon
[root@ceph01 ~]# cd /etc/ceph
[root@ceph01 ~]# ceph-deploy new ceph01 ceph02
4、在ceph01初始化mon並收取祕鑰
[root@ceph01 ~]# cd /etc/ceph
[root@ceph01 ~]# ceph-deploy mon create-initial
[root@ceph01 ~]# ceph -s ##查看狀態
5、在ceph01創建osd
[root@ceph01 ~]# ceph-deploy osd create --data /dev/sdb ceph01
[root@ceph01 ~]# ceph-deploy osd create --data /dev/sdb ceph02
[root@ceph01 ~]# ceph -s //查看狀態有2個osd
#查看osd狀態信息
[root@ceph01 ~]# ceph osd tree
[root@ceph01 ~]# ceph osd stat
6、在ceph01將配置文件和admin祕鑰下發到ceph01 ceph02
[root@ceph01 ~]# ceph-deploy admin ceph01 ceph02
7、分別在ceph01和ceph02中給祕鑰增加讀的權限
[root@ceph01 ~]# chmod +x /etc/ceph/ceph.client.admin.keyring
[root@ceph02 ~]# chmod +x /etc/ceph/ceph.client.admin.keyring
[root@ceph01 ~]# ceph -s
到目前爲止集羣就搭建好了
三、CEPH擴容
1、登錄ceph01將ceph03 osd加入到集羣中
[root@ceph01 ~]# ceph-deploy osd create --data /dev/sdb ceph03
[root@ceph01 ~]# ceph -s
2、登錄ceph01將ceph03 mon加入到集羣中
[root@ceph01 ~]# ceph-deploy mon add ceph03
[root@ceph01 ~]# ceph -s
[root@ceph01 ~]# vim /etc/ceph/ceph.conf
mon_initial members = ceph01,ceph02,ceph03 //新增ceph03
mon_host = 192.168.100.101,192.168.100.102,192.168.100.103 //新增192.168.100.103
#將配置文件下發給ceph01 ceph02 ceph03
[root@ceph01 ~]# cd /etc/ceph/
[root@ceph01 ceph]# ceph-deploy --overwrite-conf config push ceph01 ceph02 ceph03
#三個節點重啓mon服務
[root@ceph01 ~]# systemctl restart ceph-mon.target
#如果不知道重啓mon服務，可以通過如下命令查看
systemctl list-unit-files l grep mon
四、osd數據恢復
1、模擬故障
#登錄ceph03，先把信息拷貝下來
[root@ceph01 ~]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 2.99696 root default
-3 0.99899 host ceph01
0 hdd 0.99899 osd.0 up 1.00000 1.00000
-5 0.99899 host ceph02
1 hdd 0.99899 osd.1 up 1.00000 1.00000
-7 0.99898 host ceph03
2 hdd 0.99898 osd.2 up 1.00000 1.00000
#移除osd.2
[root@ceph01 ceph]# ceph osd out osd.2
[root@ceph01 ceph]# ceph osd tree
#刪除osd.2
[root@ceph01 ceph]# ceph osd crush remove osd.2
#刪除osd.2的認證
[root@ceph01 ceph]# ceph auth del osd.2
[root@ceph01 ceph]# ceph osd tree
#徹底刪除osd.2
[root@ceph01 ceph]# ceph osd rm osd.2
#ceph03重啓osd服務
[root@ceph03 ~]# systemctl restart ceph-osd.target
[root@ceph03 ~]# ceph osd tree
2、恢復osd到集羣中
[root@ceph03 ~]# df -hT //查看ceph信息
文件系統 類型 容量 已用 可用 已用% 掛載點
devtmpfs devtmpfs 1.9G 0 1.9G 0% /dev
tmpfs tmpfs 1.9G 0 1.9G 0% /dev/shm
tmpfs tmpfs 1.9G 22M 1.9G 2% /run
tmpfs tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
/dev/sda3 xfs 17G 4.9G 13G 29% /
/dev/sda1 xfs 1014M 217M 798M 22% /boot
tmpfs tmpfs 378M 4.0K 378M 1% /run/user/988
/dev/sr0 iso9660 4.4G 4.4G 0 100% /run/media/ml/CentOS 7 x86_64
tmpfs tmpfs 378M 4.0K 378M 1% /run/user/42
tmpfs tmpfs 378M 60K 378M 1% /run/user/0
tmpfs tmpfs 1.9G 52K 1.9G 1% /var/lib/ceph/osd/ceph-2
[root@ceph03 ~]# cd /var/lib/ceph/osd/ceph-2
[root@ceph03 ceph-2]# more fsid //查看fsid
daf9b00a-7562-475e-8e6d-96455c808402
[root@ceph03 ceph-2]# ceph osd create daf9b00a-7562-475e-8e6d-96455c808402 //ceph osd create uuid
[root@ceph03 ceph-2]# ceph auth add osd.2 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-2/keyring //增加權限
[root@ceph03 ceph-2]# ceph osd crush add 2 0.99899 host=ceph03 //0.99899是權重，host=主機名稱
[root@ceph03 ceph-2]# ceph osd in osd.2
[root@ceph03 ceph-2]# systemctl restart ceph-osd.target
[root@ceph03 ceph-2]# ceph osd tree
恢復完成！
五、ceph常規的維護命令
1、創建mgr服務
[root@ceph01 ceph]# ceph-deploy mgr create ceph01 ceph02 ceph03
2、創建pool
[root@ceph01 ceph]# ceph osd pool create cinder 64
[root@ceph01 ceph]# ceph osd pool create nova 64
[root@ceph01 ceph]# ceph osd pool create glance 64
[root@ceph01 ceph]# ceph osd pool ls
3、刪除pool
[root@ceph01 ceph]# ceph osd pool rm cinder cinder --yes-i-really-really-mean-it //提示設置權限
[root@ceph01 ceph]# vim /etc/ceph/ceph.conf
mon_allow_pool_delete= true //新增刪除權限
[root@ceph01 ceph]# ceph-deploy --overwrite-conf admin ceph02 ceph03 //將文件下發ceph02 ceph03
[root@ceph01 ceph]# systemctl restart ceph-mon.target //三個節點重啓
[root@ceph01 ceph]# ceph osd pool rm cinder cinder --yes-i-really-really-mean-it
pool 'cinder' removed //提示刪除
[root@ceph01 ceph]# ceph osd pool ls
4、修改pool名字
[root@ceph01 ceph]# ceph osd pool rename nova nova1 //將Nova改成nova1
[root@ceph01 ceph]# ceph osd pool ls
5、查看ceph命令
[root@ceph01 ceph]# ceph --help
[root@ceph01 ceph]# ceph osd --help
6、配置ceph內部通信網段
[root@ceph01 ceph]# vim /etc/ceph/ceph.conf
public network= 192.168.100.0/24 //添加網段
[root@ceph01 ceph]# ceph-deploy --overwrite-conf admin ceph01 ceph02 ceph03 //將文件下發ceph02 ceph03
[root@ceph01 ceph]# systemctl restart ceph-mon.target //將各個節點的mon服務重啓下
[root@ceph01 ceph]# systemctl restart ceph-osd.target //將各個節點的osd服務重啓下
六、製作centos_ceph離線包
#將三個節點中的/var/cache/yum 所有packages文件整合到一起
#上傳至/opt/ceph_packages/
#安裝工具
[root@ceph01 opt]# yum -y install createrepo
[root@ceph01 opt]# cd ceph_packages
[root@ceph01 ceph_packages]# createrepo ./ //當前環境下生成新的依賴性關係
