規劃集羣
- 生產環境
- 至少3臺物理機組成Ceph集羣
- 雙網卡
- 測試環境
- 1臺主機也可以
- 單網卡也可以
準備工作
-
在所有Ceph節點上安裝NTP
[root@test~]# yum install ntp -
在所有Ceph節點上檢查Iptables規則，確定打開6789端口，及
6800:7300端口[root@test~]# iptables -A INPUT -i eth0 -p tcp -s10.10.8.0/24--dport6789-j ACCEPT[root@test~]# iptables -A INPUT -i eth0 -p tcp -s10.10.8.0/24--dport6800:7300-j ACCEPT -
關閉SELinux
[root@test~]# setenforce0
獲取Ceph包
-
添加源，在/etc/yum.repos.d/目錄下創建ceph.repo
[root@test~]# cd /etc/yum.repos.d/[root@test~]# touch ceph.repo將下面的內容複製到ceph.repo中，但是注意將ceph-release字段替換爲想要安裝的ceph版本，這裏安裝的是jewel版本；將distro替換爲自己系統的版本，當前用的是el7[ceph]name=Ceph packagesfor$basearchbaseurl=http://download.ceph.com/rpm-{ceph-release}/{distro}/$basearchenabled=1priority=2gpgcheck=1type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc[ceph-noarch]name=Ceph noarch packagesbaseurl=http://download.ceph.com/rpm-{ceph-release}/{distro}/noarchenabled=1priority=2gpgcheck=1type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc[ceph-source]name=Ceph source packagesbaseurl=http://download.ceph.com/rpm-{ceph-release}/{distro}/SRPMSenabled=0priority=2gpgcheck=1type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc[root@test~]# cat ceph.repo[ceph]name=Ceph packagesfor$basearchbaseurl=http://download.ceph.com/rpm-jewel/el7/$basearchenabled=1priority=2gpgcheck=1type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc[ceph-noarch]name=Ceph noarch packagesbaseurl=http://download.ceph.com/rpm-jewel/el7/noarchenabled=1priority=2gpgcheck=1type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc[ceph-source]name=Ceph source packagesbaseurl=http://download.ceph.com/rpm-jewel/el7/SRPMSenabled=0priority=2gpgcheck=1type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc -
安裝ceph包
[root@test~]# yum install ceph[root@test~]# rpm -qa | grep cephceph-mds-10.2.10-0.el7.x86_64ceph-10.2.10-0.el7.x86_64libcephfs1-10.2.10-0.el7.x86_64python-cephfs-10.2.10-0.el7.x86_64ceph-common-10.2.10-0.el7.x86_64ceph-base-10.2.10-0.el7.x86_64ceph-osd-10.2.10-0.el7.x86_64ceph-mon-10.2.10-0.el7.x86_64ceph-selinux-10.2.10-0.el7.x86_64
手動部署
配置Ceph
-
配置ceph.conf
[root@test~]# touch ceph.conf -
生成Ceph Cluster ID
[root@test~]# uuidgen1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b[root@test~]# echo"fsid = 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b">> /etc/ceph/ceph.conf[root@test~]# cat ceph.conf[global]fsid = 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b
部署Monitor
-
配置中添加初始Monitor hostname
[root@test~]# echo"mon_initial_members = test">> /etc/ceph/ceph.conf -
配置中添加初始Monitor IP地址
[root@test~]# echo"mon_host = 10.10.8.19">> /etc/ceph/ceph.conf -
生成Monitor的keyring
[root@test~]# ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon'allow *' -
生成client的keyring
[root@test~]# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0--cap mon'allow *'--cap osd'allow *'--cap mds'allow' -
將client keyring加入到monitor keyring
[root@test~]# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring -
生成monmap
[root@test~]# monmaptool --create --add test10.10.8.19--fsid 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b /tmp/monmap -
創建Monitor目錄
[root@test~]# mkdir /var/lib/ceph/mon/ceph-test -
創建Monitor
[root@test~]# ceph-mon --mkfs -i test --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring -
填充ceph.conf
將以下內容添加到ceph.conf中，注意替換相關字段publicnetwork = {network}[, {network}]cluster network = {network}[, {network}]auth cluster required = cephxauth service required = cephxauth client required = cephxosd journal size = {n}[root@test~]# cat /etc/ceph/ceph.conf[global]fsid = 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3bmon initial members = testmon_host =10.10.8.19public_network =10.10.8.0/24cluster_network =10.10.8.0/24osd_journal_size =2048 -
創建done文件
[root@test~]# touch /var/lib/ceph/mon/ceph-test/done -
修改權限
[root@test~]# chown ceph:ceph /var/lib/ceph/mon/ceph-test/ -R -
啓動Monitor服務
[root@test~]# systemctl start ceph-mon@test.service -
驗證
[root@test~]# ceph -scluster 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3bhealth HEALTH_ERRno osdsmonmap e1:1mons at {test=10.10.8.19:6789/0}election epoch3, quorum0testosdmap e1:0osds:0up,0inflags sortbitwise,require_jewel_osdspgmap v2:64pgs,1pools,0bytes data,0objects0kB used,0kB /0kB avail64creating
部署OSD
-
生成OSD的UUID
[root@test~]# uuidgen8ff77024-27dd-423b-9f66-0a7cefcd3f53 -
創建OSD
[root@test~]# ceph osd create 8ff77024-27dd-423b-9f66-0a7cefcd3f530 -
創建OSD目錄
[root@test~]# mkdir /var/lib/ceph/osd/ceph-0 -
格式化OSD磁盤
[root@test~]# mkfs -t xfs /dev/vdb -
掛載OSD磁盤
[root@test~]# mount /dev/vdb /var/lib/ceph/osd/ceph-0/ -
初始化OSD
[root@test~]# ceph-osd -i0--mkfs --mkkey --osd-uuid 8ff77024-27dd-423b-9f66-0a7cefcd3f53 -
註冊OSD keyring
[root@test~]# ceph auth add osd.0osd'allow *'mon'allow profile osd'-i /var/lib/ceph/osd/ceph-0/keyring -
添加host到CRUSH map
[root@test~]# ceph osd treeID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY-10rootdefault00osd.0down01.0000[root@test~]# ceph osd crush add-bucket test host -
移動host到CRUSH map中的root
[root@test~]# ceph osd crush move test root=default -
添加OSD到CRUSH下的host
[root@test~]# ceph osd crush add osd.01.0host=test -
修改權限
[root@test~]# chown ceph:ceph /var/lib/ceph/osd/ceph-0/ -R
-
啓動OSD服務
[root@test~]# systemctl start ceph-osd@0.service -
驗證
[root@test~]# ceph -scluster 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3bhealth HEALTH_OKmonmap e1:1mons at {test=10.10.8.19:6789/0}election epoch3, quorum0testosdmap e12:1osds:1up,1inflags sortbitwise,require_jewel_osdspgmap v16:64pgs,1pools,0bytes data,0objects2080MB used,18389MB /20470MB avail64active+clean[root@test~]# ceph osd treeID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY-11.00000rootdefault-21.00000host test01.00000osd.0up1.000001.00000