CentOS 7.9 用 ceph-deploy 部署 Ceph（Nautilus）集群

发表于 更新于 分类于 ， 本文字数： 9.5k 阅读时长 ≈ 9 分钟

概览：从三台裸 CentOS 7.9 到一套带 Dashboard 的 Ceph Nautilus 集群

基础配置 → 安装 ceph → MON / MGR / OSD → 高可用 → Dashboard

环境配置概览

节点 IP OS 配置(测试环境)
ceph01 192.168.150.26 centos 7 2C 4G 16G
ceph02 192.168.150.32 centos 7 2C 4G 16G
ceph03 192.168.150.34 centos 7 2C 4G 16G

基础配置

1，配置计算机名

1
2
3
hostnamectl set-hostname ceph01 # 节点ceph01执行
hostnamectl set-hostname ceph02 # 节点ceph02执行
hostnamectl set-hostname ceph03 # 节点ceph03执行

2，所有节点关闭防火墙selinux

1
2
3
4
5
6
systemctl stop firewalld
systemctl disable firewalld
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config ##  所有节点

3，配置hosts

1
2
3
4
5
6
7
8
vi /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.150.26 ceph01
192.168.150.32 ceph02
192.168.150.34 ceph03
scp /etc/hosts [email protected]:/etc/
scp /etc/hosts [email protected]:/etc/ ##同步给另外两台节点

4，配置yum源

1
vi /etc/yum.repos.d/ceph.repo
1
2
3
4
5
6
7
8
9
10
11
12
[ceph-noarch]
name=ceph-noarch
baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/
enabled=1
gpgcheck=0

[ceph-x86_64]
name=ceph-x86_64
baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/x86_64/
enabled=1
gpgcheck=0 
##所有节点
1
vim /etc/yum.repos.d/epel.repo
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
[epel]
name=Extra Packages for Enterprise Linux 7 - $basearch
baseurl=http://mirrors.aliyun.com/epel/7/$basearch
failovermethod=priority
enabled=1
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7

[epel-debuginfo]
name=Extra Packages for Enterprise Linux 7 - $basearch - Debug
baseurl=http://mirrors.aliyun.com/epel/7/$basearch/debug
failovermethod=priority
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
gpgcheck=0

[epel-source]
name=Extra Packages for Enterprise Linux 7 - $basearch - Source
baseurl=http://mirrors.aliyun.com/epel/7/SRPMS
failovermethod=priority
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
gpgcheck=0
##所有节点
1
vim /etc/yum.repos.d/CentOS-Base.repo
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
[base]
name=CentOS-$releasever - Base - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/
        http://mirrors.aliyuncs.com/centos/$releasever/os/$basearch/
        http://mirrors.cloud.aliyuncs.com/centos/$releasever/os/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

#released updates
[updates]
name=CentOS-$releasever - Updates - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/
        http://mirrors.aliyuncs.com/centos/$releasever/updates/$basearch/
        http://mirrors.cloud.aliyuncs.com/centos/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/
        http://mirrors.aliyuncs.com/centos/$releasever/extras/$basearch/
        http://mirrors.cloud.aliyuncs.com/centos/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/
        http://mirrors.aliyuncs.com/centos/$releasever/centosplus/$basearch/
        http://mirrors.cloud.aliyuncs.com/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

#contrib - packages by Centos Users
[contrib]
name=CentOS-$releasever - Contrib - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/contrib/$basearch/
        http://mirrors.aliyuncs.com/centos/$releasever/contrib/$basearch/
        http://mirrors.cloud.aliyuncs.com/centos/$releasever/contrib/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
##所有节点
1
yum makecache && yum repolist

image-20220512092547406

5，时间同步

1
2
3
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm #添加wlnmp的yum源
yum install wntp -y #安装wntp
ntpdate ntp1.aliyun.com #同步时间，所有节点

6，配置免密

1
2
3
4
5
6
ssh-keygen #一路回车
cd ~/.ssh
ssh-copy-id -i ~/.ssh/id_rsa.pub root@ceph02 #复制ssh pubkey到ceph02节点
ssh-copy-id -i ~/.ssh/id_rsa.pub root@ceph03 #复制ssh pubkey到ceph03节点
ssh root@ceph03 #不用输密码登录即为成功
reboot ##初始化配置完成后重启一下

安装ceph

1，安装ceph软件包

1
2
yum install -y python-setuptools
yum install -y ceph-deploy #所有节点

2，创建配置目录

1
2
mkdir /root/ceph-deploy
cd /root/ceph-deploy #主要节点（ceph01）执行，后续 ceph-deploy 命令都在这个目录跑

3，创建新节点

1
2
3
4
5
ceph-deploy new ceph01 --public-network 192.168.150.0/24
#参数设置
--public-network 公共网络（客户端/MON 通信走这个网段）
--cluster-network 集群网络（OSD 之间复制/心跳的内部网段，不指定则与 public 共用）
#--public-network建议添加，否则后面添加monitor节点会提示错误 #主要节点
1
2
3
4
ls /root/ceph-deploy
ceph.conf       ceph-deploy-ceph.log  ceph.mon.keyring
ceph配置文件      ceph日志文件       keyring主要做身份验证
#我们可以根据自身需要修改ceph.conf文件，比如上面创建集群中添加的网络，在这里也可以添加 #主要节点
1
2
3
#添加允许ceph时间偏移
echo "mon clock drift allowed = 2" >>/root/ceph-deploy/ceph.conf
echo "mon clock drift warn backoff = 30" >>/root/ceph-deploy/ceph.conf #主要节点

4，安装软件包

1
yum install -y ceph ceph-mon ceph-mgr ceph-radosgw ceph-mds #所有节点

5，初始化主节点

1
2
cd /root/ceph-deploy
ceph-deploy mon create-initial

image-20220512094118354

6，分配密钥到所有节点

1
ceph-deploy admin ceph01 ceph02 ceph03

image-20220512094321784

1
2
3
ceph config set mon auth_allow_insecure_global_id_reclaim false #禁用不安全模式
ceph-deploy --overwrite-conf config push ceph01 ceph02 ceph03 #修改配置后推送到对应节点
systemctl restart ceph-mon.target #修改后重启这个服务生效

7，初始化完毕查询

1
ceph -s

image-20220512094702429

配置ceph

1，创建MGR

1
ceph-deploy mgr create ceph01

image-20220512094845724

2，查询刚才创建的MGR

1
ceph -s

image-20220512095233311

3，创建OSD

1
2
3
4
#需要给虚拟机新增硬盘，我的虚拟机是KVM的，可以参考如下，VMware和实体机的自行了解，硬盘插上就行，暂时不需要分区
qemu-img create -f qcow2 virtiob.qcow2 10G #创建10G新硬盘到虚拟机目录下，所有节点
#创建后记得把盘挂载给虚拟机（virsh attach-disk 或管理面板加盘），VMware/实体机直接插盘即可
lsblk #查新硬盘

image-20220512100825703

1
2
3
4
cd /root/ceph-deploy/ #进入目录
ceph-deploy osd create ceph01 --data /dev/vdb #创建ceph01OSD
ceph-deploy osd create ceph02 --data /dev/vdb #创建ceph02OSD
ceph-deploy osd create ceph03 --data /dev/vdb #创建ceph03OSD

4，查询OSD

1
ceph -s

image-20220512101214647

1
ceph osd tree

image-20220512101250229

1
2
ceph-deploy --overwrite-conf config push ceph01 ceph02 ceph03
#如果期间我们有需要修改cpeh.conf的操作，只需要在ceph-01上修改，使用下面的命令同步到其他节点上

5，节点高可用

1
2
3
4
5
6
#扩展主节点（monitor）
#monitor负责保存OSD的元数据，所以monitor当然也需要高可用。 这里的monitor推荐使用奇数节点进行部署，我这里以3台节点部署，当我们添加上3个monitor节点后，#monitor会自动进行选举，自动进行高可用
#添加monitor
cd /root/ceph-deploy
ceph-deploy mon add ceph02 --address 192.168.150.32
ceph-deploy mon add ceph03 --address 192.168.150.34

6，查询节点高可用状态

1
ceph -s

image-20220512102639021

1
2
ceph quorum_status --format json-pretty 
# 查询节点选举状态
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
{
    "election_epoch": 12,
    "quorum": [
        0,
        1,
        2
    ],
    "quorum_names": [
        "ceph01",
        "ceph02",
        "ceph03"
    ],
    "quorum_leader_name": "ceph01", ##当前leader节点
    "quorum_age": 98,
    "monmap": {
        "epoch": 3, #数量
        "fsid": "cae384c6-2be0-4bee-8f76-2f02941d71ee",
        "modified": "2022-05-12 10:25:21.969392",
        "created": "2022-05-12 09:40:29.840989",
        "min_mon_release": 14,
        "min_mon_release_name": "nautilus",
        "features": {
            "persistent": [
                "kraken",
                "luminous",
                "mimic",
                "osdmap-prune",
                "nautilus"
            ],
            "optional": []
        },
        "mons": [
            {
                "rank": 0,
                "name": "ceph01",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.150.26:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.150.26:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.150.26:6789/0",
                "public_addr": "192.168.150.26:6789/0"
            },
            {
                "rank": 1,
                "name": "ceph02",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.150.32:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.150.32:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.150.32:6789/0",
                "public_addr": "192.168.150.32:6789/0"
            },
            {
                "rank": 2,
                "name": "ceph03",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.150.34:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.150.34:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.150.34:6789/0",
                "public_addr": "192.168.150.34:6789/0"
            }
        ]
    }
}
[root@ceph01 ceph-deploy]# ceph -s
  cluster:
    id:     cae384c6-2be0-4bee-8f76-2f02941d71ee
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum ceph01,ceph02,ceph03 (age 119s)
    mgr: ceph01(active, since 40m)
    osd: 3 osds: 3 up (since 16m), 3 in (since 16m)

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   3.0 GiB used, 27 GiB / 30 GiB avail
    pgs:

7，部署MGR高可用

1
2
3
#mgr的方法与monitor（节点）方法类似
cd /root/ceph-deploy
ceph-deploy mgr create ceph02 ceph03

8，查询MGR高可用状态

1
2
3
ceph -s
#这里已经看到mgr已经添加上去，只有ceph-01节点为active，其余节点均为standbys
#mgr集群只有一个节点为active状态，其它的节点都为standby。只有当主节点出现故障后，standby节点才会去接管，并且状态变更为active

image-20220512103120624

9，部署dashboard（仪表盘）

1
2
3
4
5
6
sudo yum -y install ceph-mgr-dashboard #安装
ceph mgr module enable dashboard --force #启动仪表盘
sudo ceph dashboard create-self-signed-cert #创建自签证书
echo admin > userpass #创建密码
sudo ceph dashboard set-login-credentials admin -i userpass #创建用户，引用刚才的密码
sudo ceph mgr services #查询仪表盘后台地址

image-20220512104028607

image-20220512104223978