Cephadm 容器化部署 Ceph 集群（Ubuntu 18.04）

发表于 更新于 分类于 ， 本文字数： 3.4k 阅读时长 ≈ 3 分钟

概览

  1. 节点配置

  2. 初始化集群

  3. 集群维护

配置概览

节点 IP OS 版本 配置(测试环境)
ceph01 192.168.3.164 Ubuntu 18.04 16.2.9-1bionic 2C 4G 16G(系统) 100G(OSD)
ceph02 192.168.3.167 Ubuntu 18.04 16.2.9-1bionic 2C 4G 16G(系统) 100G(OSD)
ceph03 192.168.3.168 Ubuntu 18.04 16.2.9-1bionic 2C 4G 16G(系统) 100G(OSD)

基础配置

1，配置计算机名

1
2
3
hostnamectl set-hostname ceph01 # 节点ceph01执行
hostnamectl set-hostname ceph02 # 节点ceph02执行
hostnamectl set-hostname ceph03 # 节点ceph03执行

2，所有节点关闭防火墙

1
2
3
4
5
systemctl stop ufw
systemctl disable ufw
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT
#以上所有节点执行；Ubuntu 没有 SELinux，无需再 setenforce

3，配置hosts

1
2
3
4
5
6
7
8
9
10
11
12
vim /etc/hosts
127.0.0.1 localhost
#127.0.1.1 ubantu18-4
192.168.3.164 ceph01
192.168.3.167 ceph02
192.168.3.168 ceph03
# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters #所有 节点

4，配置免密

1
2
3
4
5
ssh-keygen #一路回车
cd ~/.ssh
ssh-copy-id -i ~/.ssh/id_rsa.pub root@ceph02 #复制ssh pubkey到ceph02节点
ssh-copy-id -i ~/.ssh/id_rsa.pub root@ceph03 #复制ssh pubkey到ceph03节点
ssh root@ceph03 #不用输密码登录即为成功

5，配置apt源

1
2
3
cp /etc/apt/sources.list /etc/apt/sources.list.back #备份
>/etc/apt/sources.list #清空
vim /etc/apt/sources.list #输入下面内容
1
2
3
4
5
6
7
8
9
10
11
12
13
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic main
deb https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-octopus/ buster main
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-octopus/ buster main
1
2
3
4
5
scp /etc/apt/sources.list ceph02:/etc/apt/ #文件拷贝到对应节点
scp /etc/apt/sources.list ceph03:/etc/apt/ #文件拷贝到对应节点
apt-get update #报错NO_PUBKEY E84AC2C0460F3994 ，安装对应的key
sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com E84AC2C0460F3994 #安装完再执行
apt-cache madison ceph #检查ceph版本，要三个节点一致

image-20220526162911308

6，时间同步

1
2
3
apt install ntpdate -y #安装ntpdate
timedatectl set-timezone Asia/Shanghai #更改时区
ntpdate ntp1.aliyun.com #同步aliyun时间

初始化集群

1，安装软件

1
2
apt install -y cephadm # 安装cephadm
dpkg -l |grep ceph # 检查版本

2，初始化集群

1
2
3
4
5
6
7
8
9
10
11
12
13
cephadm bootstrap --mon-ip 192.168.3.164 # 主要节点
cephadm shell # 进入 cephadmshell
# 设置免密
ceph cephadm get-ssh-config > ssh_config
ceph config-key get mgr/cephadm/ssh_identity_key > ~/cephadm_private_key
chmod 0600 ~/cephadm_private_key
# 添加节点
ceph cephadm get-pub-key > ~/ceph.pub #获得密钥
ssh-copy-id -f -i ~/ceph.pub root@ceph02 # 复制密钥到对应节点
ssh-copy-id -f -i ~/ceph.pub root@ceph03 # 复制密钥到对应节点
ceph orch host add ceph02 #添加节点
ceph orch host add ceph03 #添加节点
ceph orch host ls #查看所有节点

image-20220528192445573

集群维护

1
2
3
4
5
6
7
ceph orch apply mon 3 # 设置mon节点数量为3个，根据节点数量自行评估
ceph config set global mon_max_pg_per_osd 2000 # 调大每个OSD允许承载的最大PG数（默认250）
ceph daemon osd.0 config show | grep pg # 查看pg相关设置
ceph config set mon mon_allow_pool_delete true #开启删除
ceph osd pool rm test test --yes-i-really-really-mean-it # 删除pool
ceph osd pool ls detail # 查看pools
#其余维护等操作，在web上操作就可以了