Ceph 手动部署

规划集群

  • 生产环境
    • 至少3台物理机组成Ceph集群
    • 双网卡
  • 测试环境
    • 1台主机也可以
    • 单网卡也可以

准备工作

  • 在所有Ceph节点上安装NTP

    [root @test ~]# yum install ntp
  • 在所有Ceph节点上检查Iptables规则,确定打开6789端口,及6800:7300端口

    [root @test ~]# iptables -A INPUT -i eth0 -p tcp -s 10.10 . 8.0 / 24 --dport 6789 -j ACCEPT
    [root @test ~]# iptables -A INPUT -i eth0 -p tcp -s 10.10 . 8.0 / 24 --dport 6800 : 7300 -j ACCEPT
  • 关闭SELinux

    [root @test ~]# setenforce 0

获取Ceph包

  • 添加源,在/etc/yum.repos.d/目录下创建ceph.repo

    [root @test ~]# cd /etc/yum.repos.d/
    [root @test ~]# touch ceph.repo
     
    将下面的内容复制到ceph.repo中,但是注意将ceph-release字段替换为想要安装的ceph版本,这里安装的是jewel版本;
    将distro替换为自己系统的版本,当前用的是el7
    [ceph]
    name=Ceph packages for $basearch
    baseurl=http: //download.ceph.com/rpm-{ceph-release}/{distro}/$basearch
    enabled= 1
    priority= 2
    gpgcheck= 1
    type=rpm-md
    gpgkey=https: //download.ceph.com/keys/release.asc
     
    [ceph-noarch]
    name=Ceph noarch packages
    baseurl=http: //download.ceph.com/rpm-{ceph-release}/{distro}/noarch
    enabled= 1
    priority= 2
    gpgcheck= 1
    type=rpm-md
    gpgkey=https: //download.ceph.com/keys/release.asc
     
    [ceph-source]
    name=Ceph source packages
    baseurl=http: //download.ceph.com/rpm-{ceph-release}/{distro}/SRPMS
    enabled= 0
    priority= 2
    gpgcheck= 1
    type=rpm-md
    gpgkey=https: //download.ceph.com/keys/release.asc
     
     
    [root @test ~]# cat ceph.repo
    [ceph]
    name=Ceph packages for $basearch
    baseurl=http: //download.ceph.com/rpm-jewel/el7/$basearch
    enabled= 1
    priority= 2
    gpgcheck= 1
    type=rpm-md
    gpgkey=https: //download.ceph.com/keys/release.asc
     
    [ceph-noarch]
    name=Ceph noarch packages
    baseurl=http: //download.ceph.com/rpm-jewel/el7/noarch
    enabled= 1
    priority= 2
    gpgcheck= 1
    type=rpm-md
    gpgkey=https: //download.ceph.com/keys/release.asc
     
    [ceph-source]
    name=Ceph source packages
    baseurl=http: //download.ceph.com/rpm-jewel/el7/SRPMS
    enabled= 0
    priority= 2
    gpgcheck= 1
    type=rpm-md
    gpgkey=https: //download.ceph.com/keys/release.asc
  • 安装ceph包

    [root @test ~]# yum install ceph
    [root @test ~]# rpm -qa | grep ceph
    ceph-mds- 10.2 . 10 - 0 .el7.x86_64
    ceph- 10.2 . 10 - 0 .el7.x86_64
    libcephfs1- 10.2 . 10 - 0 .el7.x86_64
    python-cephfs- 10.2 . 10 - 0 .el7.x86_64
    ceph-common- 10.2 . 10 - 0 .el7.x86_64
    ceph-base- 10.2 . 10 - 0 .el7.x86_64
    ceph-osd- 10.2 . 10 - 0 .el7.x86_64
    ceph-mon- 10.2 . 10 - 0 .el7.x86_64
    ceph-selinux- 10.2 . 10 - 0 .el7.x86_64

手动部署

配置Ceph

  •  配置ceph.conf

    [root @test ~]# touch ceph.conf
  • 生成Ceph Cluster ID

    [root @test ~]# uuidgen
    1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b
    [root @test ~]# echo "fsid = 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b" >> /etc/ceph/ceph.conf
    [root @test ~]# cat ceph.conf
    [global]
    fsid = 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b

部署Monitor

  • 配置中添加初始Monitor hostname

    [root @test ~]# echo "mon_initial_members = test" >> /etc/ceph/ceph.conf
  • 配置中添加初始Monitor IP地址

    [root @test ~]# echo "mon_host = 10.10.8.19" >> /etc/ceph/ceph.conf
  • 生成Monitor的keyring

    [root @test ~]# ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
  • 生成client的keyring

    [root @test ~]# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid= 0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'
  • 将client keyring加入到monitor keyring

    [root @test ~]# ceph-authtool /tmp/ceph.mon.keyring -- import -keyring /etc/ceph/ceph.client.admin.keyring
  • 生成monmap

    [root @test ~]# monmaptool --create --add test 10.10 . 8.19 --fsid 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b /tmp/monmap
  • 创建Monitor目录

    [root @test ~]# mkdir /var/lib/ceph/mon/ceph-test
  • 创建Monitor

    [root @test ~]# ceph-mon --mkfs -i test --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
  • 填充ceph.conf

    将以下内容添加到ceph.conf中,注意替换相关字段
    public network = {network}[, {network}]
    cluster network = {network}[, {network}]
    auth cluster required = cephx
    auth service required = cephx
    auth client required = cephx
    osd journal size = {n}
     
    [root @test ~]# cat /etc/ceph/ceph.conf
    [global]
    fsid = 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b
    mon initial members = test
    mon_host = 10.10 . 8.19
    public_network = 10.10 . 8.0 / 24
    cluster_network = 10.10 . 8.0 / 24
    osd_journal_size = 2048
  • 创建done文件

    [root @test ~]# touch /var/lib/ceph/mon/ceph-test/done
  • 修改权限

    [root @test ~]# chown ceph:ceph  /var/lib/ceph/mon/ceph-test/ -R
  • 启动Monitor服务

    [root @test ~]# systemctl start ceph-mon @test .service
  • 验证

    [root @test ~]# ceph -s
          cluster 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b
          health HEALTH_ERR
                 no osds
          monmap e1: 1 mons at {test= 10.10 . 8.19 : 6789 / 0 }
                 election epoch 3 , quorum 0 test
          osdmap e1: 0 osds: 0 up, 0 in
                 flags sortbitwise,require_jewel_osds
           pgmap v2: 64 pgs, 1 pools, 0 bytes data, 0 objects
                 0 kB used, 0 kB / 0 kB avail
                       64 creating

部署OSD

  • 生成OSD的UUID

    [root @test ~]# uuidgen
    8ff77024-27dd-423b-9f66-0a7cefcd3f53
  • 创建OSD

    [root @test ~]# ceph osd create 8ff77024-27dd-423b-9f66-0a7cefcd3f53
    0
  • 创建OSD目录

    [root @test ~]# mkdir /var/lib/ceph/osd/ceph- 0
  • 格式化OSD磁盘

    [root @test ~]# mkfs -t xfs /dev/vdb
  • 挂载OSD磁盘

    [root @test ~]# mount /dev/vdb /var/lib/ceph/osd/ceph- 0 /
  • 初始化OSD

    [root @test ~]# ceph-osd -i 0 --mkfs --mkkey --osd-uuid 8ff77024-27dd-423b-9f66-0a7cefcd3f53
  • 注册OSD keyring

    [root @test ~]# ceph auth add osd. 0 osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/ceph- 0 /keyring
  • 添加host到CRUSH map

    [root @test ~]# ceph osd tree
    ID WEIGHT TYPE NAME    UP/DOWN REWEIGHT PRIMARY-AFFINITY
    - 1      0 root default                                  
      0      0 osd. 0           down        0          1.0000
    [root @test ~]# ceph osd crush add-bucket test host
  • 移动host到CRUSH map中的root

    [root @test ~]# ceph osd crush move test root= default
  • 添加OSD到CRUSH下的host

    [root @test ~]# ceph osd crush add osd. 0 1.0 host=test
  • 修改权限

    [root @test ~]# chown ceph:ceph /var/lib/ceph/osd/ceph- 0 / -R
  • 启动OSD服务

    [root @test ~]# systemctl start ceph-osd @0 .service
  • 验证

    [root @test ~]# ceph -s
          cluster 1ea51317-3ccf-4a59-88ef-7d86f7ee1c3b
          health HEALTH_OK
          monmap e1: 1 mons at {test= 10.10 . 8.19 : 6789 / 0 }
                 election epoch 3 , quorum 0 test
          osdmap e12: 1 osds: 1 up, 1 in
                 flags sortbitwise,require_jewel_osds
           pgmap v16: 64 pgs, 1 pools, 0 bytes data, 0 objects
                 2080 MB used, 18389 MB / 20470 MB avail
                       64 active+clean
    [root @test ~]# ceph osd tree
    ID WEIGHT  TYPE NAME                UP/DOWN REWEIGHT PRIMARY-AFFINITY
    - 1 1.00000 root default                                                                                       
    - 2 1.00000     host test                                 
      0 1.00000         osd. 0                 up  1.00000          1.00000
相关文章
相关标签/搜索