Le operazioni vanno eseguite sulla sulla macchina ceph-ctrl-00 ( con ssh root@mgmt-ceph-ctrl-00 ), o su un’altra macchina facente parte della controlplane.

cephadm bootstrap --cluster-network 192.168.32.0/24 --mon-ip 192.168.33.10 --ssh-user cephadm --allow-fqdn-hostname
ceph orch apply mon --unmanaged
ceph orch apply mgr --unmanaged
ssh-copy-id -f -i /etc/ceph/ceph.pub cephadm@ceph-data-01
ssh-copy-id -f -i /etc/ceph/ceph.pub cephadm@ceph-data-00
ssh-copy-id -f -i /etc/ceph/ceph.pub cephadm@ceph-data-02
ssh-copy-id -f -i /etc/ceph/ceph.pub cephadm@ceph-ctrl-02
ssh-copy-id -f -i /etc/ceph/ceph.pub cephadm@ceph-ctrl-01
ssh-copy-id -f -i /etc/ceph/ceph.pub cephadm@ceph-ctrl-00
ceph orch host add ceph-ctrl-01.lnf.infn.it 192.168.33.11 _admin mon mgr mds
ceph orch host add ceph-ctrl-02.lnf.infn.it 192.168.33.12 _admin mon mgr mds
ceph orch host add ceph-data-02.lnf.infn.it 192.168.33.22
ceph orch host add ceph-data-01.lnf.infn.it 192.168.33.21
ceph orch host add ceph-data-00.lnf.infn.it 192.168.33.20
ceph orch host label add ceph-ctrl-00.lnf.infn.it mon
ceph orch host label add ceph-ctrl-00.lnf.infn.it mgr
ceph orch host label add ceph-ctrl-00.lnf.infn.it mds
ceph orch apply mon --placement="label:mon"
ceph orch apply mgr --placement="label:mgr"
ceph orch apply mds benchmark --placement="2 label:mds"

ceph orch apply -i osd_service.yaml --dry-run

Verificare dopo alcuni secondi che la configurazione generata sia quella desiderata ripetendo il comando.

Per installare gli OSD utilizando gli SSD come db utilizzare osd_service.yaml con il seguente contenuto:

service_type: osd
service_id: hdd_plus_db_on_ssd
placement:
  hosts:
    - ceph-data-00.lnf.infn.it
    - ceph-data-01.lnf.infn.it
    - ceph-data-02.lnf.infn.it
spec:
  data_devices:
    paths:
      - /dev/sdb
      - /dev/sdc
      - /dev/sdd
      - /dev/sde
      - /dev/sdf
      - /dev/sdg
      - /dev/sdh
      - /dev/sdi
      - /dev/sdj
      - /dev/sdk
      - /dev/sdl
      - /dev/sdm
      - /dev/sdn
      - /dev/sdo
      - /dev/sdp
      - /dev/sdq
  db_devices:
    paths:
      - /dev/sdr
      - /dev/sds
      - /dev/sdt
      - /dev/sdu
      - /dev/sdv
      - /dev/sdw
      - /dev/sdx
      - /dev/sdy
  db_slots: 2

Per utilizzare gli SSD e HDD come OSD separati utilizzare invece:

service_type: osd
service_id: hdd_and_ssd_apart
placement:
  hosts:
    - ceph-data-00.lnf.infn.it
    - ceph-data-01.lnf.infn.it
    - ceph-data-02.lnf.infn.it
spec:
  data_devices:
    paths:
      - /dev/sdb
      - /dev/sdc
      - /dev/sdd
      - /dev/sde
      - /dev/sdf
      - /dev/sdg
      - /dev/sdh
      - /dev/sdi
      - /dev/sdj
      - /dev/sdk
      - /dev/sdl
      - /dev/sdm
      - /dev/sdn
      - /dev/sdo
      - /dev/sdp
      - /dev/sdq
      - /dev/sdr
      - /dev/sds
      - /dev/sdt
      - /dev/sdu
      - /dev/sdv
      - /dev/sdw
      - /dev/sdx
      - /dev/sdy

ceph orch apply -i osd_service.yaml

ceph fs volume create benchmark
ceph fs authorize benchmark client.benchmark / rw
ceph osd pool create rbd 128 128 replicated
rbd pool init rbd
ceph auth caps client.benchmark mds "allow rw fsname=benchmark" mon "allow r fsname=benchmark, profile rbd" osd "allow rw tag cephfs data=benchmark, profile rbd pool=rbd" mgr 'profile rbd pool=rbd'
ceph osd pool set cephfs.benchmark.data pg_num 128
ceph osd pool set cephfs.benchmark.data pgp_num 128
ceph osd pool set cephfs.benchmark.meta pg_num 128
ceph osd pool set cephfs.benchmark.meta pgp_num 128
ceph osd pool set rbd pg_num 128
ceph osd pool set rbd pgp_num 128

ceph config generate-minimal-conf  # -> /etc/ceph/ceph.conf
ceph auth get client.benchmark  # -> /etc/ceph/ceph.client.benchmark.keyring

Sulla macchina client:

mkdir /mnt/benchmark_rbd
mkdir /mnt/benchmark_cephfs
echo '/dev/rbd/rbd/benchmark /mnt/benchmark_rbd xfs     noauto          0 0' >> /etc/fstab
echo 'benchmark@.benchmark=/ /mnt/benchmark_cephfs  ceph    defaults    0 0' >> /etc/fstab
rbd create --id benchmark --size 550G benchmark
rbd --id benchmark map benchmark
dd if=/dev/urandom of=/dev/rbd0 bs=4M count=137000
mkfs.xfs /dev/rbd0
rbd --id benchmark unmap benchmark
echo 'benchmark    id=benchmark' >> /etc/ceph/rbdmap
systemctl start rbdmap.service

ceph osd crush rule create-replicated ssd-only default host ssd
ceph osd crush rule create-replicated hdd-only default host hdd
ceph osd pool set cephfs.benchmark.data crush_rule hdd-only # (oppure ssd-only)
ceph osd pool set cephfs.benchmark.meta crush_rule hdd-only # (oppure ssd-only)
ceph osd pool set rbd crush_rule hdd-only # (oppure ssd-only)