From 3d5457c45e96d65ac5cf3316ad259b61e47b4798 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 11 Dec 2024 12:51:32 +0100 Subject: [PATCH] Add ceph (#226) * Add ceph class and packages Co-authored-by: Mikael Frykholm --- manifests/ceph.pp | 159 ++++++++++++++++++++++++ manifests/packages/ceph_common.pp | 4 + manifests/packages/ceph_mds.pp | 4 + manifests/packages/ceph_mon.pp | 4 + manifests/packages/ceph_osd.pp | 4 + manifests/packages/cephadm.pp | 4 + templates/ceph/bootstrap.erb.sh | 9 ++ templates/ceph/ceph-cluster.erb.yaml | 12 ++ templates/ceph/ceph-mgr.yaml | 4 + templates/ceph/cluster-bootstrap.erb.sh | 27 ++++ 10 files changed, 231 insertions(+) create mode 100644 manifests/ceph.pp create mode 100644 manifests/packages/ceph_common.pp create mode 100644 manifests/packages/ceph_mds.pp create mode 100644 manifests/packages/ceph_mon.pp create mode 100644 manifests/packages/ceph_osd.pp create mode 100644 manifests/packages/cephadm.pp create mode 100644 templates/ceph/bootstrap.erb.sh create mode 100644 templates/ceph/ceph-cluster.erb.yaml create mode 100644 templates/ceph/ceph-mgr.yaml create mode 100644 templates/ceph/cluster-bootstrap.erb.sh diff --git a/manifests/ceph.pp b/manifests/ceph.pp new file mode 100644 index 00000000..a23cf411 --- /dev/null +++ b/manifests/ceph.pp @@ -0,0 +1,159 @@ +# Ceph for SUNET +class sunet::ceph( + Array $adm, + Array $clients, + String $type, + String $firstmon, +) +{ + $adm_public_key = lookup('adm_public_key', undef, undef, 'NOT_SET_IN_HIERA'); + $packages = ['lvm2', 'podman'] + $packages.each |$package| { + package { $package: + ensure => 'present', + } + } + file {'/root/.ssh/': + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0700', + } + file {'/root/.ssh/authorized_keys': + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0600', + } + if $adm_public_key != 'NOT_SET_IN_HIERA' { + file_line { 'adm_public_key': + path => '/root/.ssh/authorized_keys', + line => $adm_public_key, + } + } + $nodes = lookup('nodes', undef, undef, []); + if $type == 'adm' { + $extra_ports = [] + include sunet::packages::cephadm + file {'/opt/ceph': + ensure => 'directory', + } + $adm_private_key = lookup('adm_private_key', undef, undef, 'NOT_SET_IN_HIERA'); + $adm_keyring = lookup('adm_keyring', undef, undef, 'NOT_SET_IN_HIERA'); + if $adm_keyring != 'NOT_SET_IN_HIERA' { + file {'/etc/ceph/ceph.client.admin.keyring': + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0600', + content => $adm_keyring, + } + } + if $adm_private_key != 'NOT_SET_IN_HIERA' { + file {'/root/.ssh/id_ed25519_adm': + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0600', + content => $adm_private_key, + } + } + if $adm_public_key != 'NOT_SET_IN_HIERA' { + file {'/root/.ssh/id_ed25519_adm.pub': + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0600', + content => $adm_public_key, + } + } + file {'/opt/ceph/ceph-cluster.yaml': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0600', + content => template('sunet/ceph/ceph-cluster.erb.yaml'), + } + file {'/opt/ceph/cluster-bootstrap.sh': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0700', + content => template('sunet/ceph/cluster-bootstrap.erb.sh'), + } + } + elsif $type == 'osd' { + $extra_ports = [] + } + elsif $type == 'mds' { + $extra_ports = [] + } + elsif $type == 'firstmon' { + include sunet::packages::cephadm + $adm_private_key = lookup('adm_private_key', undef, undef, 'NOT_SET_IN_HIERA'); + if $adm_private_key != 'NOT_SET_IN_HIERA' { + file {'/root/.ssh/id_ed25519_adm': + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0600', + content => $adm_private_key, + } + } + if $adm_public_key != 'NOT_SET_IN_HIERA' { + file {'/root/.ssh/id_ed25519_adm.pub': + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0600', + content => $adm_public_key, + } + } + $extra_ports = [ { 'from' => $clients, 'to' => '3300' } ] + file {'/opt/ceph': + ensure => 'directory', + } + file {'/opt/ceph/bootstrap.sh': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0700', + content => template('sunet/ceph/bootstrap.erb.sh'), + } + file {'/etc/alloy/targets.d/ceph-mgr.yaml': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0744', + content => template('sunet/ceph/ceph-mgr.yaml'), + } + } + elsif $type == 'mon' { + $extra_ports = [ { 'from' => $clients, 'to' => '3300' } ] + file {'/opt/ceph': + ensure => 'directory', + } + sunet::nftables::allow { 'expose-allow-ssh': + from => $adm, + port => 22, + } + file {'/etc/alloy/targets.d/ceph-mgr.yaml': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0744', + content => template('sunet/ceph/ceph-mgr.yaml'), + } + } + $internal_nodes = $nodes.map |$node| { + $node['addr'] + } + $internal_ports = [ { 'from' => $internal_nodes, 'to' => ['22', '3300', '6800-7300'] } ] + $ceph_ports = $extra_ports + $internal_ports + $ceph_ports.each |$port| { + sunet::nftables::allow { "expose-allow-${port['to']}": + from => $port['from'], + port => $port['to'], + } + } +} diff --git a/manifests/packages/ceph_common.pp b/manifests/packages/ceph_common.pp new file mode 100644 index 00000000..59c9848c --- /dev/null +++ b/manifests/packages/ceph_common.pp @@ -0,0 +1,4 @@ +# ceph_common +class sunet::packages::ceph_common { + package { 'ceph-common': ensure => installed } +} diff --git a/manifests/packages/ceph_mds.pp b/manifests/packages/ceph_mds.pp new file mode 100644 index 00000000..83c79f1d --- /dev/null +++ b/manifests/packages/ceph_mds.pp @@ -0,0 +1,4 @@ +# ceph_mds +class sunet::packages::ceph_mds { + package { 'ceph-mds': ensure => installed } +} diff --git a/manifests/packages/ceph_mon.pp b/manifests/packages/ceph_mon.pp new file mode 100644 index 00000000..ac0279d1 --- /dev/null +++ b/manifests/packages/ceph_mon.pp @@ -0,0 +1,4 @@ +# ceph_mon +class sunet::packages::ceph_mon { + package { 'ceph-mon': ensure => installed } +} diff --git a/manifests/packages/ceph_osd.pp b/manifests/packages/ceph_osd.pp new file mode 100644 index 00000000..d52c7af1 --- /dev/null +++ b/manifests/packages/ceph_osd.pp @@ -0,0 +1,4 @@ +# ceph_osd +class sunet::packages::ceph_osd { + package { 'ceph-osd': ensure => installed } +} diff --git a/manifests/packages/cephadm.pp b/manifests/packages/cephadm.pp new file mode 100644 index 00000000..7ac67737 --- /dev/null +++ b/manifests/packages/cephadm.pp @@ -0,0 +1,4 @@ +# cephadm +class sunet::packages::cephadm { + package { 'cephadm': ensure => installed } +} diff --git a/templates/ceph/bootstrap.erb.sh b/templates/ceph/bootstrap.erb.sh new file mode 100644 index 00000000..dfdd7b39 --- /dev/null +++ b/templates/ceph/bootstrap.erb.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +cephadm bootstrap \ + --mon-ip "<%= @facts['networking']['ip'] %>" \ + --ssh-user root \ + --ssh-private-key /root/.ssh/id_ed25519_adm \ + --ssh-public-key /root/.ssh/id_ed25519_adm.pub \ + --allow-fqdn-hostname \ + --allow-overwrite diff --git a/templates/ceph/ceph-cluster.erb.yaml b/templates/ceph/ceph-cluster.erb.yaml new file mode 100644 index 00000000..0aa5116f --- /dev/null +++ b/templates/ceph/ceph-cluster.erb.yaml @@ -0,0 +1,12 @@ +<% @nodes.each do |node| %> +--- +service_type: <%= node['service_type'] %> +addr: <%= node['addr'] %> +hostname: <%= node['hostname'] %> +<% if node['labels'] %> +labels: +<% node['labels'].each do |label| %> +- <%= label -%> +<% end -%> +<% end -%> +<% end -%> diff --git a/templates/ceph/ceph-mgr.yaml b/templates/ceph/ceph-mgr.yaml new file mode 100644 index 00000000..ab476412 --- /dev/null +++ b/templates/ceph/ceph-mgr.yaml @@ -0,0 +1,4 @@ +- targets: + - 127.0.0.1:9283 + labels: + job: ceph diff --git a/templates/ceph/cluster-bootstrap.erb.sh b/templates/ceph/cluster-bootstrap.erb.sh new file mode 100644 index 00000000..a86dfbeb --- /dev/null +++ b/templates/ceph/cluster-bootstrap.erb.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +ceph="/usr/sbin/cephadm shell ceph" + +adm_private_key="$(cat /root/.ssh/id_ed25519_adm)" +adm_public_key="$(ssh-keygen -y -f /root/.ssh/id_ed25519_adm)" +echo "$adm_public_key" > /root/.ssh/id_ed25519_adm.pub + +ssh -4 -i /root/.ssh/id_ed25519_adm "<%= @firstmon %>" /opt/ceph/bootstrap.sh # Run bootstrap script on first monitor +scp -4 -i /root/.ssh/id_ed25519_adm "<%= @firstmon %>:/etc/ceph/*" /etc/ceph/ # Copy over config <% monitors = [] %><% osd = [] %><% @nodes.each do |node| %><% hostname = node['hostname'] %> +${ceph} orch host add "<%= hostname %>" "<%= node['addr'] %>" # Add <%= node['hostname'] %><% node['labels'].each do |label| %><% if label == 'mon' %><% monitors.append(node['hostname']) %><% elsif label == 'osd' %><% osd.append(node['hostname']) %><% end %> +${ceph} orch host label add "<%= hostname %>" "<%= label %>" # <% end %><% end %> +${ceph} orch apply -i /rootfs/opt/ceph/nordunet-cephcluster.yaml + +adm_keyring="$(cat /etc/ceph/ceph.client.admin.keyring)" +echo "Now run:" +echo -e "\t ./edit-secrets $(hostname -f)" +echo "and add:" +echo "adm_private_key: >" +echo " DEC::PKCS7[$adm_private_key" +echo "]!" +echo "adm_keyring: >" +echo " DEC::PKCS7[$adm_keyring" +echo "]!" +echo -e "\n\n\nFinaly add:" +echo "adm_public_key: '$adm_public_key'" +echo "to the common group.yaml file"