(CAT-2095): Fixed puppetlabs-kubernetes modules CI & nightly failures

1. replaced CENTOS-7 with RHEL-8
2. Updated test as per new changes

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix: {'platform':['centos-7'],'collection':['puppet7-nightly', 'puppet8-nightly']}
+      matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']}
 
     steps:
     - name: Checkout Source

.github/workflows/nightly.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix: {'platform':['centos-7'],'collection':['puppet7-nightly', 'puppet8-nightly']}
+      matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']}
 
     steps:
     - name: Checkout Source

Gemfile

@@ -76,3 +76,8 @@ extra_gemfiles.each do |gemfile|
   end
 end
 # vim: syntax=ruby
+
+
+# Fixed version for puppet-modulebuilder gem, as newer version of this gem does not include tooling folder.
+# We will keep this until we find a solution to either move the tooling folder in to some other folder or get rid of it altogether.
+gem 'puppet-modulebuilder', '1.1.0'

manifests/config/kubeadm.pp

@@ -11,7 +11,8 @@
 #   When set to true, etcd will be downloaded from the specified source URL.
 #   Defaults to true.
 # @param delegated_pki
-#   Set to true if all required X509 certificates will be provided by external means. Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files.
+#   Set to true if all required X509 certificates will be provided by external means. 
+#   Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files.
 #   Defaults to false
 # @param etcd_install_method
 #   The method on how to install etcd. Can be either wget (using etcd_source) or package (using $etcd_package_name)
@@ -96,7 +97,8 @@
 #   "periodic" or "revision"
 #   Defaults to "periodic"
 # @param etcd_compaction_retention
-#   This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`. An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision"
+#   This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`.
+#   An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision"
 #   Integer or String
 #   Defaults to 0 (disabled)
 # @param api_server_count
@@ -157,8 +159,9 @@
 #   A string array of extra arguments to be passed to scheduler.
 #   Defaults to []
 # @param kubelet_extra_arguments
-#   A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied to both control planes and nodes.
-#   Use this for critical Kubelet settings such as `pod-infra-container-image` which may be problematic to configure via kubelet_extra_config
+#   A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied
+#   to both control planes and nodes. Use this for critical Kubelet settings such as `pod-infra-container-image` 
+#   which may be problematic to configure via kubelet_extra_config
 #   Defaults to []
 # @param service_cidr
 #   The IP assdress range for service VIPs
@@ -381,9 +384,9 @@
   }
 
   # to_yaml emits a complete YAML document, so we must remove the leading '---'
-  $kubeadm_extra_config_yaml = regsubst(to_yaml($kubeadm_extra_config), '^---\n', '')
-  $kubelet_extra_config_yaml = regsubst(to_yaml($kubelet_extra_config), '^---\n', '')
-  $kubelet_extra_config_alpha1_yaml = regsubst(to_yaml($kubelet_extra_config_alpha1), '^---\n', '')
+  $kubeadm_extra_config_yaml = regsubst(stdlib::to_yaml($kubeadm_extra_config), '^---\n', '')
+  $kubelet_extra_config_yaml = regsubst(stdlib::to_yaml($kubelet_extra_config), '^---\n', '')
+  $kubelet_extra_config_alpha1_yaml = regsubst(stdlib::to_yaml($kubelet_extra_config_alpha1), '^---\n', '')
 
   $config_version = $kubernetes_version ? {
     /^1\.1(0|1)/              => 'v1alpha1',

manifests/config/worker.pp

@@ -48,8 +48,9 @@
 #     [{'key' => 'dedicated','value' => 'NewNode','effect' => 'NoSchedule', 'operator' => 'Equal'}]
 #   Defaults to undef
 # @param kubelet_extra_arguments
-#   A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied to both control planes and nodes.
-#   Use this for critical Kubelet settings such as `pod-infra-container-image` which may be problematic to configure via kubelet_extra_config
+#   A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied
+#   to both control planes and nodes. Use this for critical Kubelet settings such as `pod-infra-container-image`
+#   which may be problematic to configure via kubelet_extra_config
 #   Defaults to []
 # @param kubelet_extra_config
 #   A hash containing extra configuration data to be serialised with `to_yaml` and appended to Kubelet configuration file for the cluster.

manifests/init.pp

@@ -216,7 +216,8 @@
 #   Defaults to "new"
 #
 # [*etcd_compaction_retention*]
-#     This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`. An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision"
+#     This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`. 
+#     An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision"
 #   Integer or String
 #   Defaults to 0 (disabled)
 #
@@ -294,7 +295,8 @@
 #   Defaults to []
 #
 # [*delegated_pki*]
-#   Set to true if all required X509 certificates will be provided by external means. Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files.
+#   Set to true if all required X509 certificates will be provided by external means. 
+#   Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files.
 #   Defaults to false
 #
 # [*kubernetes_ca_crt*]
@@ -388,8 +390,9 @@
 #  Defaults to {}
 #
 # [*kubelet_extra_arguments*]
-#  A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied to both control planes and nodes.
-#  Use this for critical Kubelet settings such as `pod-infra-container-image` which may be problematic to configure via kubelet_extra_config
+#  A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied 
+#  to both control planes and nodes. Use this for critical Kubelet settings such as `pod-infra-container-image` 
+#  which may be problematic to configure via kubelet_extra_config
 #  Defaults to []
 #
 # [*proxy_mode*]

manifests/repos.pp

@@ -64,24 +64,24 @@
       'Debian': {
         $codename = fact('os.distro.codename')
         apt::source { 'kubernetes':
-          location => pick($kubernetes_apt_location,'https://apt.kubernetes.io'),
-          repos    => pick($kubernetes_apt_repos,'main'),
-          release  => pick($kubernetes_apt_release,'kubernetes-xenial'),
+          location => pick($kubernetes_apt_location, 'https://apt.kubernetes.io'),
+          repos    => pick($kubernetes_apt_repos, 'main'),
+          release  => pick($kubernetes_apt_release, 'kubernetes-xenial'),
           key      => {
-            'id'     => pick($kubernetes_key_id,'A362B822F6DEDC652817EA46B53DC80D13EDEF05'),
-            'source' => pick($kubernetes_key_source,'https://packages.cloud.google.com/apt/doc/apt-key.gpg'),
+            'id'     => pick($kubernetes_key_id, 'A362B822F6DEDC652817EA46B53DC80D13EDEF05'),
+            'source' => pick($kubernetes_key_source, 'https://packages.cloud.google.com/apt/doc/apt-key.gpg'),
           },
         }
 
         if ($container_runtime == 'docker' and $manage_docker == true) or
         ($container_runtime == 'cri_containerd' and $containerd_install_method == 'package') {
           apt::source { 'docker':
-            location => pick($docker_apt_location,'https://download.docker.com/linux/ubuntu/'),
-            repos    => pick($docker_apt_repos,'stable'),
+            location => pick($docker_apt_location, 'https://download.docker.com/linux/ubuntu/'),
+            repos    => pick($docker_apt_repos, 'stable'),
             release  => pick($docker_apt_release,$codename),
             key      => {
-              'id'     => pick($docker_key_id,'9DC858229FC7DD38854AE2D88D81803C0EBFCD88'),
-              'source' => pick($docker_key_source,'https://download.docker.com/linux/ubuntu/gpg'),
+              'id'     => pick($docker_key_id, '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'),
+              'source' => pick($docker_key_source, 'https://download.docker.com/linux/ubuntu/gpg'),
             },
           }
         }
@@ -91,16 +91,16 @@
         ($container_runtime == 'cri_containerd' and $containerd_install_method == 'package') {
           yumrepo { 'docker':
             descr    => 'docker',
-            baseurl  => pick($docker_yum_baseurl,'https://download.docker.com/linux/centos/7/x86_64/stable'),
-            gpgkey   => pick($docker_yum_gpgkey,'https://download.docker.com/linux/centos/gpg'),
+            baseurl  => pick($docker_yum_baseurl, 'https://download.docker.com/linux/centos/7/x86_64/stable'),
+            gpgkey   => pick($docker_yum_gpgkey, 'https://download.docker.com/linux/centos/gpg'),
             gpgcheck => true,
           }
         }
 
         yumrepo { 'kubernetes':
           descr    => 'Kubernetes',
-          baseurl  => pick($kubernetes_yum_baseurl,'https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64'),
-          gpgkey   => pick($kubernetes_yum_gpgkey,'https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg'),
+          baseurl  => pick($kubernetes_yum_baseurl, 'https://pkgs.k8s.io/core:/stable:/v1.28/rpm/'),
+          gpgkey   => pick($kubernetes_yum_gpgkey, 'https://pkgs.k8s.io/core:/stable:/v1.28/rpm/repodata/repomd.xml.key'),
           gpgcheck => true,
         }
       }

spec/acceptance/integration_kubernetes_spec.rb

@@ -55,7 +55,7 @@
       it 'verify the k8 nodes' do
         run_shell('sleep 20')
         run_shell('KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes') do |r|
-          expect(r.stdout).to match(%r{#{hostname1}(\s)+Ready(\s)+control-plane,master})
+          expect(r.stdout).to match(%r{#{hostname1}(\s)+Ready(\s)+control-plane})
           expect(r.stdout).to match(%r{#{hostname2}(\s)+Ready})
           expect(r.stdout).to match(%r{#{hostname3}(\s)+Ready})
         end

spec/acceptance/kubernetes_spec.rb

@@ -14,10 +14,10 @@
 
         pp = <<-MANIFEST
         case $facts['os']['family'] {
-          'RedHat', 'CentOS':  {
+          /^(RedHat|CentOS)$/:  {
             class {'kubernetes':
-              kubernetes_version => '1.22.0',
-              kubernetes_package_version => '1.22.0',
+              kubernetes_version => '1.28.15',
+              kubernetes_package_version => '1.28.15',
               controller_address => "#{int_ipaddr1}:6443",
               container_runtime => 'docker',
               manage_docker => false,
@@ -26,6 +26,7 @@ class {'kubernetes':
               environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
               ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
               cgroup_driver => 'systemd',
+              service_cidr => '10.138.0.0/12',
             }
           }
           /^(Debian|Ubuntu)$/: {
@@ -59,6 +60,8 @@ class {'kubernetes': } # any other OS are not supported
         before(:all) { change_target_host('controller') }
         after(:all) { reset_target_host }
 
+        int_ipaddr1 = fetch_ip_hostname_by_role('controller')[2]
+
         it 'can deploy an application into a namespace and expose it' do
           run_shell('KUBECONFIG=/etc/kubernetes/admin.conf kubectl create -f /tmp/nginx.yml') do |r|
             expect(r.stdout).to match(%r{my-nginx created\nservice/my-nginx created\n})
@@ -67,7 +70,8 @@ class {'kubernetes': } # any other OS are not supported
 
         it 'can access the deployed service' do
           run_shell('sleep 60')
-          run_shell('curl --retry 10 --retry-delay 15 -s 10.96.188.5') do |r|
+          shell_command = "curl --retry 10 --retry-delay 15 -s #{int_ipaddr1}"
+          run_shell(shell_command) do |r|
             expect(r.stdout).to match(%r{Welcome to nginx!})
           end
         end

spec/spec_helper_acceptance_local.rb

@@ -40,7 +40,8 @@ def fetch_ip_hostname_by_role(role)
   platform = fetch_platform_by_node(ipaddr)
   ENV['TARGET_HOST'] = target_roles(role)[0][:name]
   hostname = run_shell('hostname').stdout.strip
-  int_ipaddr = if os[:family] == 'redhat'
+  os_family = run_shell("facter -y os.family | cut -d':' -f2 | tr -d ' '").stdout.strip
+  int_ipaddr = if os_family.casecmp('redhat').zero?
                  run_shell("ip route get 8.8.8.8 | awk '{print $7; exit}'").stdout.strip
                else
                  run_shell("ip route get 8.8.8.8 | awk '{print $NF; exit}'").stdout.strip
@@ -72,8 +73,8 @@ def configure_puppet_server(controller, worker1, worker2)
   site_pp = <<-EOS
   node /#{controller[0]}/ {
     class {'kubernetes':
-      kubernetes_version => '1.20.6',
-      kubernetes_package_version => '1.20.6',
+      kubernetes_version => '1.28.15',
+      kubernetes_package_version => '1.28.15',
       controller_address => "#{controller[1]}:6443",
       container_runtime => 'docker',
       manage_docker => false,
@@ -82,6 +83,7 @@ class {'kubernetes':
       environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
       ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
       cgroup_driver => 'systemd',
+      service_cidr => '10.138.0.0/12',
     }
   }
   node /#{worker1}/ {
@@ -128,6 +130,27 @@ def execute_agent(role)
   run_shell('puppet agent --test', expect_failures: true)
 end
 
+def reset_and_restart_containerd
+  ['controller', 'worker1', 'worker2'].each do |node|
+    ENV['TARGET_HOST'] = target_roles(node)[0][:name]
+    run_shell('rm -f /etc/containerd/config.toml')
+    run_shell('systemctl restart containerd')
+  end
+end
+
+def open_communication_ports
+  ['controller', 'worker1', 'worker2'].each do |node|
+    ENV['TARGET_HOST'] = target_roles(node)[0][:name]
+    if node == 'controller'
+      run_shell('iptables -I INPUT -p tcp -m multiport --dports 2379,2380,6443,10250,10251,10252,30000:32767 -j ACCEPT')
+    else
+      run_shell('iptables -I INPUT -p tcp -m multiport --dports 10251,10252,10255,30000:32767 -j ACCEPT')
+    end
+    run_shell('iptables -I INPUT -p udp -m multiport --dports 8472 -j ACCEPT')
+    run_shell('iptables-save > /etc/sysconfig/iptables')
+  end
+end
+
 RSpec.configure do |c|
   c.before :suite do
     # Fetch hostname and  ip adress for each node
@@ -189,7 +212,7 @@ def execute_agent(role)
         labels:
           run: my-nginx
       spec:
-        clusterIP: 10.96.188.5
+        clusterIP: #{int_ipaddr1}
         ports:
         - port: 80
           protocol: TCP
@@ -216,24 +239,26 @@ def execute_agent(role)
             - "Redhat.yaml"
             - "common.yaml"
     EOS
+
     k8repo = <<~EOS
       [kubernetes]
       name=Kubernetes
-      baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
+      baseurl=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/
       enabled=1
-      gpgcheck=0
-      repo_gpgcheck=0
-      gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+      gpgcheck=1
+      gpgkey=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/repodata/repomd.xml.key
     EOS
+
     pp = <<-PUPPETCODE
-    # needed by tests
-    package { 'curl':
-      ensure   => 'latest',
-    }
-    package { 'git':
-      ensure   => 'latest',
-    }
+      # needed by tests
+      package { 'curl':
+        ensure   => 'latest',
+      }
+      package { 'git':
+        ensure   => 'latest',
+      }
     PUPPETCODE
+
     apply_manifest(pp)
     if %r{debian|ubuntu-1604-lts}.match?(family)
       runtime = 'cri_containerd'
@@ -255,7 +280,7 @@ def execute_agent(role)
         run_shell('/sbin/iptables -F')
       end
     end
-    if %r{redhat|centos}.match?(family)
+    if %r{rhel|redhat|centos}.match?(family)
       runtime = 'docker'
       cni = 'weave'
       ['controller', 'worker1', 'worker2'].each do |node|
@@ -265,9 +290,9 @@ def execute_agent(role)
         run_shell('systemctl stop firewalld && systemctl disable firewalld')
         run_shell('yum install -y yum-utils device-mapper-persistent-data lvm2')
         run_shell('yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo')
-        run_shell('yum update -y')
-        run_shell('yum install -y docker-ce-cli-18.09.0-3.el7.x86_64')
-        run_shell('yum install -y docker-ce-18.09.5-3.el7.x86_64')
+        run_shell('yum update -y --nobest')
+        run_shell('yum install -y docker-ce-cli')
+        run_shell('yum install -y docker-ce')
         run_shell('usermod -aG docker $(whoami)')
         run_shell('systemctl start docker.service')
         run_shell('systemctl enable docker.service')
@@ -278,7 +303,20 @@ def execute_agent(role)
 
     ENV['TARGET_HOST'] = target_roles('controller')[0][:name]
     run_shell('docker build -t kubetool:latest /etc/puppetlabs/code/environments/production/modules/kubernetes/tooling')
-    run_shell("docker run --rm -v $(pwd)/hieradata:/mnt -e OS=#{family} -e VERSION=1.20.6 -e CONTAINER_RUNTIME=#{runtime} -e CNI_PROVIDER=#{cni} -e ETCD_INITIAL_CLUSTER=#{hostname1}:#{int_ipaddr1} -e ETCD_IP=#{int_ipaddr1} -e ETCD_PEERS=[#{int_ipaddr1},#{int_ipaddr2},#{int_ipaddr3}] -e KUBE_API_ADVERTISE_ADDRESS=#{int_ipaddr1} -e INSTALL_DASHBOARD=true kubetool:latest") # rubocop:disable Layout/LineLength
+
+    docker_run = <<~DOCKER
+      docker run --rm -v $(pwd)/hieradata:/mnt -e OS=#{family} \
+                                               -e VERSION=1.28.15 \
+                                               -e CONTAINER_RUNTIME=#{runtime} \
+                                               -e CNI_PROVIDER=#{cni} \
+                                               -e ETCD_INITIAL_CLUSTER=#{hostname1}:#{int_ipaddr1} \
+                                               -e ETCD_IP=#{int_ipaddr1} \
+                                               -e ETCD_PEERS=[#{int_ipaddr1},#{int_ipaddr2},#{int_ipaddr3}] \
+                                               -e KUBE_API_ADVERTISE_ADDRESS=#{int_ipaddr1} \
+                                               -e INSTALL_DASHBOARD=true kubetool:latest
+    DOCKER
+
+    run_shell(docker_run)
     create_remote_file('nginx', '/tmp/nginx.yml', nginx)
     create_remote_file('hiera', '/etc/puppetlabs/puppet/hiera.yaml', hiera)
     run_shell('chmod 644 /etc/puppetlabs/puppet/hiera.yaml')
@@ -288,7 +326,6 @@ def execute_agent(role)
     run_shell('cp $HOME/hieradata/*.yaml /etc/puppetlabs/code/environments/production/hieradata/')
 
     run_shell("sed -i /cni_network_provider/d /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml")
-
     if %r{debian|ubuntu-1604-lts}.match?(family)
       run_shell("echo 'kubernetes::cni_network_provider: https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s-1.11.yaml' >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml") # rubocop:disable Layout/LineLength
     end
@@ -300,10 +337,14 @@ def execute_agent(role)
     run_shell("echo 'kubernetes::schedule_on_controller: true'  >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml")
     run_shell("echo 'kubernetes::taint_master: false' >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml")
     run_shell("echo 'kubernetes::manage_docker: false' >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml")
+
     run_shell("export KUBECONFIG='/etc/kubernetes/admin.conf'")
+    reset_and_restart_containerd
+    open_communication_ports
     execute_agent('controller')
     execute_agent('worker1')
     execute_agent('worker2')
     puppet_cert_sign
+    run_shell('KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml')
   end
 end