From 29c3139ad84b8921f5968c551790cb4090466cee Mon Sep 17 00:00:00 2001 From: Saurabh Pandit Date: Mon, 23 Dec 2024 21:04:13 +0530 Subject: [PATCH] (CAT-2095): Fixed puppetlabs-kubernetes modules CI & nightly failures 1. replaced CENTOS-7 with RHEL-8 2. Updated test as per new changes --- .github/workflows/ci.yml | 2 +- .github/workflows/nightly.yml | 2 +- Gemfile | 5 ++ manifests/config/kubeadm.pp | 17 ++-- manifests/config/worker.pp | 5 +- manifests/init.pp | 11 ++- manifests/repos.pp | 26 +++--- metadata.json | 2 +- .../acceptance/integration_kubernetes_spec.rb | 2 +- spec/acceptance/kubernetes_spec.rb | 12 ++- spec/spec_helper_acceptance_local.rb | 83 ++++++++++++++----- 11 files changed, 112 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24eedaa8..3721d6ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-20.04 strategy: fail-fast: false - matrix: {'platform':['centos-7'],'collection':['puppet7-nightly', 'puppet8-nightly']} + matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']} steps: - name: Checkout Source diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index dd654ab6..7ac9198e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 strategy: fail-fast: false - matrix: {'platform':['centos-7'],'collection':['puppet7-nightly', 'puppet8-nightly']} + matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']} steps: - name: Checkout Source diff --git a/Gemfile b/Gemfile index 2d8e1608..ad6d8ecf 100644 --- a/Gemfile +++ b/Gemfile @@ -76,3 +76,8 @@ extra_gemfiles.each do |gemfile| end end # vim: syntax=ruby + + +# Fixed version for puppet-modulebuilder gem, as newer version of this gem does not include tooling folder. +# We will keep this until we find a solution to either move the tooling folder in to some other folder or get rid of it altogether. +gem 'puppet-modulebuilder', '1.1.0' diff --git a/manifests/config/kubeadm.pp b/manifests/config/kubeadm.pp index e0de2b6c..79a045c7 100644 --- a/manifests/config/kubeadm.pp +++ b/manifests/config/kubeadm.pp @@ -11,7 +11,8 @@ # When set to true, etcd will be downloaded from the specified source URL. # Defaults to true. # @param delegated_pki -# Set to true if all required X509 certificates will be provided by external means. Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files. +# Set to true if all required X509 certificates will be provided by external means. +# Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files. # Defaults to false # @param etcd_install_method # The method on how to install etcd. Can be either wget (using etcd_source) or package (using $etcd_package_name) @@ -96,7 +97,8 @@ # "periodic" or "revision" # Defaults to "periodic" # @param etcd_compaction_retention -# This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`. An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision" +# This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`. +# An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision" # Integer or String # Defaults to 0 (disabled) # @param api_server_count @@ -157,8 +159,9 @@ # A string array of extra arguments to be passed to scheduler. # Defaults to [] # @param kubelet_extra_arguments -# A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied to both control planes and nodes. -# Use this for critical Kubelet settings such as `pod-infra-container-image` which may be problematic to configure via kubelet_extra_config +# A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied +# to both control planes and nodes. Use this for critical Kubelet settings such as `pod-infra-container-image` +# which may be problematic to configure via kubelet_extra_config # Defaults to [] # @param service_cidr # The IP assdress range for service VIPs @@ -381,9 +384,9 @@ } # to_yaml emits a complete YAML document, so we must remove the leading '---' - $kubeadm_extra_config_yaml = regsubst(to_yaml($kubeadm_extra_config), '^---\n', '') - $kubelet_extra_config_yaml = regsubst(to_yaml($kubelet_extra_config), '^---\n', '') - $kubelet_extra_config_alpha1_yaml = regsubst(to_yaml($kubelet_extra_config_alpha1), '^---\n', '') + $kubeadm_extra_config_yaml = regsubst(stdlib::to_yaml($kubeadm_extra_config), '^---\n', '') + $kubelet_extra_config_yaml = regsubst(stdlib::to_yaml($kubelet_extra_config), '^---\n', '') + $kubelet_extra_config_alpha1_yaml = regsubst(stdlib::to_yaml($kubelet_extra_config_alpha1), '^---\n', '') $config_version = $kubernetes_version ? { /^1\.1(0|1)/ => 'v1alpha1', diff --git a/manifests/config/worker.pp b/manifests/config/worker.pp index 89c15921..8a5f24b0 100644 --- a/manifests/config/worker.pp +++ b/manifests/config/worker.pp @@ -48,8 +48,9 @@ # [{'key' => 'dedicated','value' => 'NewNode','effect' => 'NoSchedule', 'operator' => 'Equal'}] # Defaults to undef # @param kubelet_extra_arguments -# A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied to both control planes and nodes. -# Use this for critical Kubelet settings such as `pod-infra-container-image` which may be problematic to configure via kubelet_extra_config +# A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied +# to both control planes and nodes. Use this for critical Kubelet settings such as `pod-infra-container-image` +# which may be problematic to configure via kubelet_extra_config # Defaults to [] # @param kubelet_extra_config # A hash containing extra configuration data to be serialised with `to_yaml` and appended to Kubelet configuration file for the cluster. diff --git a/manifests/init.pp b/manifests/init.pp index bdbcb535..3a75e107 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -216,7 +216,8 @@ # Defaults to "new" # # [*etcd_compaction_retention*] -# This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`. An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision" +# This will tell etcd how much retention to be applied. This value can change depending on `etcd_compaction_method`. +# An integer or time string (i.e.: "5m") can be used in case of "periodic". Only integer allowed in case of "revision" # Integer or String # Defaults to 0 (disabled) # @@ -294,7 +295,8 @@ # Defaults to [] # # [*delegated_pki*] -# Set to true if all required X509 certificates will be provided by external means. Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files. +# Set to true if all required X509 certificates will be provided by external means. +# Setting this to true will ignore all *_crt and *_key including sa.key and sa.pub files. # Defaults to false # # [*kubernetes_ca_crt*] @@ -388,8 +390,9 @@ # Defaults to {} # # [*kubelet_extra_arguments*] -# A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied to both control planes and nodes. -# Use this for critical Kubelet settings such as `pod-infra-container-image` which may be problematic to configure via kubelet_extra_config +# A string array to be appended to kubeletExtraArgs in the Kubelet's nodeRegistration configuration applied +# to both control planes and nodes. Use this for critical Kubelet settings such as `pod-infra-container-image` +# which may be problematic to configure via kubelet_extra_config # Defaults to [] # # [*proxy_mode*] diff --git a/manifests/repos.pp b/manifests/repos.pp index b6fbc63f..f236e8fb 100644 --- a/manifests/repos.pp +++ b/manifests/repos.pp @@ -64,24 +64,24 @@ 'Debian': { $codename = fact('os.distro.codename') apt::source { 'kubernetes': - location => pick($kubernetes_apt_location,'https://apt.kubernetes.io'), - repos => pick($kubernetes_apt_repos,'main'), - release => pick($kubernetes_apt_release,'kubernetes-xenial'), + location => pick($kubernetes_apt_location, 'https://apt.kubernetes.io'), + repos => pick($kubernetes_apt_repos, 'main'), + release => pick($kubernetes_apt_release, 'kubernetes-xenial'), key => { - 'id' => pick($kubernetes_key_id,'A362B822F6DEDC652817EA46B53DC80D13EDEF05'), - 'source' => pick($kubernetes_key_source,'https://packages.cloud.google.com/apt/doc/apt-key.gpg'), + 'id' => pick($kubernetes_key_id, 'A362B822F6DEDC652817EA46B53DC80D13EDEF05'), + 'source' => pick($kubernetes_key_source, 'https://packages.cloud.google.com/apt/doc/apt-key.gpg'), }, } if ($container_runtime == 'docker' and $manage_docker == true) or ($container_runtime == 'cri_containerd' and $containerd_install_method == 'package') { apt::source { 'docker': - location => pick($docker_apt_location,'https://download.docker.com/linux/ubuntu/'), - repos => pick($docker_apt_repos,'stable'), + location => pick($docker_apt_location, 'https://download.docker.com/linux/ubuntu/'), + repos => pick($docker_apt_repos, 'stable'), release => pick($docker_apt_release,$codename), key => { - 'id' => pick($docker_key_id,'9DC858229FC7DD38854AE2D88D81803C0EBFCD88'), - 'source' => pick($docker_key_source,'https://download.docker.com/linux/ubuntu/gpg'), + 'id' => pick($docker_key_id, '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'), + 'source' => pick($docker_key_source, 'https://download.docker.com/linux/ubuntu/gpg'), }, } } @@ -91,16 +91,16 @@ ($container_runtime == 'cri_containerd' and $containerd_install_method == 'package') { yumrepo { 'docker': descr => 'docker', - baseurl => pick($docker_yum_baseurl,'https://download.docker.com/linux/centos/7/x86_64/stable'), - gpgkey => pick($docker_yum_gpgkey,'https://download.docker.com/linux/centos/gpg'), + baseurl => pick($docker_yum_baseurl, 'https://download.docker.com/linux/centos/7/x86_64/stable'), + gpgkey => pick($docker_yum_gpgkey, 'https://download.docker.com/linux/centos/gpg'), gpgcheck => true, } } yumrepo { 'kubernetes': descr => 'Kubernetes', - baseurl => pick($kubernetes_yum_baseurl,'https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64'), - gpgkey => pick($kubernetes_yum_gpgkey,'https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg'), + baseurl => pick($kubernetes_yum_baseurl, 'https://pkgs.k8s.io/core:/stable:/v1.28/rpm/'), + gpgkey => pick($kubernetes_yum_gpgkey, 'https://pkgs.k8s.io/core:/stable:/v1.28/rpm/repodata/repomd.xml.key'), gpgcheck => true, } } diff --git a/metadata.json b/metadata.json index a480ec09..4c4122a8 100644 --- a/metadata.json +++ b/metadata.json @@ -14,7 +14,7 @@ }, { "name": "puppetlabs-apt", - "version_requirement": "< 10.0.0" + "version_requirement": "<= 10.0.0" }, { "name": "puppet-archive", diff --git a/spec/acceptance/integration_kubernetes_spec.rb b/spec/acceptance/integration_kubernetes_spec.rb index 6d79c978..a5f8c4ef 100644 --- a/spec/acceptance/integration_kubernetes_spec.rb +++ b/spec/acceptance/integration_kubernetes_spec.rb @@ -55,7 +55,7 @@ it 'verify the k8 nodes' do run_shell('sleep 20') run_shell('KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes') do |r| - expect(r.stdout).to match(%r{#{hostname1}(\s)+Ready(\s)+control-plane,master}) + expect(r.stdout).to match(%r{#{hostname1}(\s)+Ready(\s)+control-plane}) expect(r.stdout).to match(%r{#{hostname2}(\s)+Ready}) expect(r.stdout).to match(%r{#{hostname3}(\s)+Ready}) end diff --git a/spec/acceptance/kubernetes_spec.rb b/spec/acceptance/kubernetes_spec.rb index ad3ac867..8fd225ab 100644 --- a/spec/acceptance/kubernetes_spec.rb +++ b/spec/acceptance/kubernetes_spec.rb @@ -14,10 +14,10 @@ pp = <<-MANIFEST case $facts['os']['family'] { - 'RedHat', 'CentOS': { + /^(RedHat|CentOS)$/: { class {'kubernetes': - kubernetes_version => '1.22.0', - kubernetes_package_version => '1.22.0', + kubernetes_version => '1.28.15', + kubernetes_package_version => '1.28.15', controller_address => "#{int_ipaddr1}:6443", container_runtime => 'docker', manage_docker => false, @@ -26,6 +26,7 @@ class {'kubernetes': environment => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'], ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'], cgroup_driver => 'systemd', + service_cidr => '10.138.0.0/12', } } /^(Debian|Ubuntu)$/: { @@ -59,6 +60,8 @@ class {'kubernetes': } # any other OS are not supported before(:all) { change_target_host('controller') } after(:all) { reset_target_host } + int_ipaddr1 = fetch_ip_hostname_by_role('controller')[2] + it 'can deploy an application into a namespace and expose it' do run_shell('KUBECONFIG=/etc/kubernetes/admin.conf kubectl create -f /tmp/nginx.yml') do |r| expect(r.stdout).to match(%r{my-nginx created\nservice/my-nginx created\n}) @@ -67,7 +70,8 @@ class {'kubernetes': } # any other OS are not supported it 'can access the deployed service' do run_shell('sleep 60') - run_shell('curl --retry 10 --retry-delay 15 -s 10.96.188.5') do |r| + shell_command = "curl --retry 10 --retry-delay 15 -s #{int_ipaddr1}" + run_shell(shell_command) do |r| expect(r.stdout).to match(%r{Welcome to nginx!}) end end diff --git a/spec/spec_helper_acceptance_local.rb b/spec/spec_helper_acceptance_local.rb index 74fb0478..8158c328 100644 --- a/spec/spec_helper_acceptance_local.rb +++ b/spec/spec_helper_acceptance_local.rb @@ -40,7 +40,8 @@ def fetch_ip_hostname_by_role(role) platform = fetch_platform_by_node(ipaddr) ENV['TARGET_HOST'] = target_roles(role)[0][:name] hostname = run_shell('hostname').stdout.strip - int_ipaddr = if os[:family] == 'redhat' + os_family = run_shell("facter -y os.family | cut -d':' -f2 | tr -d ' '").stdout.strip + int_ipaddr = if os_family.casecmp('redhat').zero? run_shell("ip route get 8.8.8.8 | awk '{print $7; exit}'").stdout.strip else run_shell("ip route get 8.8.8.8 | awk '{print $NF; exit}'").stdout.strip @@ -72,8 +73,8 @@ def configure_puppet_server(controller, worker1, worker2) site_pp = <<-EOS node /#{controller[0]}/ { class {'kubernetes': - kubernetes_version => '1.20.6', - kubernetes_package_version => '1.20.6', + kubernetes_version => '1.28.15', + kubernetes_package_version => '1.28.15', controller_address => "#{controller[1]}:6443", container_runtime => 'docker', manage_docker => false, @@ -82,6 +83,7 @@ class {'kubernetes': environment => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'], ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'], cgroup_driver => 'systemd', + service_cidr => '10.138.0.0/12', } } node /#{worker1}/ { @@ -128,6 +130,27 @@ def execute_agent(role) run_shell('puppet agent --test', expect_failures: true) end +def reset_and_restart_containerd + ['controller', 'worker1', 'worker2'].each do |node| + ENV['TARGET_HOST'] = target_roles(node)[0][:name] + run_shell('rm -f /etc/containerd/config.toml') + run_shell('systemctl restart containerd') + end +end + +def open_communication_ports + ['controller', 'worker1', 'worker2'].each do |node| + ENV['TARGET_HOST'] = target_roles(node)[0][:name] + if node == 'controller' + run_shell('iptables -I INPUT -p tcp -m multiport --dports 2379,2380,6443,10250,10251,10252,30000:32767 -j ACCEPT') + else + run_shell('iptables -I INPUT -p tcp -m multiport --dports 10251,10252,10255,30000:32767 -j ACCEPT') + end + run_shell('iptables -I INPUT -p udp -m multiport --dports 8472 -j ACCEPT') + run_shell('iptables-save > /etc/sysconfig/iptables') + end +end + RSpec.configure do |c| c.before :suite do # Fetch hostname and ip adress for each node @@ -189,7 +212,7 @@ def execute_agent(role) labels: run: my-nginx spec: - clusterIP: 10.96.188.5 + clusterIP: #{int_ipaddr1} ports: - port: 80 protocol: TCP @@ -216,24 +239,26 @@ def execute_agent(role) - "Redhat.yaml" - "common.yaml" EOS + k8repo = <<~EOS [kubernetes] name=Kubernetes - baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 + baseurl=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/ enabled=1 - gpgcheck=0 - repo_gpgcheck=0 - gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + gpgcheck=1 + gpgkey=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/repodata/repomd.xml.key EOS + pp = <<-PUPPETCODE - # needed by tests - package { 'curl': - ensure => 'latest', - } - package { 'git': - ensure => 'latest', - } + # needed by tests + package { 'curl': + ensure => 'latest', + } + package { 'git': + ensure => 'latest', + } PUPPETCODE + apply_manifest(pp) if %r{debian|ubuntu-1604-lts}.match?(family) runtime = 'cri_containerd' @@ -255,7 +280,7 @@ def execute_agent(role) run_shell('/sbin/iptables -F') end end - if %r{redhat|centos}.match?(family) + if %r{rhel|redhat|centos}.match?(family) runtime = 'docker' cni = 'weave' ['controller', 'worker1', 'worker2'].each do |node| @@ -265,9 +290,9 @@ def execute_agent(role) run_shell('systemctl stop firewalld && systemctl disable firewalld') run_shell('yum install -y yum-utils device-mapper-persistent-data lvm2') run_shell('yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo') - run_shell('yum update -y') - run_shell('yum install -y docker-ce-cli-18.09.0-3.el7.x86_64') - run_shell('yum install -y docker-ce-18.09.5-3.el7.x86_64') + run_shell('yum update -y --nobest') + run_shell('yum install -y docker-ce-cli') + run_shell('yum install -y docker-ce') run_shell('usermod -aG docker $(whoami)') run_shell('systemctl start docker.service') run_shell('systemctl enable docker.service') @@ -278,7 +303,20 @@ def execute_agent(role) ENV['TARGET_HOST'] = target_roles('controller')[0][:name] run_shell('docker build -t kubetool:latest /etc/puppetlabs/code/environments/production/modules/kubernetes/tooling') - run_shell("docker run --rm -v $(pwd)/hieradata:/mnt -e OS=#{family} -e VERSION=1.20.6 -e CONTAINER_RUNTIME=#{runtime} -e CNI_PROVIDER=#{cni} -e ETCD_INITIAL_CLUSTER=#{hostname1}:#{int_ipaddr1} -e ETCD_IP=#{int_ipaddr1} -e ETCD_PEERS=[#{int_ipaddr1},#{int_ipaddr2},#{int_ipaddr3}] -e KUBE_API_ADVERTISE_ADDRESS=#{int_ipaddr1} -e INSTALL_DASHBOARD=true kubetool:latest") # rubocop:disable Layout/LineLength + + docker_run = <<~DOCKER + docker run --rm -v $(pwd)/hieradata:/mnt -e OS=#{family} \ + -e VERSION=1.28.15 \ + -e CONTAINER_RUNTIME=#{runtime} \ + -e CNI_PROVIDER=#{cni} \ + -e ETCD_INITIAL_CLUSTER=#{hostname1}:#{int_ipaddr1} \ + -e ETCD_IP=#{int_ipaddr1} \ + -e ETCD_PEERS=[#{int_ipaddr1},#{int_ipaddr2},#{int_ipaddr3}] \ + -e KUBE_API_ADVERTISE_ADDRESS=#{int_ipaddr1} \ + -e INSTALL_DASHBOARD=true kubetool:latest + DOCKER + + run_shell(docker_run) create_remote_file('nginx', '/tmp/nginx.yml', nginx) create_remote_file('hiera', '/etc/puppetlabs/puppet/hiera.yaml', hiera) run_shell('chmod 644 /etc/puppetlabs/puppet/hiera.yaml') @@ -288,7 +326,6 @@ def execute_agent(role) run_shell('cp $HOME/hieradata/*.yaml /etc/puppetlabs/code/environments/production/hieradata/') run_shell("sed -i /cni_network_provider/d /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml") - if %r{debian|ubuntu-1604-lts}.match?(family) run_shell("echo 'kubernetes::cni_network_provider: https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s-1.11.yaml' >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml") # rubocop:disable Layout/LineLength end @@ -300,10 +337,14 @@ def execute_agent(role) run_shell("echo 'kubernetes::schedule_on_controller: true' >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml") run_shell("echo 'kubernetes::taint_master: false' >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml") run_shell("echo 'kubernetes::manage_docker: false' >> /etc/puppetlabs/code/environments/production/hieradata/#{family.capitalize}.yaml") + run_shell("export KUBECONFIG='/etc/kubernetes/admin.conf'") + reset_and_restart_containerd + open_communication_ports execute_agent('controller') execute_agent('worker1') execute_agent('worker2') puppet_cert_sign + run_shell('KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml') end end