From 6576a600d4721f36913d9ae9d48f04bc83eb0af4 Mon Sep 17 00:00:00 2001 From: Michele Costa Date: Tue, 27 Sep 2022 15:26:49 +0100 Subject: [PATCH] Add flow for using agent subcommand of openshift installer --- deploy_cluster.yml | 21 +-- deploy_prerequisites.yml | 1 + .../crucible_installer_features_comparison.md | 29 ++++ docs/network_config.md | 135 ++++++++++++++++++ .../deploy_cluster_agent_based_installer.yml | 15 ++ .../deploy_cluster_assisted_installer.yml | 19 +++ playbooks/extract_agent_based_installer.yml | 15 ++ playbooks/generate_agent_iso.yml | 5 + playbooks/generate_manifests.yml | 14 ++ playbooks/monitor_agent_based_installer.yml | 5 + .../defaults/main.yml | 6 + .../tasks/main.yml | 40 ++++++ roles/generate_agent_iso/defaults/main.yml | 9 ++ roles/generate_agent_iso/tasks/main.yml | 36 +++++ roles/generate_manifests/defaults/main.yml | 18 +++ roles/generate_manifests/tasks/main.yml | 55 +++++++ roles/generate_manifests/tasks/manifest.yml | 26 ++++ roles/generate_manifests/tasks/static.yml | 29 ++++ .../templates/agent-config.yaml.j2 | 18 +++ .../templates/install-config.yaml.j2 | 60 ++++++++ .../templates/registry-config.j2 | 15 ++ roles/get_image_hash/defaults/main.yml | 2 + roles/get_image_hash/tasks/main.yml | 3 +- .../defaults/main.yml | 2 + .../tasks/main.yml | 10 ++ roles/populate_mirror_registry/tasks/main.yml | 1 + roles/post_install/defaults/main.yml | 8 +- .../credentails_agent_based_installer.yml | 11 ++ .../tasks/credentails_assisted_installer.yml | 17 +++ .../fetch_kubeconfg_agent_based_installer.yml | 5 + .../fetch_kubeconfg_assisted_installer.yml | 5 + roles/post_install/tasks/main.yml | 46 +++--- .../agent_based_installer_feature_gates.yml | 26 ++++ .../agent_based_installer_requirements.yml | 12 ++ roles/validate_inventory/tasks/main.yml | 17 +++ site.yml | 1 + 36 files changed, 696 insertions(+), 41 deletions(-) create mode 100644 docs/crucible_installer_features_comparison.md create mode 100644 docs/network_config.md create mode 100644 playbooks/deploy_cluster_agent_based_installer.yml create mode 100644 playbooks/deploy_cluster_assisted_installer.yml create mode 100644 playbooks/extract_agent_based_installer.yml create mode 100644 playbooks/generate_agent_iso.yml create mode 100644 playbooks/generate_manifests.yml create mode 100644 playbooks/monitor_agent_based_installer.yml create mode 100644 roles/extract_openshift_installer/defaults/main.yml create mode 100644 roles/extract_openshift_installer/tasks/main.yml create mode 100644 roles/generate_agent_iso/defaults/main.yml create mode 100644 roles/generate_agent_iso/tasks/main.yml create mode 100644 roles/generate_manifests/defaults/main.yml create mode 100644 roles/generate_manifests/tasks/main.yml create mode 100644 roles/generate_manifests/tasks/manifest.yml create mode 100644 roles/generate_manifests/tasks/static.yml create mode 100644 roles/generate_manifests/templates/agent-config.yaml.j2 create mode 100644 roles/generate_manifests/templates/install-config.yaml.j2 create mode 100644 roles/generate_manifests/templates/registry-config.j2 create mode 100644 roles/monitor_agent_based_installer/defaults/main.yml create mode 100644 roles/monitor_agent_based_installer/tasks/main.yml create mode 100644 roles/post_install/tasks/credentails_agent_based_installer.yml create mode 100644 roles/post_install/tasks/credentails_assisted_installer.yml create mode 100644 roles/post_install/tasks/fetch_kubeconfg_agent_based_installer.yml create mode 100644 roles/post_install/tasks/fetch_kubeconfg_assisted_installer.yml create mode 100644 roles/validate_inventory/tasks/agent_based_installer_feature_gates.yml create mode 100644 roles/validate_inventory/tasks/agent_based_installer_requirements.yml diff --git a/deploy_cluster.yml b/deploy_cluster.yml index c07066a7..0d9a25ee 100644 --- a/deploy_cluster.yml +++ b/deploy_cluster.yml @@ -1,19 +1,6 @@ --- -- import_playbook: playbooks/generate_ssh_key_pair.yml - when: generate_ssh_keys | default(True) == True +- import_playbook: playbooks/deploy_cluster_agent_based_installer.yml + when: (use_agent_based_installer | default(false)) | bool -- import_playbook: playbooks/create_cluster.yml - -- import_playbook: playbooks/generate_discovery_iso.yml - -- import_playbook: playbooks/mount_discovery_iso_for_pxe.yml - -- import_playbook: playbooks/boot_iso.yml - vars: - boot_iso_hosts: masters,workers - -- import_playbook: playbooks/install_cluster.yml - -- import_playbook: playbooks/monitor_hosts.yml - -- import_playbook: playbooks/monitor_cluster.yml +- import_playbook: playbooks/deploy_cluster_assisted_installer.yml + when: not ((use_agent_based_installer | default(false)) | bool) diff --git a/deploy_prerequisites.yml b/deploy_prerequisites.yml index 8df42f4f..8a0063c1 100644 --- a/deploy_prerequisites.yml +++ b/deploy_prerequisites.yml @@ -21,5 +21,6 @@ - import_playbook: playbooks/deploy_registry.yml - import_playbook: playbooks/deploy_assisted_installer_onprem.yml + when: not ((use_agent_based_installer | default(false)) | bool) - import_playbook: playbooks/deploy_sushy_tools.yml diff --git a/docs/crucible_installer_features_comparison.md b/docs/crucible_installer_features_comparison.md new file mode 100644 index 00000000..3b739a78 --- /dev/null +++ b/docs/crucible_installer_features_comparison.md @@ -0,0 +1,29 @@ +# Crucible Features + +This is a comparison of the features available through crucible depending on which installer is used + +| Feature | Assisted installer (on-prem) | Agent based installer | +| -------------------------------------------------- | ----------------------------- | ------------------------ | +| Compact cluster | Y | Y | +| Workers | Y | Y | +| SNO | Y | Y | +| 2 day workers | Y | N[1] | +| Set Network type | Y | Y | +| DHCP | Y | Y[2] | +| IPV6 | Y | Y | +| Dual Stack | Y | Y | +| NMState network config | Y | Y | +| Mirror Registry support | Y | Y | +| Set hostname | Y | Y | +| Set role | Y | Y | +| Proxy | Y | Y | +| Install OLM Operators (LSO, ODF, CNV) | Y | N[3] | +| Patitions | Y | N[4] | +| Discovery iso password | Y | N[4] | +| - | - | - | + +Footnotes: +[1] There are plans for the agent based method to install the [multicluster engine operator](https://docs.openshift.com/container-platform/4.12/architecture/mce-overview-ocp.html) which crucible could then leverage to add day2 workers. +[2] A `network_config` is still required however you could provide a raw nmstate, which configures the interfaces for dhcp and the corisponding `mac_interface_map`. If you are not using the DHCP provided by crucible you would need to provide the correct IP for the bootstrap node (by default the first node in the masters group). +[3] It is possible to apply extra manifests to deploy those operators as part of the install. The MCE deploy ment mentioned in [1] will likely expose this feature as well. +[4] This feature of crucible is done by modifing an iginition file which is not currently possible in the agent based flow. diff --git a/docs/network_config.md b/docs/network_config.md new file mode 100644 index 00000000..b69002e2 --- /dev/null +++ b/docs/network_config.md @@ -0,0 +1,135 @@ +# Network configuration +Network configuration can currently be used in two places in the inventory to configure the network config of a node and the network config of a vm_host. + +The `network_config` entry on a node is a simplified version of the `nmstate`([nmstate.io](http://nmstate.io/)) required by the [assisted installer api](https://github.com/openshift/assisted-service/blob/3bcaca8abef5173b0e2175b5d0b722e851e39cee/docs/user-guide/restful-api-guide.md). + +#### Static IPs + +To activate static IPs in the discovery iso and resulting cluster there is some configuration required in the inventory. + +```yaml +network_config: + interfaces: + - name: "{{ interface }}" + mac: "{{ mac }}" + addresses: + ipv4: + - ip: "{{ ansible_host}}" + prefix: "{{ mask }}" + dns_server_ips: + - "{{ dns }}" + - "{{ dns2 }}" + routes: # optional + - destination: 0.0.0.0/0 + address: "{{ gateway }}" + interface: "{{ interface }}" +``` + +where the variables are as follows: + +- `ip`: The static IP is set +- `dns` & `dns2`: IPs of the DNS servers +- `gateway`: IP of the gateway +- `mask`: Length of subnet mask (e.g. 24) +- `interface`: The name of the interface you wish to configure +- `mac`: Mac address of the interface you wish to configure + +## Examples + +### Link Aggregation + +```yaml +network_config: + interfaces: + - name: bond0 + type: bond + state: up + addresses: + ipv4: + - ip: 172.17.0.101 + prefix: 24 + link_aggregation: + mode: active-backup + options: + miimon: "1500" + slaves: + - ens7f0 + - ens7f1 + - name: ens1f0 + type: ethernet + mac: "40:A6:B7:3D:B3:70" + state: up + - name: ens1f1 + type: ethernet + mac: "40:A6:B7:3D:B3:71" + state: up + dns_server_ips: + - 10.40.0.100 + routes: + - destination: 0.0.0.0/0 + address: 172.17.0.1 + interface: bond0 +``` + +### Dual Stack: +``` yaml +network_config: + interfaces: + - name: "enp1s0" + mac: "{{ mac }}" + addresses: + ipv4: + - ip: "{{ ansible_host }}" + prefix: "{{ ipv4.mask }}" + ipv6: + - ip: "{{ ipv6_address }}" + prefix: "{{ ipv6.mask }}" + dns_server_ips: + - "{{ ipv6.dns }}" + - "{{ ipv4.dns }}" + routes: + - destination: "0:0:0:0:0:0:0:0/0" + address: "{{ ipv6.gateway }}" + interface: "enp1s0" + - destination: 0.0.0.0/0 + address: "{{ ipv4.gateway }}" + interface: "enp1s0" +``` + +## Advanced + +### Raw nmstate + + If you wish to write the `nmstate` by hand you can use the `network_config.raw` entry, however you will also need to add `mac_interface_map`, the following is static ipv4 address + +```yaml +mac_interface_map: + - logical_nic_name: "enp1s0" + mac_address: "{{ mac }}" +network_config: + raw: + dns-resolver: + config: + server: + - "{{ dns }}" + interfaces: + - name: enp1s0 + state: up + type: ethernet + ipv4: + address: + - ip: "{{ ansible_host }}" + prefix-length: "{{ mask }}" + dhcp: false + enabled: true + routes: + config: + - destination: 0.0.0.0/0 + next-hop-address: "{{ gateway }}" + next-hop-interface: enp1s0 + table-id: 254 +``` + + +### Custom template +If you wish to use your own template you can set `network_config.template` with a path to your desired template the default can be found [here](../roles/generate_discovery_iso/templates/nmstate.yml.j2). diff --git a/playbooks/deploy_cluster_agent_based_installer.yml b/playbooks/deploy_cluster_agent_based_installer.yml new file mode 100644 index 00000000..823d2ec2 --- /dev/null +++ b/playbooks/deploy_cluster_agent_based_installer.yml @@ -0,0 +1,15 @@ +--- +- import_playbook: extract_agent_based_installer.yml + when: agent_based_installer_path is not defined + +- import_playbook: generate_ssh_key_pair.yml + when: generate_ssh_keys | default(True) == True + +- import_playbook: generate_manifests.yml +- import_playbook: generate_agent_iso.yml + +- import_playbook: boot_iso.yml + vars: + boot_iso_hosts: masters,workers + +- import_playbook: monitor_agent_based_installer.yml diff --git a/playbooks/deploy_cluster_assisted_installer.yml b/playbooks/deploy_cluster_assisted_installer.yml new file mode 100644 index 00000000..a9d425fc --- /dev/null +++ b/playbooks/deploy_cluster_assisted_installer.yml @@ -0,0 +1,19 @@ +--- +- import_playbook: generate_ssh_key_pair.yml + when: generate_ssh_keys | default(True) == True + +- import_playbook: create_cluster.yml + +- import_playbook: generate_discovery_iso.yml + +- import_playbook: mount_discovery_iso_for_pxe.yml + +- import_playbook: boot_iso.yml + vars: + boot_iso_hosts: masters,workers + +- import_playbook: install_cluster.yml + +- import_playbook: monitor_hosts.yml + +- import_playbook: monitor_cluster.yml diff --git a/playbooks/extract_agent_based_installer.yml b/playbooks/extract_agent_based_installer.yml new file mode 100644 index 00000000..24d996d6 --- /dev/null +++ b/playbooks/extract_agent_based_installer.yml @@ -0,0 +1,15 @@ +--- +- name: Extract openshift installer + hosts: bastion + gather_facts: false + vars: + destination_hosts: + - bastion + pre_tasks: + - name: pre-compute need to get hashes + set_fact: + run_get_hash: "{{ assisted_installer_release_images | default({}) | length == 0 }}" + roles: + - role: get_image_hash + when: run_get_hash | bool + - extract_openshift_installer diff --git a/playbooks/generate_agent_iso.yml b/playbooks/generate_agent_iso.yml new file mode 100644 index 00000000..07b11c45 --- /dev/null +++ b/playbooks/generate_agent_iso.yml @@ -0,0 +1,5 @@ +--- +- name: Generate agent iso using agent_based_installer + hosts: bastion + roles: + - generate_agent_iso diff --git a/playbooks/generate_manifests.yml b/playbooks/generate_manifests.yml new file mode 100644 index 00000000..f20b9b82 --- /dev/null +++ b/playbooks/generate_manifests.yml @@ -0,0 +1,14 @@ +--- +- name: Generate manfiests for agent_based_installer + hosts: bastion + vars: + destination_hosts: + - bastion + pre_tasks: + - name: pre-compute need to get hashes + set_fact: + run_get_hash: "{{ image_hashes | default({}) | length == 0 }}" + roles: + - role: get_image_hash + when: run_get_hash + - generate_manifests diff --git a/playbooks/monitor_agent_based_installer.yml b/playbooks/monitor_agent_based_installer.yml new file mode 100644 index 00000000..add6a7c8 --- /dev/null +++ b/playbooks/monitor_agent_based_installer.yml @@ -0,0 +1,5 @@ +- name: Monitor install process of agent_based_installer + hosts: bastion + gather_facts: False + roles: + - monitor_agent_based_installer diff --git a/roles/extract_openshift_installer/defaults/main.yml b/roles/extract_openshift_installer/defaults/main.yml new file mode 100644 index 00000000..e2093907 --- /dev/null +++ b/roles/extract_openshift_installer/defaults/main.yml @@ -0,0 +1,6 @@ +arch: x86_64 +version_filter: "[?(openshift_version == '{{ openshift_version }}') && (cpu_architecture == '{{ arch }}')]" +release_image: "{{ (assisted_installer_release_images | json_query(version_filter))[0].url }}" +extact_dest_path: /tmp/wip/extract/ +pull_secret_file: "{{ extact_dest_path }}/pull_secret.txt" +openshift_installer_path: "{{ extact_dest_path }}/openshift-install" diff --git a/roles/extract_openshift_installer/tasks/main.yml b/roles/extract_openshift_installer/tasks/main.yml new file mode 100644 index 00000000..64212a42 --- /dev/null +++ b/roles/extract_openshift_installer/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: "Create {{ extact_dest_path }}" + ansible.builtin.file: + path: "{{ extact_dest_path }}" + state: directory + mode: 0755 + recurse: true + +- name: Create pull_secret_file + ansible.builtin.copy: + content: "{{ pull_secret }}" + dest: "{{ pull_secret_file }}" + mode: "0600" + +- name: Extract openshift_installer + ansible.builtin.shell: + cmd: > + oc adm + -a {{ pull_secret_file }} + release extract + --command=openshift-install + {{ release_image }} + --to={{ extact_dest_path }} + +- name: Check extracted installer has agent subcommand + ansible.builtin.shell: + cmd: "{{ openshift_installer_path }} agent --help" + register: res + failed_when: false + +- name: Check agent sub-commmand output + ansible.builtin.fail: + msg: > + Version of openshift install extracted from + release image does not have agent subcommand + when: "'unknown command' in res.stderr" + +- name: Set agent_based_installer_path + ansible.builtin.set_fact: + agent_based_installer_path: "{{ openshift_installer_path }}" diff --git a/roles/generate_agent_iso/defaults/main.yml b/roles/generate_agent_iso/defaults/main.yml new file mode 100644 index 00000000..c68942dc --- /dev/null +++ b/roles/generate_agent_iso/defaults/main.yml @@ -0,0 +1,9 @@ +generated_dir: "{{ repo_root_path }}/generated" +manifests_dir: "{{ generated_dir }}/{{ cluster_name }}" +download_agent_dest_file: "{{ discovery_iso_name }}" +download_dest_path: "{{ iso_download_dest_path | default('/opt/http_store/data') }}" +config_file_path: /tmp/wip/config +arch: x86_64 +version_filter: "[?(openshift_version == '{{ openshift_version }}') && (cpu_architecture == '{{ arch }}')]" +release_image: "{{ (assisted_installer_release_images | json_query(version_filter))[0].url }}" +use_local_mirror_registry: false diff --git a/roles/generate_agent_iso/tasks/main.yml b/roles/generate_agent_iso/tasks/main.yml new file mode 100644 index 00000000..f8cbdc07 --- /dev/null +++ b/roles/generate_agent_iso/tasks/main.yml @@ -0,0 +1,36 @@ +- name: Create podman auth dir + ansible.builtin.file: + path: "{{ ansible_env.HOME }}/.docker" + state: directory + mode: 0755 + recurse: true + +- name: Copy pull_secret file. + ansible.builtin.copy: + content: "{{ pull_secret }}" + dest: "{{ ansible_env.HOME }}/.docker/config.json" + mode: 0644 + +- name: Generate ISO + ansible.builtin.shell: + cmd: "{{ agent_based_installer_path }} --log-level=debug agent create image" + chdir: "{{ manifests_dir }}" + environment: + XDG_RUNTIME_DIR: "{{ config_file_path }}" + REGISTRY_AUTH_FILE: "{{ config_file_path }}/containers/auth.json" + +- name: Put discovery iso in http store + delegate_to: http_store + become: true + block: + - name: Create discovery directory + ansible.builtin.file: + path: "{{ download_dest_path }}/{{ download_agent_dest_file | dirname }}" + recurse: true + state: directory + + - name: Copy agent iso to discovery directory + ansible.builtin.copy: + src: "{{ manifests_dir }}/agent.{{ arch | default('x86_64') }}.iso" + dest: "{{ download_dest_path }}/{{ download_agent_dest_file }}" + mode: 0644 diff --git a/roles/generate_manifests/defaults/main.yml b/roles/generate_manifests/defaults/main.yml new file mode 100644 index 00000000..aa3a1c4d --- /dev/null +++ b/roles/generate_manifests/defaults/main.yml @@ -0,0 +1,18 @@ +generated_dir: "{{ repo_root_path }}/generated" +manifests_dir: "{{ generated_dir }}/{{ cluster_name }}" +cluster_manifest_dir: "{{ manifests_dir }}/cluster-manifests" +extra_manifest_dir: "{{ manifests_dir }}/openshift" +mac_interface_default_mapping: "interfaces[?(name != null && mac != null)].{logical_nic_name: name, mac_address: mac}" +static_network_config: {} +arch: x86_64 +version_filter: "[?(openshift_version == '{{ openshift_version }}') && (cpu_architecture == '{{ arch }}')]" +release_image: "{{ (assisted_installer_release_images | json_query(version_filter))[0].url }}" +mirror_registry: "{{ hostvars['registry_host']['registry_fqdn'] }}:{{ hostvars['registry_host']['registry_port'] }}" +agent_based_installer_bootstrap_node: "{{ groups['masters'][0] }}" +host_ip_keyword: ansible_host +use_local_mirror_registry: "{{ setup_registry_service | default(true) }}" +single_node_openshift_enabled: "{{ is_valid_single_node_openshift_config | default(false) }}" + +manifests: true +extra_manifests: [] +manifest_templates: "{{ extra_manifests }}" diff --git a/roles/generate_manifests/tasks/main.yml b/roles/generate_manifests/tasks/main.yml new file mode 100644 index 00000000..49c983dc --- /dev/null +++ b/roles/generate_manifests/tasks/main.yml @@ -0,0 +1,55 @@ +- name: Render nmstate + ansible.builtin.include_tasks: static.yml + loop: "{{ groups['nodes'] }}" + +- name: Make cluster-manifests dir + ansible.builtin.file: + name: "{{ manifests_dir }}" + state: absent + ignore_errors: true + +- name: Make cluster-manifests dir + ansible.builtin.file: + name: "{{ cluster_manifest_dir }}" + mode: 0775 + recurse: true + state: directory + +- name: Update pull_secret variable + ansible.builtin.set_fact: + local_pull_secret: "{{ pull_secret | combine({ + 'auths': pull_secret['auths'] | combine({ + 'registry.ci.openshift.org': { + 'auth': pull_secret['auths'][mirror_registry]['auth'], + } + }) + }) + }}" + when: use_local_mirror_registry | bool + +- name: Render agent-config templates + ansible.builtin.template: + src: "{{ item }}" + dest: "{{ manifests_dir }}/{{ item.rsplit('.', 1)[0] }}" + mode: 0644 + trim_blocks: true + lstrip_blocks: true + loop: + - agent-config.yaml.j2 + - install-config.yaml.j2 + +- name: Create extra_manifest_dir dir + ansible.builtin.file: + name: "{{ extra_manifest_dir }}" + mode: 0775 + recurse: true + state: directory + when: + - manifests | bool + - manifest_templates is defined + - manifest_templates | length >= 0 + +- name: Render extra_manifests + ansible.builtin.include_tasks: manifest.yml + loop: "{{ manifest_templates }}" + when: manifests | bool diff --git a/roles/generate_manifests/tasks/manifest.yml b/roles/generate_manifests/tasks/manifest.yml new file mode 100644 index 00000000..95cf72cf --- /dev/null +++ b/roles/generate_manifests/tasks/manifest.yml @@ -0,0 +1,26 @@ +--- +# tasks file for manifests + +- name: Fail if manifest is not a mapping + ansible.builtin.fail: + msg: "Manifest item must be a dict with either template or file as the key" + when: item is not mapping and (("template" in item) or ("file" in item)) + +- name: Load manifest + ansible.builtin.set_fact: + manifest_content: "{{ lookup('template', item.template) }}" + manifest_name: "{{ item.template.rsplit('.j2', 1)[0] | basename }}" + when: ("template" in item) + + +- name: Load manifest + ansible.builtin.set_fact: + manifest_content: "{{ lookup('file', item.file) }}" + manifest_name: "{{ item.file | basename }}" + when: ("file" in item) + +- name: Save Manifest + ansible.builtin.copy: + content: "{{ manifest_content }}" + dest: "{{ extra_manifest_dir }}/{{ manifest_name }}" + mode: 0644 diff --git a/roles/generate_manifests/tasks/static.yml b/roles/generate_manifests/tasks/static.yml new file mode 100644 index 00000000..a3c53fe2 --- /dev/null +++ b/roles/generate_manifests/tasks/static.yml @@ -0,0 +1,29 @@ +- name: "Set network config for {{ item }}" + ansible.builtin.set_fact: + item_network_config: "{{ hostvars[item].network_config }}" + +- name: "Set default value of mac_interface_map for {{ item }}" + ansible.builtin.set_fact: + mac_interface_map: "{{ hostvars[item].mac_interface_map | default([]) }}" + +- name: "Using query to set mac_interface_map for {{ item }}" + ansible.builtin.set_fact: + mac_interface_map: "{{ item_network_config | json_query(network_config.mapping_query | default(mac_interface_default_mapping)) }}" + when: hostvars[item].mac_interface_map is not defined + +- name: Process network_config + ansible.builtin.import_role: + name: process_nmstate + vars: + network_config: "{{ item_network_config }}" + target_name: "{{ item }}" + +- name: "Set static network config for {{ item }}" + ansible.builtin.set_fact: + static_network_config_entry: + network_yaml: "{{ rendered_nmstate_yml }}" + mac_interface_map: "{{ mac_interface_map }}" + +- name: Update static_network_config_items + ansible.builtin.set_fact: + static_network_config: "{{ static_network_config | combine({item: static_network_config_entry}) }}" diff --git a/roles/generate_manifests/templates/agent-config.yaml.j2 b/roles/generate_manifests/templates/agent-config.yaml.j2 new file mode 100644 index 00000000..1a09d03d --- /dev/null +++ b/roles/generate_manifests/templates/agent-config.yaml.j2 @@ -0,0 +1,18 @@ +#jinja2:trim_blocks: True, lstrip_blocks: True +apiVersion: v1alpha1 +kind: AgentConfig +rendezvousIP: {{ hostvars[agent_based_installer_bootstrap_node][host_ip_keyword] }} +hosts: +{% for hostname, network_config in static_network_config.items() %} + - role: {{ hostvars[hostname]['role'] }} + hostname: {{ hostname }} + interfaces: + - name: {{ network_config.mac_interface_map[0].logical_nic_name }} + macAddress: {{ network_config.mac_interface_map[0].mac_address }} + networkConfig: + {{ network_config.network_yaml | indent(6) }} + {% if hostvars[hostname]['installation_disk_path'] is defined %} + rootDeviceHints: + deviceName: {{ hostvars[hostname]['installation_disk_path'] }} + {% endif %} +{% endfor %} diff --git a/roles/generate_manifests/templates/install-config.yaml.j2 b/roles/generate_manifests/templates/install-config.yaml.j2 new file mode 100644 index 00000000..6bcc34b3 --- /dev/null +++ b/roles/generate_manifests/templates/install-config.yaml.j2 @@ -0,0 +1,60 @@ +#jinja2:trim_blocks: True, lstrip_blocks: True +apiVersion: v1 +baseDomain: {{ base_dns_domain}} +controlPlane: + name: master + replicas: {{ groups['masters'] | length }} +compute: + - name: worker + replicas: {{ (groups['workers'] | default([]))| length }} +metadata: + namespace: cluster0 + name: {{ cluster_name }} +networking: + clusterNetwork: + - cidr: {{ cluster_network_cidr }} + hostPrefix: {{ cluster_network_host_prefix }} + {% for cluster_network in (extra_cluster_networks | default([])) %} + - cidr: {{ cluster_network.cidr }} + hostPrefix: {{ cluster_network.host_prefix }} + {% endfor %} + {% if network_type is defined %} + networkType: {{ network_type }} + {% endif %} + machineNetwork: + - cidr: {{ machine_network_cidr }} + {% for machine_network in (extra_machine_networks | default([])) %} + - cidr: {{ machine_network.cidr }} + {% endfor %} + serviceNetwork: + - {{ service_network_cidr }} + {% for service_network in (extra_service_networks | default([])) %} + - {{ service_network.cidr }} + {% endfor %} +platform: + {% if single_node_openshift_enabled | bool %} + none: {} + {% else %} + baremetal: + apiVips: + - {{ api_vip }} + ingressVips: + - {{ ingress_vip }} + {% endif %} +sshKey: {{ ssh_public_key }} +pullSecret: '{{ pull_secret | to_json }}' +{% if (http_proxy is defined and http_proxy != "") or (https_proxy is defined and https_proxy != "") or (no_proxy is defined and no_proxy != "") %} +proxy: + {% if http_proxy is defined and http_proxy != "" %} + httpProxy: {{ http_proxy }} + {% endif %} + {% if https_proxy is defined and https_proxy != "" %} + httpsProxy: {{ https_proxy }} + {% endif %} + {% if no_proxy is defined and no_proxy != "" %} + noProxy: {{ no_proxy }} + {% endif %} +{% endif %} +{% if (use_local_mirror_registry | bool) and (True != True) %} +{{ lookup('template', registry-config.j2) }} +{% endif %} diff --git a/roles/generate_manifests/templates/registry-config.j2 b/roles/generate_manifests/templates/registry-config.j2 new file mode 100644 index 00000000..9c728f5b --- /dev/null +++ b/roles/generate_manifests/templates/registry-config.j2 @@ -0,0 +1,15 @@ +imageContentSources: + - source: quay.io/openshift-release-dev/ocp-release + mirrors: + - {{ mirror_registry }}/ocp4/openshift4 + - source: quay.io/openshift-release-dev/ocp-v4.0-art-dev + mirrors: + - {{ mirror_registry }}/ocp4/openshift4 + - source: quay.io/edge-infrastructure + mirrors: + - {{ mirror_registry }}/ocpmetal + - source: quay.io/edge-infrastructure + mirrors: + - {{ mirror_registry }}/edge-infrastructure +additionalTrustBundle: | +{{ mirror_certificate | indent(2) }} diff --git a/roles/get_image_hash/defaults/main.yml b/roles/get_image_hash/defaults/main.yml index 4b7c46a8..267115f1 100644 --- a/roles/get_image_hash/defaults/main.yml +++ b/roles/get_image_hash/defaults/main.yml @@ -12,6 +12,8 @@ get_all_release_versions: false images_to_get_hash_for: "{{ assisted_installer_images | combine(processed_release_images | default({})) }}" destination_hosts: + - bastion + - localhost - registry_host - assisted_installer diff --git a/roles/get_image_hash/tasks/main.yml b/roles/get_image_hash/tasks/main.yml index 46abe97b..d4a9a932 100644 --- a/roles/get_image_hash/tasks/main.yml +++ b/roles/get_image_hash/tasks/main.yml @@ -99,8 +99,9 @@ set_fact: assisted_installer_os_images: "{{ os_images }}" assisted_installer_release_images: "{{ release_images }}" - delegate_to: "assisted_installer" + delegate_to: "{{ item }}" delegate_facts: true + loop: "{{ destination_hosts }}" - name: "Set image hashes in {{ item }}" set_fact: diff --git a/roles/monitor_agent_based_installer/defaults/main.yml b/roles/monitor_agent_based_installer/defaults/main.yml new file mode 100644 index 00000000..8148a413 --- /dev/null +++ b/roles/monitor_agent_based_installer/defaults/main.yml @@ -0,0 +1,2 @@ +generated_dir: "{{ repo_root_path }}/generated" +manifests_dir: "{{ generated_dir}}/{{ cluster_name }}" diff --git a/roles/monitor_agent_based_installer/tasks/main.yml b/roles/monitor_agent_based_installer/tasks/main.yml new file mode 100644 index 00000000..0ed9b4d4 --- /dev/null +++ b/roles/monitor_agent_based_installer/tasks/main.yml @@ -0,0 +1,10 @@ +- name: Wait for bootstrap complete + ansible.builtin.shell: + cmd: "{{ agent_based_installer_path }} --log-level=debug agent wait-for bootstrap-complete" + chdir: "{{ manifests_dir }}" + ignore_errors: True # Timeout is fixed and some bare metal clusters complete bootstrap just after the timeout + +- name: Wait for install complete + ansible.builtin.shell: + cmd: "{{ agent_based_installer_path }} --log-level=debug agent wait-for install-complete" + chdir: "{{ manifests_dir }}" diff --git a/roles/populate_mirror_registry/tasks/main.yml b/roles/populate_mirror_registry/tasks/main.yml index 50f37c8c..a8c9fa9b 100644 --- a/roles/populate_mirror_registry/tasks/main.yml +++ b/roles/populate_mirror_registry/tasks/main.yml @@ -7,6 +7,7 @@ - import_tasks: prerequisites.yml tags: - populate_registry + - populate_registry_prerequisites - import_tasks: populate_registry.yml tags: diff --git a/roles/post_install/defaults/main.yml b/roles/post_install/defaults/main.yml index 20f2217c..49993c0c 100644 --- a/roles/post_install/defaults/main.yml +++ b/roles/post_install/defaults/main.yml @@ -1,9 +1,13 @@ secure: false -ASSISTED_INSTALLER_HOST: "{{ hostvars['assisted_installer']['host'] | default(ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0])) }}" +ASSISTED_INSTALLER_HOST: "{{ hostvars['assisted_installer']['host'] | default(ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0])) }}" ASSISTED_INSTALLER_PORT: "{{ hostvars['assisted_installer']['port'] | default(8090) }}" ASSISTED_INSTALLER_BASE_URL: "{{ secure | ternary('https', 'http') }}://{{ ASSISTED_INSTALLER_HOST }}:{{ ASSISTED_INSTALLER_PORT }}/api/assisted-install/v2" URL_ASSISTED_INSTALLER_CLUSTER: "{{ ASSISTED_INSTALLER_BASE_URL }}/clusters/{{ cluster_id }}" kube_filename: "{{ kubeconfig_dest_filename | default(cluster_name + '-kubeconfig') }}" dest_dir: "{{ kubeconfig_dest_dir | default(ansible_env.HOME) }}" kubeconfig_path: "{{ dest_dir }}/{{ kube_filename }}" -kubeadmin_vault_name: "{{ kubeadmin_dest_filename | default(cluster_name +'-kubeadmin.vault.yml') }}" +kubeadmin_vault_name: "{{ kubeadmin_dest_filename | default(cluster_name + '-kubeadmin.vault.yml') }}" +domain: "{{ cluster_name }}.{{ base_dns_domain }}" +generated_dir: "{{ repo_root_path }}/generated" +manifests_dir: "{{ generated_dir }}/{{ cluster_name }}" +auth_dir: "{{ manifests_dir }}/auth" diff --git a/roles/post_install/tasks/credentails_agent_based_installer.yml b/roles/post_install/tasks/credentails_agent_based_installer.yml new file mode 100644 index 00000000..d15ab854 --- /dev/null +++ b/roles/post_install/tasks/credentails_agent_based_installer.yml @@ -0,0 +1,11 @@ +- name: Fetch kubeadmin password + ansible.builtin.slurp: + src: "{{ auth_dir }}/kubeadmin-password" + register: kubeadmin_password + +- name: Construct kubeadmin credentials + ansible.builtin.set_fact: + credentials: + console_url: "https://console-openshift-console.apps.{{ domain }}" + password: "{{ kubeadmin_password.content | b64decode }}" + username: "kubeadmin" diff --git a/roles/post_install/tasks/credentails_assisted_installer.yml b/roles/post_install/tasks/credentails_assisted_installer.yml new file mode 100644 index 00000000..b7f769f7 --- /dev/null +++ b/roles/post_install/tasks/credentails_assisted_installer.yml @@ -0,0 +1,17 @@ +- name: Get credentials from assisted installer + ansible.builtin.uri: + url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}/credentials" + return_content: true + register: credentials_reply + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + +- name: Set credentials + ansible.builtin.set_fact: + credentials: "{{ credentials_reply.json }}" + +- name: Login to add token to kubeconfig + ansible.builtin.shell: + cmd: "oc login -u {{ credentials.username }} -p '{{ credentials.password }}'" + environment: + KUBECONFIG: "{{ kubeconfig_path }}" diff --git a/roles/post_install/tasks/fetch_kubeconfg_agent_based_installer.yml b/roles/post_install/tasks/fetch_kubeconfg_agent_based_installer.yml new file mode 100644 index 00000000..360b33d1 --- /dev/null +++ b/roles/post_install/tasks/fetch_kubeconfg_agent_based_installer.yml @@ -0,0 +1,5 @@ +- name: Download kubeconfig + ansible.builtin.copy: + src: "{{ auth_dir }}/kubeconfig" + dest: "{{ kubeconfig_path }}" + mode: 0664 diff --git a/roles/post_install/tasks/fetch_kubeconfg_assisted_installer.yml b/roles/post_install/tasks/fetch_kubeconfg_assisted_installer.yml new file mode 100644 index 00000000..96282033 --- /dev/null +++ b/roles/post_install/tasks/fetch_kubeconfg_assisted_installer.yml @@ -0,0 +1,5 @@ +- name: Download kubeconfig + ansible.builtin.get_url: + url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}/downloads/credentials?file_name=kubeconfig" + dest: "{{ kubeconfig_path }}" + mode: 0664 diff --git a/roles/post_install/tasks/main.yml b/roles/post_install/tasks/main.yml index 9e28c340..8d9072a2 100644 --- a/roles/post_install/tasks/main.yml +++ b/roles/post_install/tasks/main.yml @@ -1,17 +1,21 @@ --- -- name: Download kubeconfig - get_url: - url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}/downloads/credentials?file_name=kubeconfig" - dest: "{{ kubeconfig_path }}" - mode: 0664 +- name: Fetch kubeconfig for assisted installer + ansible.builtin.include_tasks: + file: fetch_kubeconfg_assisted_installer.yml + when: not ((use_agent_based_installer | default(false)) | bool) + +- name: Fetch kubeconfig for agent based installer + ansible.builtin.include_tasks: + file: fetch_kubeconfg_agent_based_installer.yml + when: (use_agent_based_installer | default(false)) | bool - name: Perform basic checks and login environment: KUBECONFIG: "{{ kubeconfig_path }}" block: # The retries is becuase sometimes (although rarely) the cluster isn't quite ready for requests - - name: Perform simple connectivity check with oc (wait up to 5 mins) - shell: + - name: Perform simple connectivity check with oc (wait up to 5 mins) + ansible.builtin.shell: cmd: "oc get clusterversion" register: res until: res.rc == 0 @@ -21,36 +25,36 @@ - name: Check status of cluster operators block: - name: Wait up to 20 mins for cluster to become functional - shell: + ansible.builtin.shell: cmd: oc wait clusteroperators --all --for=condition=Available --timeout=20m rescue: - name: Get better info for failure message - shell: oc get clusteroperators + ansible.builtin.shell: oc get clusteroperators register: co_result - name: Present failing clusteroperators command output - fail: + ansible.builtin.fail: msg: | Cluster has not come up correctly: {{ co_result.stdout }} - - name: Get credentials - uri: - url: "{{ URL_ASSISTED_INSTALLER_CLUSTER }}/credentials" - return_content: yes - register: credentials +- name: Fetch credentials for assisted installer + ansible.builtin.include_tasks: + file: credentails_assisted_installer.yml + when: not ((use_agent_based_installer | default(false)) | bool) - - name: Login to add token to kubeconfig - shell: - cmd: "oc login -u {{ credentials.json.username }} -p '{{ credentials.json.password }}'" +- name: Fetch credentials for agent based installer + ansible.builtin.include_tasks: + file: credentails_agent_based_installer.yml + when: (use_agent_based_installer | default(false)) | bool - name: Save credentials to file - copy: - content: "{{ credentials.json | to_nice_json }}" + ansible.builtin.copy: + content: "{{ credentials | to_nice_json }}" dest: "{{ dest_dir }}/{{ kubeadmin_vault_name }}" mode: 0600 - name: Save credentials to vault - shell: + ansible.builtin.shell: cmd: "ansible-vault encrypt --vault-password-file {{ kubeadmin_vault_password_file_path }} {{ dest_dir }}/{{ kubeadmin_vault_name }}" when: (kubeadmin_vault_password_file_path is defined) and (kubeadmin_vault_password_file_path is file) diff --git a/roles/validate_inventory/tasks/agent_based_installer_feature_gates.yml b/roles/validate_inventory/tasks/agent_based_installer_feature_gates.yml new file mode 100644 index 00000000..d81eeaf1 --- /dev/null +++ b/roles/validate_inventory/tasks/agent_based_installer_feature_gates.yml @@ -0,0 +1,26 @@ +- name: Assert no day2 nodes + ansible.builtin.assert: + that: + - (groups['day2_workers'] | default([])) | length == 0 + fail_msg: "use_agent_based_installer does not support day2 workers" + +- name: Assert no partitions + ansible.builtin.assert: + that: + - hostvars[item]['disks'] is not defined + fail_msg: "Can not use partitions when using use_agent_based_installer" + loop: "{{ groups['nodes'] | default([]) }}" + +- name: Assert discovery is password hash is not defined + ansible.builtin.assert: + that: + - hashed_discovery_password is not defined + fail_msg: "hashed_discovery_password is not supported when use_agent_based_installer" + +- name: Assert no OLM Operators + ansible.builtin.assert: + that: + - install_lso is not defined + - install_odf is not defined + - install_cnv is not defined + fail_msg: "Operator installion is not supported when use_agent_based_installer, you can use extra_manifests to deploy these operators" diff --git a/roles/validate_inventory/tasks/agent_based_installer_requirements.yml b/roles/validate_inventory/tasks/agent_based_installer_requirements.yml new file mode 100644 index 00000000..65784008 --- /dev/null +++ b/roles/validate_inventory/tasks/agent_based_installer_requirements.yml @@ -0,0 +1,12 @@ +- name: Assert agent_based_installer_path is defined + ansible.builtin.assert: + that: + - openshift_full_version is version('4.11.0', '>=') + fail_msg: Agent based installation is only avaiable for openshift_full_version >= 4.11.0 + +- name: Assert nodes have network_config + ansible.builtin.assert: + that: + - hostvars[item]['network_config'] is defined + fail_msg: "use_agent_based_installer requires that hosts have a network config" + loop: "{{ groups['nodes'] | default([]) }}" diff --git a/roles/validate_inventory/tasks/main.yml b/roles/validate_inventory/tasks/main.yml index e8fbcfb8..9aa47596 100644 --- a/roles/validate_inventory/tasks/main.yml +++ b/roles/validate_inventory/tasks/main.yml @@ -74,6 +74,23 @@ tags: validate_pxe when: (setup_pxe_service | default(false)) | bool + - include_tasks: + file: agent_based_installer_feature_gates.yml + apply: + tags: validate_agent_based_installer + when: + - (use_agent_based_installer | default(false) | bool) + - not ((ignore_agent_based_installer_feature_gates | default(false)) | bool) + tags: validate_agent_based_installer + + - include_tasks: + file: agent_based_installer_requirements.yml + apply: + tags: validate_agent_based_installer + when: + - (use_agent_based_installer | default(false) | bool) + tags: validate_agent_based_installer + when: not (inventory_validated | default(False) | bool) delegate_to: "{{ validation_host | default('bastion') }}" diff --git a/site.yml b/site.yml index fe1098d1..45fa8a4a 100644 --- a/site.yml +++ b/site.yml @@ -8,3 +8,4 @@ - import_playbook: post_install.yml - import_playbook: deploy_day2_workers.yml + when: not ((use_agent_based_installer | default(false)) | bool)