How do I modify Harvest templates included from Amazon's cloud formation template

[cgrinds]

These instructions work with containers installed by Monitoring FSx for ONTAP file systems using Harvest and Grafana.

SSH

After deploying the CloudFormation stack, ssh into the Amazon EC2 instance that your Harvest and Grafana clients are running on and switch to the directory that contains your manage_harvest.yml Ansible file.

Copy conf

Verify that the docker containers were successfully started using the following command.

sudo docker ps

You should see something like this - we are interested in the Harvest pollers which have names like harvest_cluster-1

CONTAINER ID   IMAGE                           COMMAND                  CREATED       STATUS       PORTS                    NAMES
1d5c7db1e388   ghcr.io/netapp/harvest:latest   "bin/poller --config…"   4 hours ago   Up 4 hours                            harvest_cluster-1
a58137eb7e77   prom/alertmanager               "/bin/alertmanager -…"   4 hours ago   Up 4 hours   0.0.0.0:9093->9093/tcp   harvest_prometheus_alertmanager
9527680aa655   prom/prometheus:v2.33.1         "/bin/prometheus --c…"   4 hours ago   Up 4 hours   0.0.0.0:9090->9090/tcp   harvest_prometheus
2378e5098d5e   grafana/grafana:8.3.4           "/run.sh"                4 hours ago   Up 4 hours   0.0.0.0:3000->3000/tcp   harvest_grafana

We're going to copy the conf directory from the harvest_cluster-1 container to the local directory.

Replace harvest_cluster-1 with the name of one of your Harvest container's included in the docker ps command above.

docker cp harvest_cluster-1:/opt/harvest/conf .

Make edits you want to any of the templates included in the now-local conf directory.

Modify volume mounts

We need to edit the manage_harvest.yml and add a volume mount to the Harvest containers so they will use our local templates.

vim manage_harvest.yml

At line 199, under the volumes: section add
- ./conf:/opt/harvest/conf

At line 94 add, under the volumes: section add
- ./conf:/opt/harvest/conf

manage_harvest.yml with edits above made

---
- hosts: localhost
  name: Manage Harvest
  vars:
    default_harvest_image: ghcr.io/netapp/harvest:latest
    default_prom_image: prom/prometheus:v2.33.1
    default_grafana_image: grafana/grafana:8.3.4
    default_alertmanger_image: prom/alertmanager
  vars_files:
    harvest/harvest.yml
  collections:
  - ansible.builtin
  - community.docker
  tasks:
  - name: Verify images
    docker_image:
      name: "{{ item }}"
      source: pull
    loop:
    - "{{ default_harvest_image }}"
    - "{{ default_prom_image }}"
    - "{{ default_grafana_image }}"
    - "{{ default_alertmanger_image }}"
    tags: setup

  - name: Configure/Verify Prometheus targets
    lineinfile:
      path: ./prometheus/prometheus.yml
      line: "  - targets: ['harvest_{{ item }}:{{Pollers[item].ansible_port}}']"
    loop: "{{ Pollers|list }}"

  - name: Create Harvest Network
    docker_network:
      name: harvest
    tags: setup

  - name: Create Prometheus storage
    docker_volume:
      name: prometheus_data

  - name: Setup/Verify Prometheus
    docker_container:
      name: harvest_prometheus
      image: "{{ default_prom_image }}"
      restart: true
      restart_policy: unless-stopped
      volumes:
      - prometheus_data:/prometheus
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/alert.rules.yml:/etc/prometheus/alert.rules.yml
      command:
      - '--config.file=/etc/prometheus/prometheus.yml' 
      ports: "9090:9090"
      networks: 
      - name: harvest
    tags: setup

  - name: Setup/Verify Prometheus AlertManager
    docker_container:
      name: harvest_prometheus_alertmanager
      image: prom/alertmanager
      restart: true
      restart_policy: unless-stopped
      volumes:
      - prometheus_data:/prometheus
      - ./prometheus_alertmanager/alertmanager.yml:/etc/prometheus/alertmanager.yml
      command:
      - '--config.file=/etc/prometheus/alertmanager.yml' 
      ports: "9093:9093"
      networks: 
      - name: harvest
    tags: setup

  - name: Setup/verify Grafana
    docker_container:
      name: harvest_grafana
      image: "{{ default_grafana_image }}"
      restart_policy: unless-stopped
      env:
        GF_SECURITY_ADMIN_PASSWORD=pass
      ports:
      - "3000:3000"      
      networks:
      - name: harvest
    tags: setup

  - name: Setup/verify Harvest pollers
    docker_container:
      name: "harvest_{{ item }}"
      image: "{{ default_harvest_image }}"
      restart_policy: unless-stopped
      volumes:
      - ./harvest/:/etc/harvest/
      - ./conf:/opt/harvest/conf
      command:
      - --config
      - /etc/harvest/harvest.yml
      - -p
      - "{{ item }}"
      - --promPort
      - "{{ Pollers[item].ansible_port }}"
      networks:
      - name: harvest
      detach: true
    loop: "{{ Pollers|list}}" 
    tags: setup

  - name: Create bearer token
    uri:
      url: "http://{{ansible_default_ipv4.address}}:3000/api/auth/keys"
      force_basic_auth: true
      method: POST
      user: admin
      password: pass
      headers:
        Content-Type: application/json
      body: '{"name": "apikeycurl", "role": "Admin"}'
    register: login 
    tags: never, api

  - name: Creating Prometheus datasource in Grafana
    uri:
      url: "http://{{ansible_default_ipv4.address}}:3000/api/datasources"
      method: POST
      headers:
        Content-Type: application/json
        Authorization: "Bearer {{ login.json.key }}"
      body: '{"name": "Prometheus", "type": "prometheus", "access": "proxy", "url": "http://harvest_prometheus:9090", "isDefault": true, "basicAuth": false}'
    tags: never, api     

  - name: Prep Grafana dashboard load
    lineinfile:
      path: ./harvest/harvest.yml
      insertafter: '^Tools:'
      regexp: '  grafana_api_token:'
      line: '  grafana_api_token: {{ login.json.key }}'
    tags: never, api     

  - name: Add Dashboards to Grafana
    docker_container_exec:
      container: "harvest_{{ Pollers|first }}"
      command: sh -c "bin/harvest grafana import -a harvest_grafana:3000 --config /etc/harvest/harvest.yml"
    tags: never, api, dashboard

  - name: Stop Prometheus
    docker_container:
      name: harvest_prometheus
      image: prom/prometheus
      state: stopped
    tags: never, stop

  - name: Stop Grafana
    docker_container:
      name: harvest_grafana
      image: "{{ default_grafana_image }}"
      state: stopped
    tags: never, stop

  - name: Stop Harvest pollers
    docker_container:
      name: "harvest_{{ item }}"
      image: "{{ default_harvest_image }}"
      state: stopped
      detach: true
    loop: "{{ Pollers|list}}" 
    tags: never, stop

  - name: Remove Prometheus
    docker_container:
      name: harvest_prometheus
      state: absent
    tags: never, del

  - name: Remove Grafana
    docker_container:
      name: harvest_grafana
      state: absent
    tags: never, del

  - name: Remove Harvest pollers
    docker_container:
      name: "harvest_{{ item }}"
      state: absent
    loop: "{{ Pollers|list}}" 
    tags: never, del, upgrade

  - name: update images
    docker_image:
      name: "{{ default_harvest_image }}"
      force_source: true
      source: pull
    tags: never, upgrade

  - name: Update Harvest pollers
    docker_container:
      name: "harvest_{{ item }}"
      image: "{{ default_harvest_image }}"
      restart: true
      volumes:
      - ./harvest/:/etc/harvest/
      - ./conf:/opt/harvest/conf
      command:
      - --config
      - /etc/harvest/harvest.yml
      - -p
      - "{{ item }}"
      - --promPort
      - "{{ Pollers[item].ansible_port }}"
      networks:
      - name: harvest
      detach: true
    loop: "{{ Pollers|list}}" 
    tags: never, upgrade   

Restart containers

The containers need to be restarted to use the template changes.

ansible-playbook manage_harvest.yml --tags upgrade