Merge pull request #1087 from RedHatInsights/psav/upgrade-go-version

Upgrade go version

.github/workflows/lint.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/setup-go@v4
         name: Set up Go 1.x
         with:
-          go-version: "1.21.9"
+          go-version: "1.22.7"
         env:
           GO111MODULE: off
 

.github/workflows/package.yml

@@ -14,9 +14,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/setup-go@v4
-        name: Set up golang 1.21
+        name: Set up golang 1.22.7
         with:
-          go-version: '1.21.9'
+          go-version: '1.22.7'
       - name: Check out source code
         uses: actions/checkout@v2
       - name: Run Tests

.github/workflows/platsec.yml

@@ -34,11 +34,3 @@ jobs:
     ## the defaults of root '.' for the path and 'Dockerfile' for the Dockerfile name.
     ## Additionally, if you have a Dockerfile you use as your BASE_IMG or you need to
     ## use '--build-arg', those can be define below as well.
-
-    with:
-    #   dockerfile_path: './test'
-    #   dockerfile_name: 'Dockerfile.main'
-      base_image_build: true
-    #   base_dockerfile_path: './test'
-      base_dockerfile_name: 'Dockerfile.base'
-      build_arg: '--build-arg BASE_IMAGE="localbuild/baseimage:latest"'

.gitignore

@@ -57,3 +57,9 @@ build/.build_venv
 controllers/cloud.redhat.com/version.txt
 
 tmp/*
+
+deploy.yml.old
+deploy-mutate.yml.old
+deploy-kustomize.yaml
+
+minikube/*

.tekton/clowder-pull-request.yaml

@@ -33,7 +33,7 @@ spec:
   - name: path-context
     value: .
   - name: prefetch-input
-    value: 
+    value:
     - {"type": "gomod", "path": "."}
     - {"type": "gomod", "path": "./deps/controller-gen"}
     - {"type": "gomod", "path": "./deps/kustomize"}
@@ -267,10 +267,10 @@ spec:
             args:
               - use
               - $(params.SOURCE_ARTIFACT)=/var/workdir/source
-          - name: kustomize-build
-            image: registry.access.redhat.com/ubi8/ubi:latest
+          - name: unit-tests
+            image: registry.access.redhat.com/ubi8/go-toolset:1.22.7-5.1733303300
             workingDir: /var/workdir/source
-            script: set -xe && dnf install -y go make && make test
+            script: set -xe && git config --global --add safe.directory /var/workdir/source && ls -l && pwd && make -dn test
         volumes:
           # New volume to store a copy of the source code accessible only to this Task.
           - name: workdir
@@ -298,8 +298,26 @@ spec:
             args:
               - use
               - $(params.SOURCE_ARTIFACT)=/var/workdir/source
-          - name: kustomize-build
-            image: registry.access.redhat.com/ubi8/ubi:latest
+          - name: make-template
+            image: registry.access.redhat.com/ubi8/go-toolset:1.22.7-5.1733303300
+            workingDir: /opt/app-root/src
+            env:
+              - name: LOCALBIN
+                value: /tmp/bin
+              - name: IMG
+                value: $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST)
+            script: |
+              set -xe && cp /var/workdir/source/* /opt/app-root/src -Rv && make release && make build-template-kustomize && cp manifest.yaml /var/workdir/ && cp deploy-kustomize.yaml /var/workdir/
+          - name: test-template
+            image: registry.access.redhat.com/ubi8/ubi:8.10-1161
+            workingDir: /opt/app-root/src
+            env:
+              - name: LOCALBIN
+                value: /tmp/bin
+            script: |
+              set -xe && cp /var/workdir/source/* /opt/app-root/src -Rv && cp /var/workdir/manifest.yaml . && cp /var/workdir/deploy-kustomize.yaml . && dnf install make python3.12 diffutils -y && make build-template && make test-template
+          - name: e2e-tests
+            image: registry.access.redhat.com/ubi8/ubi:8.10-1161
             workingDir: /var/workdir/source
             env:
               - name: MINIKUBE_HOST
@@ -311,7 +329,7 @@ spec:
                 valueFrom:
                   secretKeyRef:
                     name: minikube-ssh-key
-                    key: MINIKUBE_USER 
+                    key: MINIKUBE_USER
               - name: MINIKUBE_SSH_KEY
                 valueFrom:
                   secretKeyRef:
@@ -322,7 +340,8 @@ spec:
                   secretKeyRef:
                     name: minikube-ssh-key
                     key: MINIKUBE_ROOTDIR
-            script: set -e && dnf install -y git go jq make openssh-clients python3 && ./ci/konflux_minikube_e2e_tests.sh
+            # FIXME: this should be handled from the script itself
+            script: set -ex && dnf install -y git openssh-clients python3.12 make jq diffutils && ci/konflux_minikube_e2e_tests.sh
         volumes:
           # New volume to store a copy of the source code accessible only to this Task.
           - name: workdir

.tekton/clowder-push.yaml

@@ -17,6 +17,8 @@ metadata:
   namespace: hcm-eng-prod-tenant
 spec:
   params:
+  - name: hermetic
+    value: "true"
   - name: git-url
     value: '{{source_url}}'
   - name: revision
@@ -27,6 +29,11 @@ spec:
     value: Dockerfile
   - name: path-context
     value: .
+  - name: prefetch-input
+    value:
+    - {"type": "gomod", "path": "."}
+    - {"type": "gomod", "path": "./deps/controller-gen"}
+    - {"type": "gomod", "path": "./deps/kustomize"}
   pipelineSpec:
     description: |
       This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization.
@@ -257,10 +264,10 @@ spec:
             args:
               - use
               - $(params.SOURCE_ARTIFACT)=/var/workdir/source
-          - name: kustomize-build
-            image: registry.access.redhat.com/ubi8/ubi:latest
+          - name: unit-tests
+            image: registry.access.redhat.com/ubi8/go-toolset:1.22.7-5.1733303300
             workingDir: /var/workdir/source
-            script: set -xe && dnf install -y go make && make test
+            script: set -xe && git config --global --add safe.directory /var/workdir/source && ls -l && pwd && make -dn test
         volumes:
           # New volume to store a copy of the source code accessible only to this Task.
           - name: workdir
@@ -288,8 +295,26 @@ spec:
             args:
               - use
               - $(params.SOURCE_ARTIFACT)=/var/workdir/source
-          - name: kustomize-build
-            image: registry.access.redhat.com/ubi8/ubi:latest
+          - name: make-template
+            image: registry.access.redhat.com/ubi8/go-toolset:1.22.7-5.1733303300
+            workingDir: /opt/app-root/src
+            env:
+              - name: LOCALBIN
+                value: /tmp/bin
+              - name: IMG
+                value: $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST)
+            script: |
+              set -xe && cp /var/workdir/source/* /opt/app-root/src -Rv && make release && make build-template-kustomize && cp manifest.yaml /var/workdir/ && cp deploy-kustomize.yaml /var/workdir/
+          - name: test-template
+            image: registry.access.redhat.com/ubi8/ubi:8.10-1161
+            workingDir: /opt/app-root/src
+            env:
+              - name: LOCALBIN
+                value: /tmp/bin
+            script: |
+              set -xe && cp /var/workdir/source/* /opt/app-root/src -Rv && cp /var/workdir/manifest.yaml . && cp /var/workdir/deploy-kustomize.yaml . && dnf install make python3.12 diffutils -y && make build-template && make test-template
+          - name: e2e-tests
+            image: registry.access.redhat.com/ubi8/ubi:8.10-1161
             workingDir: /var/workdir/source
             env:
               - name: MINIKUBE_HOST
@@ -301,7 +326,7 @@ spec:
                 valueFrom:
                   secretKeyRef:
                     name: minikube-ssh-key
-                    key: MINIKUBE_USER 
+                    key: MINIKUBE_USER
               - name: MINIKUBE_SSH_KEY
                 valueFrom:
                   secretKeyRef:
@@ -312,7 +337,8 @@ spec:
                   secretKeyRef:
                     name: minikube-ssh-key
                     key: MINIKUBE_ROOTDIR
-            script: set -e && dnf install -y git go jq make openssh-clients python3 && ./ci/konflux_minikube_e2e_tests.sh
+            # FIXME: this should be handled from the script itself
+            script: set -ex && dnf install -y git openssh-clients python3.12 make jq diffutils && ci/konflux_minikube_e2e_tests.sh
         volumes:
           # New volume to store a copy of the source code accessible only to this Task.
           - name: workdir
@@ -557,4 +583,4 @@ spec:
   - name: git-auth
     secret:
       secretName: '{{ git_auth_secret }}'
-status: {}
+status: {}

Dockerfile

@@ -30,7 +30,7 @@ RUN CGO_ENABLED=1 GOOS=linux GO111MODULE=on go build -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1130
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1154
 WORKDIR /
 COPY --from=builder /workspace/manager .
 COPY --from=builder /workspace/manifest.yaml .

Makefile

@@ -2,6 +2,8 @@ CLOWDER_BUILD_TAG ?= $(shell git rev-parse HEAD)
 
 GO_CMD ?= go
 
+TEMPLATE_KUSTOMIZE ?= "deploy-kustomize.yaml"
+
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.28
 
@@ -70,9 +72,18 @@ api-docs:
 	./build/build_api_docs.sh
 	./build/build_config_docs.sh
 
-build-template: manifests kustomize controller-gen
-	$(KUSTOMIZE) build config/deployment-template | ./manifest2template.py --config config/deployment-template/clowder_config.yaml --mutate > deploy-mutate.yml
-	$(KUSTOMIZE) build config/deployment-template | ./manifest2template.py --config config/deployment-template/clowder_config.yaml > deploy.yml
+build-template: 
+	@echo "Checking for $(TEMPLATE_KUSTOMIZE)"
+	@if [ ! -f $(TEMPLATE_KUSTOMIZE) ]; then \
+		$(MAKE) build-template-kustomize; \
+	fi
+	TEMPLATE_KUSTOMIZE=$(TEMPLATE_KUSTOMIZE) ./build/build_template.sh
+
+build-template-kustomize: manifests kustomize controller-gen
+	$(KUSTOMIZE) build config/deployment-template > $(TEMPLATE_KUSTOMIZE)
+
+test-template: build-template
+	source ./build/template_check.sh
 
 release: manifests kustomize controller-gen
 	echo "---" > manifest.yaml
@@ -83,10 +94,10 @@ release: manifests kustomize controller-gen
 	$(KUSTOMIZE) build config/release-manifest >> manifest.yaml
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./controllers/..." output:crd:artifacts:config=config/crd/bases
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
-	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./controllers/..."
 
 fmt: ## Run go fmt against code.
 	$(GO_CMD) fmt ./...
@@ -172,28 +183,36 @@ update-version: ## Updates the version in the image
 	$(shell echo -n $(CLOWDER_VERSION) > controllers/cloud.redhat.com/version.txt)
 	echo "Building version: $(CLOWDER_VERSION)"
 
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
-
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
-
-ENVTEST = $(shell pwd)/bin/setup-envtest
-envtest: ## Download envtest-setup locally if necessary.
-	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@d0396a3d6f9fb554ef2da382a3d0bf05f7565e65)
-
-# go-install-tool will 'go install' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-install-tool
-@[ -f $(1) ] || { \
-set -e ;\
-TMP_DIR=$$(mktemp -d) ;\
-cd $$TMP_DIR ;\
-$(GO_CMD) mod init tmp ;\
-echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin $(GO_CMD) install $(2) ;\
-rm -rf $$TMP_DIR ;\
-}
-endef
+##@ Build Dependencies
+
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+KUSTOMIZE ?= $(LOCALBIN)/kustomize
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+ENVTEST ?= $(LOCALBIN)/setup-envtest
+
+## Tool Versions
+KUSTOMIZE_VERSION ?= v5.5.0
+CONTROLLER_TOOLS_VERSION ?= v0.16.4
+
+update-deps:
+	KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION) CONTROLLER_TOOLS_VERSION=$(CONTROLLER_TOOLS_VERSION) ./deps/update_e2e_deps.sh
+
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
+$(CONTROLLER_GEN): $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
+
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
+$(KUSTOMIZE): $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install sigs.k8s.io/kustomize/kustomize/v5@$(KUSTOMIZE_VERSION)
+
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest

build/build_config_docs.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-python -m venv docs/build/venv
+python3.12 -m venv docs/build/venv
 source docs/build/venv/bin/activate
-pip install json-schema-for-humans==0.47
+pip install json-schema-for-humans==v1.0.2
 generate-schema-doc --config with_footer=false --config template_name=md controllers/cloud.redhat.com/config/schema.json docs/api_ref.md

build/build_template.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+python3 -m venv "build/.build_venv"
+source build/.build_venv/bin/activate
+pip install pyyaml
+
+if [ -f "deploy-mutate.yml" ]; then
+  mv deploy-mutate.yml deploy-mutate.yml.old
+fi
+
+if [ -f "deploy.yml" ]; then
+  mv deploy.yml deploy.yml.old
+fi
+cat $TEMPLATE_KUSTOMIZE | ./manifest2template.py --config config/deployment-template/clowder_config.yaml --mutate > deploy-mutate.yml
+cat $TEMPLATE_KUSTOMIZE | ./manifest2template.py --config config/deployment-template/clowder_config.yaml > deploy.yml
+deactivate