Merge pull request #214 from borod108/monitoring/add_rejectvm_metrics

Add total_rejected_vms matrics

internal/template-validator/webhooks/hook.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 
 	"github.com/davecgh/go-spew/spew"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	admissionv1 "k8s.io/api/admission/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"kubevirt.io/client-go/log"
@@ -35,6 +37,13 @@ import (
 	"kubevirt.io/ssp-operator/internal/template-validator/virtinformers"
 )
 
+var (
+	vmsRejected = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "total_rejected_vms",
+		Help: "The total number of rejected vms",
+	})
+)
+
 const (
 	VmValidatePath       string = "/virtualmachine-validate"
 	TemplateValidatePath string = "/template-validate"
@@ -85,6 +94,7 @@ func (w *webhooks) admitVm(ar *admissionv1.AdmissionReview) *admissionv1.Admissi
 
 	causes := ValidateVm(rules, vm)
 	if len(causes) > 0 {
+		vmsRejected.Inc()
 		return ToAdmissionResponse(causes)
 	}
 

tests/metrics_test_utils.go

@@ -0,0 +1,45 @@
+package tests
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"regexp"
+	"strconv"
+
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+)
+
+var regexpForMetrics = map[string]*regexp.Regexp{
+	"total_rejected_vms": regexp.MustCompile(`total_rejected_vms ([0-9]+)`),
+}
+
+func intMetricValue(metricName string, metricsPort uint16, pod *v1.Pod) int {
+	conn, err := portForwarder.Connect(pod, metricsPort)
+	Expect(err).ToNot(HaveOccurred())
+	defer conn.Close()
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return conn, nil
+			},
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+	resp, err := client.Get(fmt.Sprintf("https://localhost:%d/metrics", metricsPort))
+	Expect(err).ToNot(HaveOccurred(), "Can't get metrics from %s", pod.Name)
+	defer resp.Body.Close()
+	body, _ := ioutil.ReadAll(resp.Body)
+	regex, ok := regexpForMetrics[metricName]
+	if !ok {
+		panic(fmt.Sprintf("metricName %s does not have a defined regexp string, please add one to the regexpForMetrics map", metricName))
+	}
+	valueOfMetric := regex.FindSubmatch(body)
+	intValue, err := strconv.Atoi(string(valueOfMetric[1]))
+	Expect(err).ToNot(HaveOccurred())
+	return intValue
+}

tests/validator_test.go

@@ -4,13 +4,14 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"reflect"
+	"strings"
+	"time"
+
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
-	"reflect"
-	"strings"
-	"time"
 
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/extensions/table"
@@ -415,7 +416,20 @@ var _ = Describe("Template validator webhooks", func() {
 		It("[test_id:2960] Negative test - Create a VM with machine type violation", func() {
 			template = TemplateWithRules()
 			Expect(apiClient.Create(ctx, template)).ToNot(HaveOccurred(), "Failed to create template: %s", template.Name)
+			// set value unfulfilling validation
+			vmi = addDomainResourcesToVMI(vmi, 2, "test", "128M")
+			vm = NewVirtualMachine(vmi)
+			vm.ObjectMeta.Annotations = map[string]string{
+				TemplateNameAnnotation:      template.Name,
+				TemplateNamespaceAnnotation: template.Namespace,
+			}
+			eventuallyFailToCreateVm(vm)
+		})
 
+		It("[test_id:7060]vm rejected metrics increses by one when vm is rejected", func() {
+			rejectedCountBefore := totalRejectedVmsMetricsValue()
+			template = TemplateWithRules()
+			Expect(apiClient.Create(ctx, template)).ToNot(HaveOccurred(), "Failed to create template: %s", template.Name)
 			// set value unfulfilling validation
 			vmi = addDomainResourcesToVMI(vmi, 2, "test", "128M")
 			vm = NewVirtualMachine(vmi)
@@ -424,7 +438,9 @@ var _ = Describe("Template validator webhooks", func() {
 				TemplateNamespaceAnnotation: template.Namespace,
 			}
 			eventuallyFailToCreateVm(vm)
+			Expect(totalRejectedVmsMetricsValue() - rejectedCountBefore).To(Equal(1))
 		})
+
 		It("[test_id:5586]test with template optional rules unfulfilled", func() {
 			template = TemplateWithRulesOptional()
 			Expect(apiClient.Create(ctx, template)).ToNot(HaveOccurred(), "Failed to create template: %s", template.Name)
@@ -856,6 +872,16 @@ func eventuallyFailToCreateVm(vm *kubevirtv1.VirtualMachine) bool {
 	}, shortTimeout).Should(Equal(metav1.StatusReasonInvalid), "Should have given the invalid error")
 }
 
+func totalRejectedVmsMetricsValue() (sum int) {
+	pods, err := GetRunningPodsByLabel(validator.VirtTemplateValidator, validator.KubevirtIo, strategy.GetNamespace())
+	Expect(err).ToNot(HaveOccurred(), "Could not find the validator pods")
+	Expect(pods.Items).ToNot(BeEmpty())
+	for _, validatorPod := range pods.Items {
+		sum += intMetricValue("total_rejected_vms", validator.MetricsPort, &validatorPod)
+	}
+	return
+}
+
 func addObjectsToTemplates(genName, validation string) *templatev1.Template {
 	editable := `/objects[0].spec.template.spec.domain.cpu.sockets
 				/objects[0].spec.template.spec.domain.cpu.cores