Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow wildcard value for namespaces in Alert eventSources #504

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docs/spec/v1beta2/alerts.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,19 @@ eventSources:
namespace: apps
```

#### Select objects from all namespaces

The `*` wildcard can be used to select events issued by Flux objects from any `namespace`:

```yaml
eventSources:
- kind: HelmRelease
name: 'service1'
namespace: '*'
```

It requires to have cross-namespace references enabled for the controller.

#### Select objects by label

To select events issued by all Flux objects of a particular `kind` with specific `labels`:
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
github.com/ktrysmt/go-bitbucket v0.9.55
github.com/microsoft/azure-devops-go-api/azuredevops/v6 v6.0.1
github.com/onsi/gomega v1.27.2
github.com/prometheus/client_golang v1.14.0
github.com/sethvargo/go-limiter v0.7.2
github.com/slok/go-http-metrics v0.10.0
github.com/spf13/pflag v1.0.5
Expand Down Expand Up @@ -110,7 +111,6 @@ require (
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
Expand Down
283 changes: 282 additions & 1 deletion internal/controllers/alert_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

"github.com/fluxcd/pkg/ssa"
. "github.com/onsi/gomega"
"github.com/prometheus/client_golang/prometheus"
"github.com/sethvargo/go-limiter/memorystore"
prommetrics "github.com/slok/go-http-metrics/metrics/prometheus"
"github.com/slok/go-http-metrics/middleware"
Expand Down Expand Up @@ -169,9 +170,12 @@ func TestAlertReconciler_EventHandler(t *testing.T) {
)
g.Expect(createNamespace(namespace)).NotTo(HaveOccurred(), "failed to create test namespace")

reg := prometheus.NewRegistry()

eventMdlw := middleware.New(middleware.Config{
Recorder: prommetrics.NewRecorder(prommetrics.Config{
Prefix: "gotk_event",
Prefix: "gotk_event",
Registry: reg,
}),
})

Expand Down Expand Up @@ -425,3 +429,280 @@ func TestAlertReconciler_EventHandler(t *testing.T) {
})
}
}

func TestAlertReconciler_EventHandler_CrossNamespaceRefs(t *testing.T) {
g := NewWithT(t)
var (
namespace = "events-" + randStringRunes(5)
req *http.Request
provider *apiv1beta2.Provider
)
g.Expect(createNamespace(namespace)).NotTo(HaveOccurred(), "failed to create test namespace")

reg := prometheus.NewRegistry()

eventMdlw := middleware.New(middleware.Config{
Recorder: prommetrics.NewRecorder(prommetrics.Config{
Prefix: "gotk_event",
Registry: reg,
}),
})

store, err := memorystore.New(&memorystore.Config{
Interval: 5 * time.Minute,
})
if err != nil {
t.Fatalf("failed to create memory storage")
}

eventServer := server.NewEventServer("127.0.0.1:56789", logf.Log, k8sClient, false)
stopCh := make(chan struct{})
go eventServer.ListenAndServe(stopCh, eventMdlw, store)

rcvServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
req = r
w.WriteHeader(200)
}))
defer rcvServer.Close()
defer close(stopCh)

providerKey := types.NamespacedName{
Name: fmt.Sprintf("provider-%s", randStringRunes(5)),
Namespace: namespace,
}
provider = &apiv1beta2.Provider{
ObjectMeta: metav1.ObjectMeta{
Name: providerKey.Name,
Namespace: providerKey.Namespace,
},
Spec: apiv1beta2.ProviderSpec{
Type: "generic",
Address: rcvServer.URL,
},
}
g.Expect(k8sClient.Create(context.Background(), provider)).To(Succeed())
g.Eventually(func() bool {
var obj apiv1beta2.Provider
g.Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(provider), &obj))
return conditions.IsReady(&obj)
}, 30*time.Second, time.Second).Should(BeTrue())

repo, err := readManifest("./testdata/repo.yaml", namespace)
g.Expect(err).ToNot(HaveOccurred())

secondRepo, err := readManifest("./testdata/gitrepo2.yaml", namespace)
g.Expect(err).ToNot(HaveOccurred())

_, err = manager.Apply(context.Background(), repo, ssa.ApplyOptions{
Force: true,
})
g.Expect(err).ToNot(HaveOccurred())

_, err = manager.Apply(context.Background(), secondRepo, ssa.ApplyOptions{
Force: true,
})
g.Expect(err).ToNot(HaveOccurred())

alertKey := types.NamespacedName{
Name: fmt.Sprintf("alert-%s", randStringRunes(5)),
Namespace: namespace,
}

alert := &apiv1beta2.Alert{
ObjectMeta: metav1.ObjectMeta{
Name: alertKey.Name,
Namespace: alertKey.Namespace,
},
Spec: apiv1beta2.AlertSpec{
ProviderRef: meta.LocalObjectReference{
Name: providerKey.Name,
},
EventSeverity: "info",
EventSources: []apiv1.CrossNamespaceObjectReference{
{
Kind: "Bucket",
Name: "hyacinth",
Namespace: namespace,
},
{
Kind: "Bucket",
Name: "daffodil",
Namespace: "test",
},
{
Kind: "Bucket",
Name: "bluebells",
Namespace: "*",
},
{
Kind: "Kustomization",
Name: "*",
Namespace: namespace,
},
{
Kind: "HelmRelease",
Name: "*",
Namespace: "*",
},
},
ExclusionList: []string{
"doesnotoccur", // not intended to match
"excluded",
},
},
}

g.Expect(k8sClient.Create(context.Background(), alert)).To(Succeed())

// wait for controller to mark the alert as ready
g.Eventually(func() bool {
var obj apiv1beta2.Alert
g.Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(alert), &obj))
return conditions.IsReady(&obj)
}, 30*time.Second, time.Second).Should(BeTrue())

event := eventv1.Event{
InvolvedObject: corev1.ObjectReference{
Kind: "Bucket",
Name: "hyacinth",
Namespace: namespace,
},
Severity: "info",
Timestamp: metav1.Now(),
Message: "well that happened",
Reason: "event-happened",
ReportingController: "source-controller",
}

testSent := func() {
buf := &bytes.Buffer{}
g.Expect(json.NewEncoder(buf).Encode(&event)).To(Succeed())
res, err := http.Post("http://localhost:56789/", "application/json", buf)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(res.StatusCode).To(Equal(202)) // event_server responds with 202 Accepted
}

testForwarded := func() {
g.Eventually(func() bool {
return req == nil
}, "2s", "0.1s").Should(BeFalse())
}

testFiltered := func() {
// The event_server does forwarding in a goroutine, after
// responding to the POST of the event. This makes it
// difficult to know whether the provider has filtered the
// event, or just not run the goroutine yet. For now, I'll use
// a timeout (and Consistently so it can fail early)
g.Consistently(func() bool {
return req == nil
}, "1s", "0.1s").Should(BeTrue())
}

tests := []struct {
name string
modifyEventFunc func(e eventv1.Event) eventv1.Event
forwarded bool
}{
{
name: "forwards when source is a match",
modifyEventFunc: func(e eventv1.Event) eventv1.Event { return e },
forwarded: true,
},
{
name: "forward events for cross-namespace sources",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "Bucket"
e.InvolvedObject.Name = "daffodil"
e.InvolvedObject.Namespace = "test"
return e
},
forwarded: true,
},
{
name: "forwards events when the namespace wildcard is used",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "Bucket"
e.InvolvedObject.Name = "bluebells"
e.InvolvedObject.Namespace = "test-" + randStringRunes(5)
return e
},
forwarded: true,
},
{
name: "drops event when source Kind does not match",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "GitRepository"
e.InvolvedObject.Namespace = namespace
return e
},
forwarded: false,
},
{
name: "drops event when source name does not match",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "Bucket"
e.InvolvedObject.Name = "slop"
e.InvolvedObject.Namespace = namespace
return e
},
forwarded: false,
},
{
name: "drops event when source namespace does not match",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "Bucket"
e.InvolvedObject.Name = "hyacinth"
e.InvolvedObject.Namespace = "all-buckets"
return e
},
forwarded: false,
},
{
name: "drops event that is matched by exclusion",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "Bucket"
e.InvolvedObject.Name = "bluebells"
e.InvolvedObject.Namespace = "test-" + randStringRunes(5)
e.Message = "this is excluded"
return e
},
forwarded: false,
},
{
name: "forwards events when name wildcard is used",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "Kustomization"
e.InvolvedObject.Name = "test"
e.InvolvedObject.Namespace = namespace
e.Message = "test"
return e
},
forwarded: true,
},
{
name: "forwards events when name and namespace wildcards is used",
modifyEventFunc: func(e eventv1.Event) eventv1.Event {
e.InvolvedObject.Kind = "HelmRelease"
e.InvolvedObject.Name = "test"
e.InvolvedObject.Namespace = "test-" + randStringRunes(5)
e.Message = "test"
return e
},
forwarded: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event = tt.modifyEventFunc(event)
testSent()
if tt.forwarded {
testForwarded()
} else {
testFiltered()
}
req = nil
})
}
}
2 changes: 1 addition & 1 deletion internal/server/event_handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ func (s *EventServer) handleEvent() func(w http.ResponseWriter, r *http.Request)
}

func (s *EventServer) eventMatchesAlert(ctx context.Context, event *eventv1.Event, source apiv1.CrossNamespaceObjectReference, severity string) bool {
if event.InvolvedObject.Namespace == source.Namespace && event.InvolvedObject.Kind == source.Kind {
if (event.InvolvedObject.Namespace == source.Namespace || source.Namespace == "*") && event.InvolvedObject.Kind == source.Kind {
if event.Severity == severity || severity == eventv1.EventSeverityInfo {
labelMatch := true
if source.Name == "*" && source.MatchLabels != nil {
Expand Down