From 287851bdd6b577c057a44583577a467e819c09dc Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Thu, 30 May 2024 11:38:45 +0200 Subject: [PATCH 1/7] Add `period` column to `container_log` table --- schema/mysql/schema.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql index 6ca7d77a..a3f4d227 100644 --- a/schema/mysql/schema.sql +++ b/schema/mysql/schema.sql @@ -173,6 +173,7 @@ CREATE TABLE container_log ( pod_uuid binary(16) NOT NULL, logs longtext NOT NULL, last_update bigint NOT NULL, + `period` bigint unsigned NOT NULL, PRIMARY KEY (container_uuid, pod_uuid) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 009a93b3a0a8ebd77322ba6517a94f9442092f90 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Thu, 30 May 2024 11:39:54 +0200 Subject: [PATCH 2/7] Update primary key of `container_log` table to include `period` --- schema/mysql/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql index a3f4d227..3e2f7943 100644 --- a/schema/mysql/schema.sql +++ b/schema/mysql/schema.sql @@ -175,7 +175,7 @@ CREATE TABLE container_log ( last_update bigint NOT NULL, `period` bigint unsigned NOT NULL, - PRIMARY KEY (container_uuid, pod_uuid) + PRIMARY KEY (container_uuid, pod_uuid, period) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE deployment ( From 766703e7b11d6f92dd9151018ba24d03671fbbe6 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Thu, 30 May 2024 11:49:20 +0200 Subject: [PATCH 3/7] Add `Period` field to `ContainerLogMeta` --- pkg/schema/v1/container.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/schema/v1/container.go b/pkg/schema/v1/container.go index 249a479d..41117914 100644 --- a/pkg/schema/v1/container.go +++ b/pkg/schema/v1/container.go @@ -86,6 +86,7 @@ type ContainerMount struct { type ContainerLogMeta struct { Logs string `db:"logs"` LastUpdate types.UnixMilli `db:"last_update"` + Period types.UnixMilli } type ContainerLog struct { From 7b82e2f9d332bf8c36873bd71d00b0ab228488e5 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Thu, 30 May 2024 11:58:48 +0200 Subject: [PATCH 4/7] Assign calculated period start to `cl.Period` - Calculate period start based on the current time in milliseconds, rounded to the nearest hour. --- pkg/schema/v1/container.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/schema/v1/container.go b/pkg/schema/v1/container.go index 41117914..cd504e6c 100644 --- a/pkg/schema/v1/container.go +++ b/pkg/schema/v1/container.go @@ -125,6 +125,10 @@ func (cl *ContainerLog) syncContainerLogs(ctx context.Context, clientset *kubern } cl.LastUpdate = types.UnixMilli(time.Now()) + // Calculate period + currentTimeMillis := time.Now().UnixMilli() + periodStartMillis := currentTimeMillis - (currentTimeMillis % (3600 * 1000)) + cl.Period = types.UnixMilli(time.UnixMilli(periodStartMillis)) cl.Logs += string(logs) entities := make(chan interface{}, 1) entities <- cl From 15be953fad8f987fbba471a8499596d8a152996b Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Thu, 30 May 2024 12:06:29 +0200 Subject: [PATCH 5/7] Add check for existing `logs` before updating `cl.Logs` --- pkg/schema/v1/container.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/schema/v1/container.go b/pkg/schema/v1/container.go index cd504e6c..6045dfd1 100644 --- a/pkg/schema/v1/container.go +++ b/pkg/schema/v1/container.go @@ -129,7 +129,19 @@ func (cl *ContainerLog) syncContainerLogs(ctx context.Context, clientset *kubern currentTimeMillis := time.Now().UnixMilli() periodStartMillis := currentTimeMillis - (currentTimeMillis % (3600 * 1000)) cl.Period = types.UnixMilli(time.UnixMilli(periodStartMillis)) - cl.Logs += string(logs) + + // Check if logs for the current period already exist + existingLog := &ContainerLog{} + err = db.Get(existingLog, "SELECT * FROM container_log WHERE container_uuid = ? AND pod_uuid = ? AND period = ?", cl.ContainerUuid, cl.PodUuid, cl.Period) + if errors.Is(err, sql.ErrNoRows) { + // No existing logs for this period, insert new log + cl.Logs = string(logs) + } else if err != nil { + return err + } else { + // Existing logs found for this period, concatenate logs + cl.Logs += string(logs) + } entities := make(chan interface{}, 1) entities <- cl close(entities) From a3b381303472c32dc5314f869ebb3ebb438be625 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Thu, 30 May 2024 12:15:21 +0200 Subject: [PATCH 6/7] Delete `container logs` older than 24 hours --- cmd/icinga-kubernetes/main.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmd/icinga-kubernetes/main.go b/cmd/icinga-kubernetes/main.go index 886c3480..70d41606 100644 --- a/cmd/icinga-kubernetes/main.go +++ b/cmd/icinga-kubernetes/main.go @@ -198,6 +198,26 @@ func main() { return } }, periodic.Immediate()).Stop() + + defer periodic.Start(ctx, time.Hour, func(tick periodic.Tick) { + olderThan := tick.Time.AddDate(0, 0, -1) + + _, err := db.CleanupOlderThan( + ctx, database.CleanupStmt{ + Table: "container_log", + PK: "container_id, pod_id, period", + Column: "last_update", + }, 5000, olderThan, + ) + if err != nil { + select { + case errs <- err: + case <-ctx.Done(): + } + + return + } + }, periodic.Immediate()).Stop() com.ErrgroupReceive(ctx, g, errs) if err := g.Wait(); err != nil { From 31f1696fd64eebfe3f92c9efd437a797692811b1 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Sat, 1 Jun 2024 23:32:57 +0200 Subject: [PATCH 7/7] Update `warmup` function to optimize `container log` retrieval --- pkg/schema/v1/container.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/schema/v1/container.go b/pkg/schema/v1/container.go index 6045dfd1..d6888561 100644 --- a/pkg/schema/v1/container.go +++ b/pkg/schema/v1/container.go @@ -289,9 +289,13 @@ func SyncContainers(ctx context.Context, db *database.Database, g *errgroup.Grou func warmup(ctx context.Context, db *database.Database) error { g, ctx := errgroup.WithContext(ctx) + query := `SELECT cl.* FROM container_log cl INNER JOIN (SELECT container_uuid, pod_uuid, MAX(last_update) as last_update + FROM container_log GROUP BY container_uuid) max_cl ON cl.container_uuid = max_cl.container_uuid AND cl.pod_uuid = max_cl.pod_uuid + AND cl.last_update = max_cl.last_update` + entities, errs := db.YieldAll(ctx, func() (interface{}, error) { return &ContainerLog{}, nil - }, db.BuildSelectStmt(ContainerLog{}, ContainerLog{})) + }, query) com.ErrgroupReceive(ctx, g, errs) g.Go(func() error {