From 279760f7d7e5d85ffad8af4c2ca4ce99328a49c7 Mon Sep 17 00:00:00 2001 From: vijeyash Date: Sun, 5 Nov 2023 10:26:11 +0530 Subject: [PATCH 1/2] modified migration with ttl with ttl interval as configurable --- cmd/cli/commands/sql.go | 27 ++----- cmd/cli/config/config.go | 39 ++-------- cmd/cli/config/utils.go | 123 +++++++++++++++++++++++++++++++ dockerfiles/migration/Dockerfile | 2 +- 4 files changed, 135 insertions(+), 56 deletions(-) create mode 100644 cmd/cli/config/utils.go diff --git a/cmd/cli/commands/sql.go b/cmd/cli/commands/sql.go index 07389422..c9d0d610 100644 --- a/cmd/cli/commands/sql.go +++ b/cmd/cli/commands/sql.go @@ -5,8 +5,6 @@ import ( "log" "os" - "github.com/golang-migrate/migrate/v4" - cm "github.com/golang-migrate/migrate/v4/database/clickhouse" _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/intelops/kubviz/cmd/cli/config" "github.com/spf13/cobra" @@ -47,28 +45,15 @@ migration sql -e --no`, if executeFlag { if yesFlag { - db, cfg, err := config.OpenClickHouseConn() + cfg, err := config.New() if err != nil { - log.Fatalf("Failed to open ClickHouse connection: %v", err) + log.Fatalf("failed to parse the env : %v", err.Error()) + return } - defer db.Close() - driver, err := cm.WithInstance(db, &cm.Config{}) - if err != nil { - log.Fatalf("Failed to create migrate driver: %v", err) - } - - m, err := migrate.NewWithDatabaseInstance( - fmt.Sprintf("file://%s", cfg.SchemaPath), - "clickhouse", - driver, - ) - if err != nil { - log.Fatalf("Clickhouse Migration initialization failed: %v", err) - } - if err := m.Up(); err != nil && err != migrate.ErrNoChange { - log.Fatalf("Migration failed: %v", err) + if err := cfg.Migrate(); err != nil { + log.Fatalf("failed to migrate : %v", err.Error()) + return } - fmt.Println("Clickhouse Migrations applied successfully!") } else { fmt.Println("Clickhouse Migration skipped due to --no flag.") } diff --git a/cmd/cli/config/config.go b/cmd/cli/config/config.go index 69d709df..7862f1c3 100644 --- a/cmd/cli/config/config.go +++ b/cmd/cli/config/config.go @@ -1,10 +1,6 @@ package config import ( - "database/sql" - "fmt" - - "github.com/ClickHouse/clickhouse-go/v2" "github.com/kelseyhightower/envconfig" ) @@ -14,40 +10,15 @@ type Config struct { ClickHouseUsername string `envconfig:"CLICKHOUSE_USERNAME"` ClickHousePassword string `envconfig:"CLICKHOUSE_PASSWORD"` SchemaPath string `envconfig:"SCHEMA_PATH" default:"/sql"` + TtlInterval string `envconfig:"TTL_INTERVAL" default:"1"` + TtlUnit string `envconfig:"TTL_UNIT" default:"MONTH"` } -func OpenClickHouseConn() (*sql.DB, *Config, error) { +func New() (*Config, error) { var cfg Config err := envconfig.Process("", &cfg) if err != nil { - return nil, nil, err - } - var options clickhouse.Options - - if cfg.ClickHouseUsername != "" && cfg.ClickHousePassword != "" { - fmt.Println("Using provided username and password") - options = clickhouse.Options{ - Addr: []string{fmt.Sprintf("%s:%d", cfg.DBAddress, cfg.DbPort)}, - Debug: true, - Auth: clickhouse.Auth{ - Username: cfg.ClickHouseUsername, - Password: cfg.ClickHousePassword, - }, - } - } else { - fmt.Println("Using connection without username and password") - options = clickhouse.Options{ - Addr: []string{fmt.Sprintf("%s:%d", cfg.DBAddress, cfg.DbPort)}, - } - } - - conn := clickhouse.OpenDB(&options) - if err := conn.Ping(); err != nil { - if exception, ok := err.(*clickhouse.Exception); ok { - return nil, nil, fmt.Errorf("[%d] %s %s", exception.Code, exception.Message, exception.StackTrace) - } else { - return nil, nil, err - } + return nil, err } - return conn, &cfg, nil + return &cfg, nil } diff --git a/cmd/cli/config/utils.go b/cmd/cli/config/utils.go new file mode 100644 index 00000000..360fbadc --- /dev/null +++ b/cmd/cli/config/utils.go @@ -0,0 +1,123 @@ +package config + +import ( + "bytes" + "database/sql" + "fmt" + "html/template" + "io" + "os" + "path/filepath" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/golang-migrate/migrate/v4" + ch "github.com/golang-migrate/migrate/v4/database/clickhouse" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +func (cfg *Config) openClickHouseConn() (*sql.DB, error) { + + var options clickhouse.Options + if cfg.ClickHouseUsername != "" && cfg.ClickHousePassword != "" { + fmt.Println("Using provided username and password") + options = clickhouse.Options{ + Addr: []string{fmt.Sprintf("%s:%d", cfg.DBAddress, cfg.DbPort)}, + Debug: true, + Auth: clickhouse.Auth{ + Username: cfg.ClickHouseUsername, + Password: cfg.ClickHousePassword, + }, + } + + } else { + fmt.Println("Using connection without username and password") + options = clickhouse.Options{ + Addr: []string{fmt.Sprintf("%s:%d", cfg.DBAddress, cfg.DbPort)}, + } + } + + conn := clickhouse.OpenDB(&options) + if err := conn.Ping(); err != nil { + if exception, ok := err.(*clickhouse.Exception); ok { + return nil, fmt.Errorf("[%d] %s %s", exception.Code, exception.Message, exception.StackTrace) + } else { + return nil, err + } + } + return conn, nil +} + +func (cfg *Config) processSQLTemplate(filePath string) (string, error) { + + file, err := os.Open(filePath) + if err != nil { + return "", err + } + data, err := io.ReadAll(file) + if err != nil { + return "", err + } + tmpl, err := template.New("sql").Parse(string(data)) + if err != nil { + return "", err + } + + params := map[string]string{ + "TTLValue": cfg.TtlInterval, + "TTLUnit": cfg.TtlUnit, + } + + var buf bytes.Buffer + err = tmpl.Execute(&buf, params) + if err != nil { + return "", err + } + return buf.String(), nil +} + +func (cfg *Config) Migrate() error { + dir := cfg.SchemaPath + files, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("failed to read directory %w", err) + } + + for _, file := range files { + if strings.HasSuffix(file.Name(), ".up.sql") { + fullPath := filepath.Join(dir, file.Name()) + processedSQL, err := cfg.processSQLTemplate(fullPath) + if err != nil { + return fmt.Errorf("failed to process the sql template for file %s : %w", file.Name(), err) + } + + err = os.WriteFile(fullPath, []byte(processedSQL), 0644) + if err != nil { + return fmt.Errorf("failed to write to file %s : %w", fullPath, err) + } + } + } + + conn, err := cfg.openClickHouseConn() + if err != nil { + return fmt.Errorf("unable to create a clickhouse conection %w", err) + } + + driver, err := ch.WithInstance(conn, &ch.Config{}) + if err != nil { + return fmt.Errorf("failed to create migrate driver %w", err) + } + m, err := migrate.NewWithDatabaseInstance( + fmt.Sprintf("file://%s", dir), + "clickhouse", + driver, + ) + if err != nil { + return fmt.Errorf("clickhouse migration initialization failed %w", err) + } + if err := m.Up(); err != nil && err != migrate.ErrNoChange { + return fmt.Errorf("migration failed %w", err) + } + fmt.Println("Clickhouse Migration applied successfully!") + return nil +} diff --git a/dockerfiles/migration/Dockerfile b/dockerfiles/migration/Dockerfile index f2aa1bea..6cc51b93 100644 --- a/dockerfiles/migration/Dockerfile +++ b/dockerfiles/migration/Dockerfile @@ -16,7 +16,7 @@ WORKDIR / COPY --from=builder /workspace/migration . COPY --from=builder /workspace/sql /sql COPY --from=builder /workspace/script /script - +RUN chmod -R 777 /sql USER 65532:65532 ENTRYPOINT ["/migration"] From c479f42f442f24616a1f2dafb1f08f9a009297f2 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Sun, 5 Nov 2023 11:17:52 +0530 Subject: [PATCH 2/2] added ttl values --- ...rivy_misconfig.down.sql => 0000010_trivy_misconfig.down.sql} | 0 ...44_trivy_misconfig.up.sql => 0000010_trivy_misconfig.up.sql} | 2 +- ...15101910_trivyimage.down.sql => 0000011_trivyimage.down.sql} | 0 ...230915101910_trivyimage.up.sql => 0000011_trivyimage.up.sql} | 2 +- ..._dockerhubbuild.down.sql => 0000012_dockerhubbuild.down.sql} | 0 ...2122_dockerhubbuild.up.sql => 0000012_dockerhubbuild.up.sql} | 2 +- ...ntainerpush.down.sql => 0000013_azurecontainerpush.down.sql} | 0 ...recontainerpush.up.sql => 0000013_azurecontainerpush.up.sql} | 2 +- ...ontainerpush.down.sql => 0000014_quaycontainerpush.down.sql} | 0 ...uaycontainerpush.up.sql => 0000014_quaycontainerpush.up.sql} | 2 +- ...0915102314_trivysbom.down.sql => 0000015_trivysbom.down.sql} | 0 ...20230915102314_trivysbom.up.sql => 0000015_trivysbom.up.sql} | 2 +- ...2348_azure_devops.down.sql => 0000016_azure_devops.down.sql} | 0 ...15102348_azure_devops.up.sql => 0000016_azure_devops.up.sql} | 2 +- sql/{20230915102437_github.down.sql => 0000017_github.down.sql} | 0 sql/{20230915102437_github.up.sql => 0000017_github.up.sql} | 2 +- sql/{20230915102739_gitlab.down.sql => 0000018_gitlab.down.sql} | 0 sql/{20230915102739_gitlab.up.sql => 0000018_gitlab.up.sql} | 2 +- ...0915102817_bitbucket.down.sql => 0000019_bitbucket.down.sql} | 0 ...20230915102817_bitbucket.up.sql => 0000019_bitbucket.up.sql} | 2 +- sql/{20230915101223_events.down.sql => 000001_events.down.sql} | 0 sql/{20230915101223_events.up.sql => 000001_events.up.sql} | 2 +- sql/{20230915102843_gitea.down.sql => 0000020_gitea.down.sql} | 0 sql/{20230915102843_gitea.up.sql => 0000020_gitea.up.sql} | 2 +- ...{20230915101317_rakkess.down.sql => 000002_rakkess.down.sql} | 0 sql/{20230915101317_rakkess.up.sql => 000002_rakkess.up.sql} | 2 +- ...8_DeprecatedAPIs.down.sql => 000003_DeprecatedAPIs.down.sql} | 0 ...01358_DeprecatedAPIs.up.sql => 000003_DeprecatedAPIs.up.sql} | 2 +- ...5101437_DeletedAPIs.down.sql => 000004_DeletedAPIs.down.sql} | 0 ...30915101437_DeletedAPIs.up.sql => 000004_DeletedAPIs.up.sql} | 2 +- ...ontainerpush.down.sql => 000005_jfrogcontainerpush.down.sql} | 0 ...rogcontainerpush.up.sql => 000005_jfrogcontainerpush.up.sql} | 2 +- ...tall_resources.down.sql => 000006_getall_resources.down.sql} | 0 ...9_getall_resources.up.sql => 000006_getall_resources.up.sql} | 2 +- ...outdated_images.down.sql => 000007_outdated_images.down.sql} | 0 ...643_outdated_images.up.sql => 000007_outdated_images.up.sql} | 2 +- ...30915101736_kubescore.down.sql => 000008_kubescore.down.sql} | 0 ...{20230915101736_kubescore.up.sql => 000008_kubescore.up.sql} | 2 +- ...30915101811_trivy_vul.down.sql => 000009_trivy_vul.down.sql} | 0 ...{20230915101811_trivy_vul.up.sql => 000009_trivy_vul.up.sql} | 2 +- 40 files changed, 20 insertions(+), 20 deletions(-) rename sql/{20230915101844_trivy_misconfig.down.sql => 0000010_trivy_misconfig.down.sql} (100%) rename sql/{20230915101844_trivy_misconfig.up.sql => 0000010_trivy_misconfig.up.sql} (88%) rename sql/{20230915101910_trivyimage.down.sql => 0000011_trivyimage.down.sql} (100%) rename sql/{20230915101910_trivyimage.up.sql => 0000011_trivyimage.up.sql} (86%) rename sql/{20230915102122_dockerhubbuild.down.sql => 0000012_dockerhubbuild.down.sql} (100%) rename sql/{20230915102122_dockerhubbuild.up.sql => 0000012_dockerhubbuild.up.sql} (79%) rename sql/{20230915102157_azurecontainerpush.down.sql => 0000013_azurecontainerpush.down.sql} (100%) rename sql/{20230915102157_azurecontainerpush.up.sql => 0000013_azurecontainerpush.up.sql} (80%) rename sql/{20230915102229_quaycontainerpush.down.sql => 0000014_quaycontainerpush.down.sql} (100%) rename sql/{20230915102229_quaycontainerpush.up.sql => 0000014_quaycontainerpush.up.sql} (80%) rename sql/{20230915102314_trivysbom.down.sql => 0000015_trivysbom.down.sql} (100%) rename sql/{20230915102314_trivysbom.up.sql => 0000015_trivysbom.up.sql} (91%) rename sql/{20230915102348_azure_devops.down.sql => 0000016_azure_devops.down.sql} (100%) rename sql/{20230915102348_azure_devops.up.sql => 0000016_azure_devops.up.sql} (79%) rename sql/{20230915102437_github.down.sql => 0000017_github.down.sql} (100%) rename sql/{20230915102437_github.up.sql => 0000017_github.up.sql} (79%) rename sql/{20230915102739_gitlab.down.sql => 0000018_gitlab.down.sql} (100%) rename sql/{20230915102739_gitlab.up.sql => 0000018_gitlab.up.sql} (79%) rename sql/{20230915102817_bitbucket.down.sql => 0000019_bitbucket.down.sql} (100%) rename sql/{20230915102817_bitbucket.up.sql => 0000019_bitbucket.up.sql} (79%) rename sql/{20230915101223_events.down.sql => 000001_events.down.sql} (100%) rename sql/{20230915101223_events.up.sql => 000001_events.up.sql} (83%) rename sql/{20230915102843_gitea.down.sql => 0000020_gitea.down.sql} (100%) rename sql/{20230915102843_gitea.up.sql => 0000020_gitea.up.sql} (79%) rename sql/{20230915101317_rakkess.down.sql => 000002_rakkess.down.sql} (100%) rename sql/{20230915101317_rakkess.up.sql => 000002_rakkess.up.sql} (77%) rename sql/{20230915101358_DeprecatedAPIs.down.sql => 000003_DeprecatedAPIs.down.sql} (100%) rename sql/{20230915101358_DeprecatedAPIs.up.sql => 000003_DeprecatedAPIs.up.sql} (79%) rename sql/{20230915101437_DeletedAPIs.down.sql => 000004_DeletedAPIs.down.sql} (100%) rename sql/{20230915101437_DeletedAPIs.up.sql => 000004_DeletedAPIs.up.sql} (82%) rename sql/{20230915101512_jfrogcontainerpush.down.sql => 000005_jfrogcontainerpush.down.sql} (100%) rename sql/{20230915101512_jfrogcontainerpush.up.sql => 000005_jfrogcontainerpush.up.sql} (83%) rename sql/{20230915101549_getall_resources.down.sql => 000006_getall_resources.down.sql} (100%) rename sql/{20230915101549_getall_resources.up.sql => 000006_getall_resources.up.sql} (77%) rename sql/{20230915101643_outdated_images.down.sql => 000007_outdated_images.down.sql} (100%) rename sql/{20230915101643_outdated_images.up.sql => 000007_outdated_images.up.sql} (81%) rename sql/{20230915101736_kubescore.down.sql => 000008_kubescore.down.sql} (100%) rename sql/{20230915101736_kubescore.up.sql => 000008_kubescore.up.sql} (76%) rename sql/{20230915101811_trivy_vul.down.sql => 000009_trivy_vul.down.sql} (100%) rename sql/{20230915101811_trivy_vul.up.sql => 000009_trivy_vul.up.sql} (89%) diff --git a/sql/20230915101844_trivy_misconfig.down.sql b/sql/0000010_trivy_misconfig.down.sql similarity index 100% rename from sql/20230915101844_trivy_misconfig.down.sql rename to sql/0000010_trivy_misconfig.down.sql diff --git a/sql/20230915101844_trivy_misconfig.up.sql b/sql/0000010_trivy_misconfig.up.sql similarity index 88% rename from sql/20230915101844_trivy_misconfig.up.sql rename to sql/0000010_trivy_misconfig.up.sql index 02ff2dae..89d758a5 100644 --- a/sql/20230915101844_trivy_misconfig.up.sql +++ b/sql/0000010_trivy_misconfig.up.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS trivy_misconfig ( misconfig_severity String, misconfig_status String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101910_trivyimage.down.sql b/sql/0000011_trivyimage.down.sql similarity index 100% rename from sql/20230915101910_trivyimage.down.sql rename to sql/0000011_trivyimage.down.sql diff --git a/sql/20230915101910_trivyimage.up.sql b/sql/0000011_trivyimage.up.sql similarity index 86% rename from sql/20230915101910_trivyimage.up.sql rename to sql/0000011_trivyimage.up.sql index e2a7aa04..572bb2b4 100644 --- a/sql/20230915101910_trivyimage.up.sql +++ b/sql/0000011_trivyimage.up.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS trivyimage ( vul_severity String, vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102122_dockerhubbuild.down.sql b/sql/0000012_dockerhubbuild.down.sql similarity index 100% rename from sql/20230915102122_dockerhubbuild.down.sql rename to sql/0000012_dockerhubbuild.down.sql diff --git a/sql/20230915102122_dockerhubbuild.up.sql b/sql/0000012_dockerhubbuild.up.sql similarity index 79% rename from sql/20230915102122_dockerhubbuild.up.sql rename to sql/0000012_dockerhubbuild.up.sql index a9d0b5c1..0485e52b 100644 --- a/sql/20230915102122_dockerhubbuild.up.sql +++ b/sql/0000012_dockerhubbuild.up.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS dockerhubbuild ( Owner String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102157_azurecontainerpush.down.sql b/sql/0000013_azurecontainerpush.down.sql similarity index 100% rename from sql/20230915102157_azurecontainerpush.down.sql rename to sql/0000013_azurecontainerpush.down.sql diff --git a/sql/20230915102157_azurecontainerpush.up.sql b/sql/0000013_azurecontainerpush.up.sql similarity index 80% rename from sql/20230915102157_azurecontainerpush.up.sql rename to sql/0000013_azurecontainerpush.up.sql index 8bc4a1fc..a0f5916d 100644 --- a/sql/20230915102157_azurecontainerpush.up.sql +++ b/sql/0000013_azurecontainerpush.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS azurecontainerpush ( Size Int32, SHAID String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; \ No newline at end of file diff --git a/sql/20230915102229_quaycontainerpush.down.sql b/sql/0000014_quaycontainerpush.down.sql similarity index 100% rename from sql/20230915102229_quaycontainerpush.down.sql rename to sql/0000014_quaycontainerpush.down.sql diff --git a/sql/20230915102229_quaycontainerpush.up.sql b/sql/0000014_quaycontainerpush.up.sql similarity index 80% rename from sql/20230915102229_quaycontainerpush.up.sql rename to sql/0000014_quaycontainerpush.up.sql index cf9c0a6e..2c249b56 100644 --- a/sql/20230915102229_quaycontainerpush.up.sql +++ b/sql/0000014_quaycontainerpush.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS quaycontainerpush ( tag String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102314_trivysbom.down.sql b/sql/0000015_trivysbom.down.sql similarity index 100% rename from sql/20230915102314_trivysbom.down.sql rename to sql/0000015_trivysbom.down.sql diff --git a/sql/20230915102314_trivysbom.up.sql b/sql/0000015_trivysbom.up.sql similarity index 91% rename from sql/20230915102314_trivysbom.up.sql rename to sql/0000015_trivysbom.up.sql index 969f1b66..76de0994 100644 --- a/sql/20230915102314_trivysbom.up.sql +++ b/sql/0000015_trivysbom.up.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS trivysbom ( component_license_exp String, component_purl String, dependency_ref String, - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102348_azure_devops.down.sql b/sql/0000016_azure_devops.down.sql similarity index 100% rename from sql/20230915102348_azure_devops.down.sql rename to sql/0000016_azure_devops.down.sql diff --git a/sql/20230915102348_azure_devops.up.sql b/sql/0000016_azure_devops.up.sql similarity index 79% rename from sql/20230915102348_azure_devops.up.sql rename to sql/0000016_azure_devops.up.sql index ddd02f2b..06a32dd2 100644 --- a/sql/20230915102348_azure_devops.up.sql +++ b/sql/0000016_azure_devops.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS azure_devops ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102437_github.down.sql b/sql/0000017_github.down.sql similarity index 100% rename from sql/20230915102437_github.down.sql rename to sql/0000017_github.down.sql diff --git a/sql/20230915102437_github.up.sql b/sql/0000017_github.up.sql similarity index 79% rename from sql/20230915102437_github.up.sql rename to sql/0000017_github.up.sql index 00b4d3ac..96d9bf24 100644 --- a/sql/20230915102437_github.up.sql +++ b/sql/0000017_github.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS github ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102739_gitlab.down.sql b/sql/0000018_gitlab.down.sql similarity index 100% rename from sql/20230915102739_gitlab.down.sql rename to sql/0000018_gitlab.down.sql diff --git a/sql/20230915102739_gitlab.up.sql b/sql/0000018_gitlab.up.sql similarity index 79% rename from sql/20230915102739_gitlab.up.sql rename to sql/0000018_gitlab.up.sql index d448fa48..2403dff1 100644 --- a/sql/20230915102739_gitlab.up.sql +++ b/sql/0000018_gitlab.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS gitlab ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102817_bitbucket.down.sql b/sql/0000019_bitbucket.down.sql similarity index 100% rename from sql/20230915102817_bitbucket.down.sql rename to sql/0000019_bitbucket.down.sql diff --git a/sql/20230915102817_bitbucket.up.sql b/sql/0000019_bitbucket.up.sql similarity index 79% rename from sql/20230915102817_bitbucket.up.sql rename to sql/0000019_bitbucket.up.sql index 4a5de37e..adf08956 100644 --- a/sql/20230915102817_bitbucket.up.sql +++ b/sql/0000019_bitbucket.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS bitbucket ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101223_events.down.sql b/sql/000001_events.down.sql similarity index 100% rename from sql/20230915101223_events.down.sql rename to sql/000001_events.down.sql diff --git a/sql/20230915101223_events.up.sql b/sql/000001_events.up.sql similarity index 83% rename from sql/20230915101223_events.up.sql rename to sql/000001_events.up.sql index 4d490012..a410aaf5 100644 --- a/sql/20230915101223_events.up.sql +++ b/sql/000001_events.up.sql @@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS events ( Event String, FirstTime String, LastTime String, - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915102843_gitea.down.sql b/sql/0000020_gitea.down.sql similarity index 100% rename from sql/20230915102843_gitea.down.sql rename to sql/0000020_gitea.down.sql diff --git a/sql/20230915102843_gitea.up.sql b/sql/0000020_gitea.up.sql similarity index 79% rename from sql/20230915102843_gitea.up.sql rename to sql/0000020_gitea.up.sql index cc038088..1b42c0e7 100644 --- a/sql/20230915102843_gitea.up.sql +++ b/sql/0000020_gitea.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS gitea ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101317_rakkess.down.sql b/sql/000002_rakkess.down.sql similarity index 100% rename from sql/20230915101317_rakkess.down.sql rename to sql/000002_rakkess.down.sql diff --git a/sql/20230915101317_rakkess.up.sql b/sql/000002_rakkess.up.sql similarity index 77% rename from sql/20230915101317_rakkess.up.sql rename to sql/000002_rakkess.up.sql index 2d7cc079..3542f59b 100644 --- a/sql/20230915101317_rakkess.up.sql +++ b/sql/000002_rakkess.up.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS rakkess ( List String, Update String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101358_DeprecatedAPIs.down.sql b/sql/000003_DeprecatedAPIs.down.sql similarity index 100% rename from sql/20230915101358_DeprecatedAPIs.down.sql rename to sql/000003_DeprecatedAPIs.down.sql diff --git a/sql/20230915101358_DeprecatedAPIs.up.sql b/sql/000003_DeprecatedAPIs.up.sql similarity index 79% rename from sql/20230915101358_DeprecatedAPIs.up.sql rename to sql/000003_DeprecatedAPIs.up.sql index 1bc1fdc4..a7ed1b6e 100644 --- a/sql/20230915101358_DeprecatedAPIs.up.sql +++ b/sql/000003_DeprecatedAPIs.up.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS DeprecatedAPIs ( Deprecated UInt8, Scope String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101437_DeletedAPIs.down.sql b/sql/000004_DeletedAPIs.down.sql similarity index 100% rename from sql/20230915101437_DeletedAPIs.down.sql rename to sql/000004_DeletedAPIs.down.sql diff --git a/sql/20230915101437_DeletedAPIs.up.sql b/sql/000004_DeletedAPIs.up.sql similarity index 82% rename from sql/20230915101437_DeletedAPIs.up.sql rename to sql/000004_DeletedAPIs.up.sql index eaeddda3..931b93e7 100644 --- a/sql/20230915101437_DeletedAPIs.up.sql +++ b/sql/000004_DeletedAPIs.up.sql @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs ( Deleted UInt8, Scope String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101512_jfrogcontainerpush.down.sql b/sql/000005_jfrogcontainerpush.down.sql similarity index 100% rename from sql/20230915101512_jfrogcontainerpush.down.sql rename to sql/000005_jfrogcontainerpush.down.sql diff --git a/sql/20230915101512_jfrogcontainerpush.up.sql b/sql/000005_jfrogcontainerpush.up.sql similarity index 83% rename from sql/20230915101512_jfrogcontainerpush.up.sql rename to sql/000005_jfrogcontainerpush.up.sql index aa470b8b..6bac6d8d 100644 --- a/sql/20230915101512_jfrogcontainerpush.up.sql +++ b/sql/000005_jfrogcontainerpush.up.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS jfrogcontainerpush ( Tag String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101549_getall_resources.down.sql b/sql/000006_getall_resources.down.sql similarity index 100% rename from sql/20230915101549_getall_resources.down.sql rename to sql/000006_getall_resources.down.sql diff --git a/sql/20230915101549_getall_resources.up.sql b/sql/000006_getall_resources.up.sql similarity index 77% rename from sql/20230915101549_getall_resources.up.sql rename to sql/000006_getall_resources.up.sql index 5a2c2e17..23991a21 100644 --- a/sql/20230915101549_getall_resources.up.sql +++ b/sql/000006_getall_resources.up.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS getall_resources ( Resource String, Age String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101643_outdated_images.down.sql b/sql/000007_outdated_images.down.sql similarity index 100% rename from sql/20230915101643_outdated_images.down.sql rename to sql/000007_outdated_images.down.sql diff --git a/sql/20230915101643_outdated_images.up.sql b/sql/000007_outdated_images.up.sql similarity index 81% rename from sql/20230915101643_outdated_images.up.sql rename to sql/000007_outdated_images.up.sql index 36a839c3..479c0902 100644 --- a/sql/20230915101643_outdated_images.up.sql +++ b/sql/000007_outdated_images.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS outdated_images ( LatestVersion String, VersionsBehind Int64, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101736_kubescore.down.sql b/sql/000008_kubescore.down.sql similarity index 100% rename from sql/20230915101736_kubescore.down.sql rename to sql/000008_kubescore.down.sql diff --git a/sql/20230915101736_kubescore.up.sql b/sql/000008_kubescore.up.sql similarity index 76% rename from sql/20230915101736_kubescore.up.sql rename to sql/000008_kubescore.up.sql index fde28707..662fab9b 100644 --- a/sql/20230915101736_kubescore.up.sql +++ b/sql/000008_kubescore.up.sql @@ -4,7 +4,7 @@ CREATE TABLE IF NOT EXISTS kubescore ( cluster_name String, recommendations String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/20230915101811_trivy_vul.down.sql b/sql/000009_trivy_vul.down.sql similarity index 100% rename from sql/20230915101811_trivy_vul.down.sql rename to sql/000009_trivy_vul.down.sql diff --git a/sql/20230915101811_trivy_vul.up.sql b/sql/000009_trivy_vul.up.sql similarity index 89% rename from sql/20230915101811_trivy_vul.up.sql rename to sql/000009_trivy_vul.up.sql index d2f8efd3..8acb241d 100644 --- a/sql/20230915101811_trivy_vul.up.sql +++ b/sql/000009_trivy_vul.up.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS trivy_vul ( vul_severity String, vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate;