From ae985e4c3937367501224afda9f5a9e79f8edb9b Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 15:49:08 +0200 Subject: [PATCH 01/27] Add pg dep --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index bf9fa48..093baf8 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/doug-martin/goqu/v9 v9.18.0 github.com/gin-gonic/gin v1.9.1 + github.com/lib/pq v1.10.1 github.com/lopezator/migrator v0.3.1 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 From c2b8abd64a0244846e757e2da13e035a160d663e Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 15:10:42 +0200 Subject: [PATCH 02/27] Rename sqlite migration function --- cli/cmds/migrate.go | 2 +- db/migrations/migrate.go | 2 +- db/setup.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/cmds/migrate.go b/cli/cmds/migrate.go index 4775d9b..29a4de7 100644 --- a/cli/cmds/migrate.go +++ b/cli/cmds/migrate.go @@ -24,7 +24,7 @@ func init() { log.Fatalf("setting up database failed: %s", err) } - err = migrations.Run(database) + err = migrations.RunSQLite(database) if err != nil { log.Fatalf("migrating database failed: %s", err) } diff --git a/db/migrations/migrate.go b/db/migrations/migrate.go index 760062c..985f92e 100644 --- a/db/migrations/migrate.go +++ b/db/migrations/migrate.go @@ -7,7 +7,7 @@ import ( "github.com/lopezator/migrator" ) -func Run(db *sql.DB) error { +func RunSQLite(db *sql.DB) error { // Configure migrations m, err := migrator.New( diff --git a/db/setup.go b/db/setup.go index c9ebd9c..fe251a1 100644 --- a/db/setup.go +++ b/db/setup.go @@ -16,7 +16,7 @@ func SetupAndMigrateDB(path string) (*sql.DB, *goqu.Database, error) { return nil, nil, err } - err = migrations.Run(db) + err = migrations.RunSQLite(db) if err != nil { return nil, nil, fmt.Errorf("migrating sqlite database failed: %s", err) } From ef3bb177f1f935cf6e87984c9fc900aa7c15fe85 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 14:02:15 +0200 Subject: [PATCH 03/27] Add postgres setup function --- db/setup.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/db/setup.go b/db/setup.go index fe251a1..0937efb 100644 --- a/db/setup.go +++ b/db/setup.go @@ -5,8 +5,10 @@ import ( "fmt" "github.com/doug-martin/goqu/v9" + _ "github.com/doug-martin/goqu/v9/dialect/postgres" _ "github.com/doug-martin/goqu/v9/dialect/sqlite3" "github.com/fabiante/persurl/db/migrations" + _ "github.com/lib/pq" _ "modernc.org/sqlite" ) @@ -32,3 +34,26 @@ func SetupDB(path string) (*sql.DB, *goqu.Database, error) { return database, goqu.New("sqlite3", database), nil } + +func SetupAndMigratePostgresDB(dsn string) (*sql.DB, *goqu.Database, error) { + db, gdb, err := SetupPostgresDB(dsn) + if err != nil { + return nil, nil, err + } + + err = migrations.RunPostgres(db) + if err != nil { + return nil, nil, fmt.Errorf("migrating postgres database failed: %s", err) + } + + return db, gdb, nil +} + +func SetupPostgresDB(dsn string) (*sql.DB, *goqu.Database, error) { + database, err := sql.Open("postgres", dsn) + if err != nil { + return nil, nil, fmt.Errorf("opening postgres database failed: %s", err) + } + + return database, goqu.New("postgres", database), nil +} From 034c4f81b2a1d9a1e456b03bf09eca575e7c79c6 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 14:14:41 +0200 Subject: [PATCH 04/27] Add migrations for postgres --- db/migrations/migrate.go | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/db/migrations/migrate.go b/db/migrations/migrate.go index 985f92e..75affbb 100644 --- a/db/migrations/migrate.go +++ b/db/migrations/migrate.go @@ -26,6 +26,25 @@ func RunSQLite(db *sql.DB) error { return nil } +func RunPostgres(db *sql.DB) error { + // Configure migrations + + m, err := migrator.New( + // type cast is required because []*migrator.MigrationNoTx is not assignable to []migrator.Migration + migrator.Migrations(migrationsPostgres...), + ) + if err != nil { + return fmt.Errorf("initializing migrations failed: %w", err) + } + + // Migrate up + if err := m.Migrate(db); err != nil { + return fmt.Errorf("running migrations failed: %w", err) + } + + return nil +} + func newMigration(name string, query string) *migrator.MigrationNoTx { return &migrator.MigrationNoTx{ Name: name, @@ -65,3 +84,31 @@ var migrationsSQLite = []any{ )`, ), } + +var migrationsPostgres = []any{ + newMigration("2023-09-18-00000001-CreateTableDomains", `create table domains +( + id serial + constraint domains_pk2 + unique, + name varchar(128) not null + constraint domains_pk + primary key +)`, + ), + newMigration("2023-09-18-00000002-CreateTablePurls", `create table purls +( + id serial + constraint purls_pk + primary key, + domain_id integer not null + constraint purls_domains_id_fk + references domains (id) + on delete restrict, + name varchar(128) not null, + target varchar(4096) not null, + constraint purls_pk2 + unique (domain_id, name) +)`, + ), +} From f5b98f4c65d56337b96e376e9a109f78e6d32cc3 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:21:14 +0200 Subject: [PATCH 05/27] Add docker-compose with postgres container This is used for local development --- docker-compose.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..79004ba --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3.8' + +services: + db: + image: postgres:14-alpine + environment: + - POSTGRES_DB=persurl + - POSTGRES_USER=persurl + - POSTGRES_PASSWORD=persurl + ports: + - '5432:5432' + volumes: + - db:/var/lib/postgresql/data:rw + +volumes: + db: From 80074b25ce66b279e0a3792b6bfc3a213a1b07c0 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:33:46 +0200 Subject: [PATCH 06/27] Fix admin test The test had a bug which did not show with SQLite it seems. --- tests/specs/admin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/specs/admin.go b/tests/specs/admin.go index 284d472..35a1903 100644 --- a/tests/specs/admin.go +++ b/tests/specs/admin.go @@ -106,7 +106,7 @@ func testDomainAdmin(t *testing.T, admin dsl.AdminAPI) { t.Run("can't create duplicate domain", func(t *testing.T) { domain := "should-exist-once-4357824758wr47895645" dsl.GivenExistingDomain(t, admin, domain) - err := admin.CreateDomain("awesome-domain-unique-name-123") + err := admin.CreateDomain("should-exist-once-4357824758wr47895645") require.Error(t, err) require.ErrorIs(t, err, app.ErrBadRequest) }) From a67d033d9aa733755d1a8e3c85fb846b4d389e78 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:39:05 +0200 Subject: [PATCH 07/27] Fix db error mapping The previous implementation was for SQLite. This is now compatible with pg --- db/database.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/db/database.go b/db/database.go index 9fc0da1..a861047 100644 --- a/db/database.go +++ b/db/database.go @@ -8,8 +8,7 @@ import ( "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exec" "github.com/fabiante/persurl/app" - "modernc.org/sqlite" - sqlite3 "modernc.org/sqlite/lib" + "github.com/lib/pq" ) // Database implements the applications core logic. @@ -114,15 +113,23 @@ func (db *Database) CreateDomain(domain string) error { } } +const ( + pgErrUniqueKeyViolation = "23505" +) + func mapDBError(err error) error { - var serr *sqlite.Error + var serr *pq.Error if !errors.As(err, &serr) { return err } - code := serr.Code() + // Error codes + // SQLite: https://www.sqlite.org/rescode.html + // Postgres: http://www.postgresql.org/docs/9.3/static/errcodes-appendix.html + + code := serr.Code switch code { - case sqlite3.SQLITE_CONSTRAINT_UNIQUE: + case pgErrUniqueKeyViolation: return fmt.Errorf("%w: %s", app.ErrBadRequest, err) default: return fmt.Errorf("unexpected error: %w", err) From f5f2f932d827baf9883e0885d0dcc1f6c1f328f8 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:41:29 +0200 Subject: [PATCH 08/27] Refactor ENV loading into dedicated package --- cli/cmds/migrate.go | 5 +++-- cli/cmds/run.go | 5 +++-- {cli/cmds => config}/env.go | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) rename {cli/cmds => config}/env.go (78%) diff --git a/cli/cmds/migrate.go b/cli/cmds/migrate.go index 29a4de7..33b2927 100644 --- a/cli/cmds/migrate.go +++ b/cli/cmds/migrate.go @@ -3,6 +3,7 @@ package cmds import ( "log" + "github.com/fabiante/persurl/config" "github.com/fabiante/persurl/db" "github.com/fabiante/persurl/db/migrations" "github.com/spf13/cobra" @@ -15,9 +16,9 @@ func init() { } cmd.Run = func(cmd *cobra.Command, args []string) { - dataDir := envDataDir() + dataDir := config.DataDir() - dbFile := envDbFile(dataDir) + dbFile := config.DbFile(dataDir) database, _, err := db.SetupDB(dbFile) if err != nil { diff --git a/cli/cmds/run.go b/cli/cmds/run.go index 0b2efb5..a3dfb0f 100644 --- a/cli/cmds/run.go +++ b/cli/cmds/run.go @@ -4,6 +4,7 @@ import ( "log" "github.com/fabiante/persurl/api" + "github.com/fabiante/persurl/config" "github.com/fabiante/persurl/db" "github.com/gin-gonic/gin" "github.com/spf13/cobra" @@ -16,9 +17,9 @@ func init() { } cmd.Run = func(cmd *cobra.Command, args []string) { - dataDir := envDataDir() + dataDir := config.DataDir() - dbFile := envDbFile(dataDir) + dbFile := config.DbFile(dataDir) _, database, err := db.SetupDB(dbFile) if err != nil { diff --git a/cli/cmds/env.go b/config/env.go similarity index 78% rename from cli/cmds/env.go rename to config/env.go index 75d6cfa..31e0dfe 100644 --- a/cli/cmds/env.go +++ b/config/env.go @@ -1,4 +1,4 @@ -package cmds +package config import ( "fmt" @@ -6,7 +6,7 @@ import ( "os" ) -func envDataDir() string { +func DataDir() string { dataDir := os.Getenv("PERSURL_DATA_DIR") if dataDir == "" { dataDir = "." @@ -15,7 +15,7 @@ func envDataDir() string { return dataDir } -func envDbFile(dataDir string) string { +func DbFile(dataDir string) string { dbFile := fmt.Sprintf("%s/prod.sqlite", dataDir) log.Printf("using database file: %s", dbFile) return dbFile From 2cd1494d07c2f66051b77a044ff3c70862c5bf3b Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:48:57 +0200 Subject: [PATCH 09/27] Add godotenv dep and add env loading helper --- config/env.go | 9 +++++++++ go.mod | 1 + go.sum | 2 ++ 3 files changed, 12 insertions(+) diff --git a/config/env.go b/config/env.go index 31e0dfe..a1419c6 100644 --- a/config/env.go +++ b/config/env.go @@ -4,8 +4,17 @@ import ( "fmt" "log" "os" + + "github.com/joho/godotenv" ) +func LoadEnv() { + err := godotenv.Load() + if err != nil { + panic(fmt.Errorf("loading env failed: %w", err)) + } +} + func DataDir() string { dataDir := os.Getenv("PERSURL_DATA_DIR") if dataDir == "" { diff --git a/go.mod b/go.mod index 093baf8..a803c42 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/doug-martin/goqu/v9 v9.18.0 github.com/gin-gonic/gin v1.9.1 + github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.1 github.com/lopezator/migrator v0.3.1 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index 7501e44..cf7db96 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= From d9a819d867aa44ce53bd822a6902876abca8f392 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:52:01 +0200 Subject: [PATCH 10/27] Add helper to load DB dsn --- config/env.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/config/env.go b/config/env.go index a1419c6..bec6ccb 100644 --- a/config/env.go +++ b/config/env.go @@ -16,7 +16,7 @@ func LoadEnv() { } func DataDir() string { - dataDir := os.Getenv("PERSURL_DATA_DIR") + dataDir := os.Getenv(" PERSURL_DATA_DIR") if dataDir == "" { dataDir = "." } @@ -29,3 +29,11 @@ func DbFile(dataDir string) string { log.Printf("using database file: %s", dbFile) return dbFile } + +func DbDSN() string { + dsn := os.Getenv("PERSURL_DB_DSN") + if dsn == "" { + log.Fatalf("persurl db dsn may not be empty") + } + return dsn +} From 2c9cb2df0dd695822e0d8a7d8eed39fc7ce6bc50 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:52:14 +0200 Subject: [PATCH 11/27] Replace sqlite with postgres in tests --- tests/http_load_test.go | 10 +++++++--- tests/http_test.go | 24 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/http_load_test.go b/tests/http_load_test.go index da20734..9537279 100644 --- a/tests/http_load_test.go +++ b/tests/http_load_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/fabiante/persurl/api" + "github.com/fabiante/persurl/config" "github.com/fabiante/persurl/db" "github.com/fabiante/persurl/tests/driver" "github.com/fabiante/persurl/tests/specs" @@ -15,6 +16,8 @@ import ( ) func TestLoadWithHTTPDriver(t *testing.T) { + config.LoadEnv() + if os.Getenv("TEST_LOAD") == "" { t.Skip("load tests are skipped because TEST_LOAD env variable is not set") } @@ -22,11 +25,12 @@ func TestLoadWithHTTPDriver(t *testing.T) { gin.SetMode(gin.TestMode) handler := gin.Default() - sqlitePath := "./test_load_http.sqlite" - _ = os.Remove(sqlitePath) // remove to ensure a clean database - _, database, err := db.SetupAndMigrateDB(sqlitePath) + _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN()) require.NoError(t, err, "setting up db failed") + err = emptyTables(database, "purls", "domains") + require.NoError(t, err, "truncating tables failed") + service := db.NewDatabase(database) server := api.NewServer(service) api.SetupRouting(handler, server) diff --git a/tests/http_test.go b/tests/http_test.go index a6560b8..1d8af51 100644 --- a/tests/http_test.go +++ b/tests/http_test.go @@ -1,12 +1,14 @@ package tests import ( + "errors" "net/http" "net/http/httptest" - "os" "testing" + "github.com/doug-martin/goqu/v9" "github.com/fabiante/persurl/api" + "github.com/fabiante/persurl/config" "github.com/fabiante/persurl/db" "github.com/fabiante/persurl/tests/driver" "github.com/fabiante/persurl/tests/specs" @@ -15,14 +17,17 @@ import ( ) func TestWithHTTPDriver(t *testing.T) { + config.LoadEnv() + gin.SetMode(gin.TestMode) handler := gin.Default() - sqlitePath := "./test_http.sqlite" - _ = os.Remove(sqlitePath) // remove to ensure a clean database - _, database, err := db.SetupAndMigrateDB(sqlitePath) + _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN()) require.NoError(t, err, "setting up db failed") + err = emptyTables(database, "purls", "domains") + require.NoError(t, err, "truncating tables failed") + service := db.NewDatabase(database) server := api.NewServer(service) api.SetupRouting(handler, server) @@ -34,3 +39,14 @@ func TestWithHTTPDriver(t *testing.T) { specs.TestResolver(t, dr) specs.TestAdministration(t, dr) } + +func emptyTables(db *goqu.Database, tables ...string) error { + var errs []error + + for _, table := range tables { + _, err := db.Delete(table).Executor().Exec() + errs = append(errs, err) + } + + return errors.Join(errs...) +} From 2becb1b3b072e4deb3bf95ede36fb636480ec7c6 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:52:18 +0200 Subject: [PATCH 12/27] Update gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 485dee6..1d75d7a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .idea +/.env +*.sqlite From 95eedd93fada919857461601da7926191ff78d92 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:52:26 +0200 Subject: [PATCH 13/27] Add example .env file --- example.env | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 example.env diff --git a/example.env b/example.env new file mode 100644 index 0000000..4f1144c --- /dev/null +++ b/example.env @@ -0,0 +1,5 @@ +# Enable this if you want to run load tests +# TEST_LOAD=1 + +# Replace with your ENV +PERSURL_DB_DSN=postgresql://persurl:persurl@localhost:5432/persurl?sslmode=disable From 6f1f1eaa8e518145273b735adcfa5e87e03b1967 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:55:01 +0200 Subject: [PATCH 14/27] Replace sqlite with postgres in cli --- cli/cmds/migrate.go | 6 +----- cli/cmds/run.go | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/cli/cmds/migrate.go b/cli/cmds/migrate.go index 33b2927..059a5d8 100644 --- a/cli/cmds/migrate.go +++ b/cli/cmds/migrate.go @@ -16,11 +16,7 @@ func init() { } cmd.Run = func(cmd *cobra.Command, args []string) { - dataDir := config.DataDir() - - dbFile := config.DbFile(dataDir) - - database, _, err := db.SetupDB(dbFile) + database, _, err := db.SetupPostgresDB(config.DbDSN()) if err != nil { log.Fatalf("setting up database failed: %s", err) } diff --git a/cli/cmds/run.go b/cli/cmds/run.go index a3dfb0f..71da623 100644 --- a/cli/cmds/run.go +++ b/cli/cmds/run.go @@ -17,11 +17,7 @@ func init() { } cmd.Run = func(cmd *cobra.Command, args []string) { - dataDir := config.DataDir() - - dbFile := config.DbFile(dataDir) - - _, database, err := db.SetupDB(dbFile) + _, database, err := db.SetupPostgresDB(config.DbDSN()) if err != nil { log.Fatalf("setting up database failed: %s", err) } From 70631979096528ea91ee7a38b40bc338b168a8b7 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:56:05 +0200 Subject: [PATCH 15/27] Add docker compose up to github action --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 878aaca..c860595 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,8 @@ on: jobs: test: runs-on: ubuntu-latest + env: + PERSURL_DB_DSN: postgresql://persurl:persurl@localhost:5432/persurl?sslmode=disable steps: - uses: actions/checkout@v3 - name: Set up Go @@ -20,6 +22,8 @@ jobs: - name: Lint API Spec run: npx @redocly/cli lint api/openapi.yml + - name: Run database + run: docker compose up --quiet-pull -d - name: Install Test Runner run: go install github.com/mfridman/tparse@latest - name: Test @@ -32,6 +36,7 @@ jobs: runs-on: ubuntu-latest env: TEST_LOAD: 1 + PERSURL_DB_DSN: postgresql://persurl:persurl@localhost:5432/persurl?sslmode=disable steps: - uses: actions/checkout@v3 - name: Set up Go @@ -39,6 +44,8 @@ jobs: with: go-version: '1.21' + - name: Run database + run: docker compose up --quiet-pull -d - name: Install Test Runner run: go install github.com/mfridman/tparse@latest - name: Test From b90b0b8472244bf3d898651e418fec5ad6d193ee Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:58:18 +0200 Subject: [PATCH 16/27] Remove sqlite from db package --- db/setup.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/db/setup.go b/db/setup.go index 0937efb..c004803 100644 --- a/db/setup.go +++ b/db/setup.go @@ -6,35 +6,10 @@ import ( "github.com/doug-martin/goqu/v9" _ "github.com/doug-martin/goqu/v9/dialect/postgres" - _ "github.com/doug-martin/goqu/v9/dialect/sqlite3" "github.com/fabiante/persurl/db/migrations" _ "github.com/lib/pq" - _ "modernc.org/sqlite" ) -func SetupAndMigrateDB(path string) (*sql.DB, *goqu.Database, error) { - db, gdb, err := SetupDB(path) - if err != nil { - return nil, nil, err - } - - err = migrations.RunSQLite(db) - if err != nil { - return nil, nil, fmt.Errorf("migrating sqlite database failed: %s", err) - } - - return db, gdb, nil -} - -func SetupDB(path string) (*sql.DB, *goqu.Database, error) { - database, err := sql.Open("sqlite", path) - if err != nil { - return nil, nil, fmt.Errorf("opening sqlite database failed: %s", err) - } - - return database, goqu.New("sqlite3", database), nil -} - func SetupAndMigratePostgresDB(dsn string) (*sql.DB, *goqu.Database, error) { db, gdb, err := SetupPostgresDB(dsn) if err != nil { From fb9806b268493325c4b8d4c3372d7c6575582cc0 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 17:58:53 +0200 Subject: [PATCH 17/27] Remove RWMutext from database implementation The lock should not be needed anymore since the change to postgres from sqlite. --- db/database.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/db/database.go b/db/database.go index a861047..a62cecf 100644 --- a/db/database.go +++ b/db/database.go @@ -3,7 +3,6 @@ package db import ( "errors" "fmt" - "sync" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exec" @@ -13,21 +12,16 @@ import ( // Database implements the applications core logic. type Database struct { - db *goqu.Database - lock *sync.RWMutex + db *goqu.Database } func NewDatabase(db *goqu.Database) *Database { return &Database{ - db: db, - lock: &sync.RWMutex{}, + db: db, } } func (db *Database) Resolve(domain, name string) (string, error) { - db.lock.RLock() - defer db.lock.RUnlock() - query := db.db.Select("purls.target"). From("purls"). Join(goqu.T("domains"), goqu.On(goqu.I("domains.id").Eq(goqu.I("purls.domain_id")))). @@ -49,9 +43,6 @@ func (db *Database) Resolve(domain, name string) (string, error) { } func (db *Database) SavePURL(domain, name, target string) error { - db.lock.Lock() - defer db.lock.Unlock() - // lookup domain first query := db.db.Select("id").From("domains").Where(goqu.C("name").Eq(domain)).Limit(1) @@ -99,9 +90,6 @@ func (db *Database) SavePURL(domain, name, target string) error { } func (db *Database) CreateDomain(domain string) error { - db.lock.Lock() - defer db.lock.Unlock() - stmt := db.db.Insert("domains"). Cols("name"). Vals(goqu.Vals{domain}) From 3901956ddd4cb794e89128eac186e186138872cf Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 18:04:44 +0200 Subject: [PATCH 18/27] Fix emptyTables not being acessible --- db/empty.go | 20 ++++++++++++++++++++ tests/http_load_test.go | 2 +- tests/http_test.go | 15 +-------------- 3 files changed, 22 insertions(+), 15 deletions(-) create mode 100644 db/empty.go diff --git a/db/empty.go b/db/empty.go new file mode 100644 index 0000000..26a8d61 --- /dev/null +++ b/db/empty.go @@ -0,0 +1,20 @@ +package db + +import ( + "errors" + + "github.com/doug-martin/goqu/v9" +) + +// EmptyTables is used to empty a collection of tables. This may be useful if truncating a +// table is not possible. +func EmptyTables(db *goqu.Database, tables ...string) error { + var errs []error + + for _, table := range tables { + _, err := db.Delete(table).Executor().Exec() + errs = append(errs, err) + } + + return errors.Join(errs...) +} diff --git a/tests/http_load_test.go b/tests/http_load_test.go index 9537279..ca6b6d3 100644 --- a/tests/http_load_test.go +++ b/tests/http_load_test.go @@ -28,7 +28,7 @@ func TestLoadWithHTTPDriver(t *testing.T) { _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN()) require.NoError(t, err, "setting up db failed") - err = emptyTables(database, "purls", "domains") + err = db.EmptyTables(database, "purls", "domains") require.NoError(t, err, "truncating tables failed") service := db.NewDatabase(database) diff --git a/tests/http_test.go b/tests/http_test.go index 1d8af51..e4bab5a 100644 --- a/tests/http_test.go +++ b/tests/http_test.go @@ -1,12 +1,10 @@ package tests import ( - "errors" "net/http" "net/http/httptest" "testing" - "github.com/doug-martin/goqu/v9" "github.com/fabiante/persurl/api" "github.com/fabiante/persurl/config" "github.com/fabiante/persurl/db" @@ -25,7 +23,7 @@ func TestWithHTTPDriver(t *testing.T) { _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN()) require.NoError(t, err, "setting up db failed") - err = emptyTables(database, "purls", "domains") + err = db.EmptyTables(database, "purls", "domains") require.NoError(t, err, "truncating tables failed") service := db.NewDatabase(database) @@ -39,14 +37,3 @@ func TestWithHTTPDriver(t *testing.T) { specs.TestResolver(t, dr) specs.TestAdministration(t, dr) } - -func emptyTables(db *goqu.Database, tables ...string) error { - var errs []error - - for _, table := range tables { - _, err := db.Delete(table).Executor().Exec() - errs = append(errs, err) - } - - return errors.Join(errs...) -} From b69a1c0aa2888159680fbea7d00c887c06074020 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 18:08:24 +0200 Subject: [PATCH 19/27] Fix env loading --- config/env.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/config/env.go b/config/env.go index bec6ccb..083ef93 100644 --- a/config/env.go +++ b/config/env.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "log" "os" @@ -9,7 +10,15 @@ import ( ) func LoadEnv() { - err := godotenv.Load() + path := ".env" + + // check if .env file exists - if not, exit early. + _, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return + } + + err = godotenv.Load(path) if err != nil { panic(fmt.Errorf("loading env failed: %w", err)) } From d906b2926e74fd6dc167664bcca5882fba69be39 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 18:35:34 +0200 Subject: [PATCH 20/27] Make db max open connections configurable This is required because if there was no limit, we could spam the pg database and get connection errors. --- cli/cmds/migrate.go | 2 +- cli/cmds/run.go | 2 +- config/env.go | 15 +++++++++++++++ db/setup.go | 8 +++++--- example.env | 3 +++ tests/http_load_test.go | 2 +- tests/http_test.go | 2 +- 7 files changed, 27 insertions(+), 7 deletions(-) diff --git a/cli/cmds/migrate.go b/cli/cmds/migrate.go index 059a5d8..84b3a8b 100644 --- a/cli/cmds/migrate.go +++ b/cli/cmds/migrate.go @@ -16,7 +16,7 @@ func init() { } cmd.Run = func(cmd *cobra.Command, args []string) { - database, _, err := db.SetupPostgresDB(config.DbDSN()) + database, _, err := db.SetupPostgresDB(config.DbDSN(), config.DbMaxConnections()) if err != nil { log.Fatalf("setting up database failed: %s", err) } diff --git a/cli/cmds/run.go b/cli/cmds/run.go index 71da623..4455782 100644 --- a/cli/cmds/run.go +++ b/cli/cmds/run.go @@ -17,7 +17,7 @@ func init() { } cmd.Run = func(cmd *cobra.Command, args []string) { - _, database, err := db.SetupPostgresDB(config.DbDSN()) + _, database, err := db.SetupPostgresDB(config.DbDSN(), config.DbMaxConnections()) if err != nil { log.Fatalf("setting up database failed: %s", err) } diff --git a/config/env.go b/config/env.go index 083ef93..34a23d2 100644 --- a/config/env.go +++ b/config/env.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "strconv" "github.com/joho/godotenv" ) @@ -46,3 +47,17 @@ func DbDSN() string { } return dsn } + +func DbMaxConnections() int { + val := os.Getenv("PERSURL_DB_MAX_CONNECTIONS") + if val == "" { + val = "10" + } + + maxCon, err := strconv.ParseInt(val, 10, 32) + if err != nil { + log.Fatalf("invalid db max connection parameter %s", val) + } + + return int(maxCon) +} diff --git a/db/setup.go b/db/setup.go index c004803..85f32d9 100644 --- a/db/setup.go +++ b/db/setup.go @@ -10,8 +10,8 @@ import ( _ "github.com/lib/pq" ) -func SetupAndMigratePostgresDB(dsn string) (*sql.DB, *goqu.Database, error) { - db, gdb, err := SetupPostgresDB(dsn) +func SetupAndMigratePostgresDB(dsn string, maxConnections int) (*sql.DB, *goqu.Database, error) { + db, gdb, err := SetupPostgresDB(dsn, maxConnections) if err != nil { return nil, nil, err } @@ -24,11 +24,13 @@ func SetupAndMigratePostgresDB(dsn string) (*sql.DB, *goqu.Database, error) { return db, gdb, nil } -func SetupPostgresDB(dsn string) (*sql.DB, *goqu.Database, error) { +func SetupPostgresDB(dsn string, maxConnections int) (*sql.DB, *goqu.Database, error) { database, err := sql.Open("postgres", dsn) if err != nil { return nil, nil, fmt.Errorf("opening postgres database failed: %s", err) } + database.SetMaxOpenConns(maxConnections) + return database, goqu.New("postgres", database), nil } diff --git a/example.env b/example.env index 4f1144c..75be8c0 100644 --- a/example.env +++ b/example.env @@ -3,3 +3,6 @@ # Replace with your ENV PERSURL_DB_DSN=postgresql://persurl:persurl@localhost:5432/persurl?sslmode=disable + +# Max count of connections to open to the database +PERSURL_DB_MAX_CONNECTIONS=10 diff --git a/tests/http_load_test.go b/tests/http_load_test.go index ca6b6d3..d30dc3b 100644 --- a/tests/http_load_test.go +++ b/tests/http_load_test.go @@ -25,7 +25,7 @@ func TestLoadWithHTTPDriver(t *testing.T) { gin.SetMode(gin.TestMode) handler := gin.Default() - _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN()) + _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN(), config.DbMaxConnections()) require.NoError(t, err, "setting up db failed") err = db.EmptyTables(database, "purls", "domains") diff --git a/tests/http_test.go b/tests/http_test.go index e4bab5a..8cc609d 100644 --- a/tests/http_test.go +++ b/tests/http_test.go @@ -20,7 +20,7 @@ func TestWithHTTPDriver(t *testing.T) { gin.SetMode(gin.TestMode) handler := gin.Default() - _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN()) + _, database, err := db.SetupAndMigratePostgresDB(config.DbDSN(), config.DbMaxConnections()) require.NoError(t, err, "setting up db failed") err = db.EmptyTables(database, "purls", "domains") From 52dc42ffbc009e84303947a716786ceed74acfbd Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 18:40:19 +0200 Subject: [PATCH 21/27] Add healthcheck to postgres container --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 79004ba..1ba4a9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,11 @@ services: - '5432:5432' volumes: - db:/var/lib/postgresql/data:rw + healthcheck: + test: [ "CMD-SHELL", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 volumes: db: From 57d09b73e3edd6538707c5ff9b74c8e04d9b2402 Mon Sep 17 00:00:00 2001 From: fabiante Date: Mon, 18 Sep 2023 18:42:11 +0200 Subject: [PATCH 22/27] Add timeout to container start to ensure tests don't fail --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c860595..53a2654 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: run: npx @redocly/cli lint api/openapi.yml - name: Run database - run: docker compose up --quiet-pull -d + run: docker compose up --quiet-pull -d && sleep 5 - name: Install Test Runner run: go install github.com/mfridman/tparse@latest - name: Test @@ -45,7 +45,7 @@ jobs: go-version: '1.21' - name: Run database - run: docker compose up --quiet-pull -d + run: docker compose up --quiet-pull -d && sleep 5 - name: Install Test Runner run: go install github.com/mfridman/tparse@latest - name: Test From 36972adc8cd2e5174354d05e44c0457fe3c09dc9 Mon Sep 17 00:00:00 2001 From: fabiante Date: Tue, 19 Sep 2023 09:00:26 +0200 Subject: [PATCH 23/27] Add optional pgadmin container for local development --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1ba4a9c..57a0400 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,5 +17,14 @@ services: timeout: 5s retries: 5 + pgadmin: + profiles: ["pgadmin"] + image: dpage/pgadmin4 + environment: + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: root + ports: + - "5454:80" + volumes: db: From 57119b8aa1c6a63823018c3122707e52700f06e2 Mon Sep 17 00:00:00 2001 From: fabiante Date: Tue, 19 Sep 2023 09:01:00 +0200 Subject: [PATCH 24/27] Remove sqlite migrations --- cli/cmds/migrate.go | 2 +- db/migrations/migrate.go | 47 ---------------------------------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/cli/cmds/migrate.go b/cli/cmds/migrate.go index 84b3a8b..29a8b75 100644 --- a/cli/cmds/migrate.go +++ b/cli/cmds/migrate.go @@ -21,7 +21,7 @@ func init() { log.Fatalf("setting up database failed: %s", err) } - err = migrations.RunSQLite(database) + err = migrations.RunPostgres(database) if err != nil { log.Fatalf("migrating database failed: %s", err) } diff --git a/db/migrations/migrate.go b/db/migrations/migrate.go index 75affbb..f5bc917 100644 --- a/db/migrations/migrate.go +++ b/db/migrations/migrate.go @@ -7,25 +7,6 @@ import ( "github.com/lopezator/migrator" ) -func RunSQLite(db *sql.DB) error { - // Configure migrations - - m, err := migrator.New( - // type cast is required because []*migrator.MigrationNoTx is not assignable to []migrator.Migration - migrator.Migrations(migrationsSQLite...), - ) - if err != nil { - return fmt.Errorf("initializing migrations failed: %w", err) - } - - // Migrate up - if err := m.Migrate(db); err != nil { - return fmt.Errorf("running migrations failed: %w", err) - } - - return nil -} - func RunPostgres(db *sql.DB) error { // Configure migrations @@ -57,34 +38,6 @@ func newMigration(name string, query string) *migrator.MigrationNoTx { } } -var migrationsSQLite = []any{ - newMigration("2023-09-18-00000001-CreateTableDomains", `create table main.domains -( - id integer not null - constraint domains_pk - primary key autoincrement, - name varchar(128) not null - constraint domains_pk2 - unique -)`, - ), - newMigration("2023-09-18-00000002-CreateTablePurls", `create table purls -( - id integer not null - constraint puls_pk - primary key autoincrement, - domain_id integer not null - constraint purls_domains_id_fk - references domains - on delete restrict, - name varchar(128) not null, - target varchar(4096) not null, - constraint purls_pk - unique (domain_id, name) -)`, - ), -} - var migrationsPostgres = []any{ newMigration("2023-09-18-00000001-CreateTableDomains", `create table domains ( From 43017535239305d061da75ec49443498e9ab8968 Mon Sep 17 00:00:00 2001 From: fabiante Date: Tue, 19 Sep 2023 09:04:24 +0200 Subject: [PATCH 25/27] Wrap table emptying into transaction --- db/empty.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/db/empty.go b/db/empty.go index 26a8d61..50672bf 100644 --- a/db/empty.go +++ b/db/empty.go @@ -9,12 +9,12 @@ import ( // EmptyTables is used to empty a collection of tables. This may be useful if truncating a // table is not possible. func EmptyTables(db *goqu.Database, tables ...string) error { - var errs []error - - for _, table := range tables { - _, err := db.Delete(table).Executor().Exec() - errs = append(errs, err) - } - - return errors.Join(errs...) + return db.WithTx(func(db *goqu.TxDatabase) error { + var errs []error + for _, table := range tables { + _, err := db.Delete(table).Executor().Exec() + errs = append(errs, err) + } + return errors.Join(errs...) + }) } From 052996cf7752e87fac085e27397a39c7c88636b8 Mon Sep 17 00:00:00 2001 From: fabiante Date: Tue, 19 Sep 2023 09:12:35 +0200 Subject: [PATCH 26/27] Remove unused env variables --- config/env.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/config/env.go b/config/env.go index 34a23d2..cf0798d 100644 --- a/config/env.go +++ b/config/env.go @@ -25,21 +25,6 @@ func LoadEnv() { } } -func DataDir() string { - dataDir := os.Getenv(" PERSURL_DATA_DIR") - if dataDir == "" { - dataDir = "." - } - log.Printf("using data dir: %s", dataDir) - return dataDir -} - -func DbFile(dataDir string) string { - dbFile := fmt.Sprintf("%s/prod.sqlite", dataDir) - log.Printf("using database file: %s", dbFile) - return dbFile -} - func DbDSN() string { dsn := os.Getenv("PERSURL_DB_DSN") if dsn == "" { From e4656dbbdbea582dbbee7ea3314d7b9b689bacb1 Mon Sep 17 00:00:00 2001 From: fabiante Date: Tue, 19 Sep 2023 09:13:52 +0200 Subject: [PATCH 27/27] Add support for DATABASE_URL env variable This will be used for deployment on Railway where the pg database's url is automatically added with this key. --- config/env.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/env.go b/config/env.go index cf0798d..bdeff3c 100644 --- a/config/env.go +++ b/config/env.go @@ -27,6 +27,9 @@ func LoadEnv() { func DbDSN() string { dsn := os.Getenv("PERSURL_DB_DSN") + if dsn == "" { + dsn = os.Getenv("DATABASE_URL") + } if dsn == "" { log.Fatalf("persurl db dsn may not be empty") }