From 6ad52b5e74c67ec6ef484058ea044229439be5e4 Mon Sep 17 00:00:00 2001 From: tamirms Date: Tue, 24 Dec 2024 09:25:22 +0000 Subject: [PATCH] Fix tests --- .../docker/captive-core-integration-tests.cfg | 19 -- ...ive-core-integration-tests.soroban-rpc.cfg | 20 -- ...-core-reingest-range-integration-tests.cfg | 12 - ...-compose.integration-tests.soroban-rpc.yml | 20 -- .../docker-compose.integration-tests.yml | 29 ++- ...stellar-core-classic-integration-tests.cfg | 25 -- .../docker/stellar-core-integration-tests.cfg | 30 --- .../horizon/internal/integration/db_test.go | 10 - .../integration/extend_footprint_ttl_test.go | 1 + .../horizon/internal/integration/load_test.go | 4 +- .../internal/integration/parameters_test.go | 53 +---- .../horizon/internal/integration/sac_test.go | 10 + .../internal/test/integration/core_config.go | 69 ++++++ .../internal/test/integration/integration.go | 224 ++++++++++-------- 14 files changed, 230 insertions(+), 296 deletions(-) delete mode 100644 services/horizon/docker/captive-core-integration-tests.cfg delete mode 100644 services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg delete mode 100644 services/horizon/docker/captive-core-reingest-range-integration-tests.cfg delete mode 100644 services/horizon/docker/docker-compose.integration-tests.soroban-rpc.yml delete mode 100644 services/horizon/docker/stellar-core-classic-integration-tests.cfg delete mode 100644 services/horizon/docker/stellar-core-integration-tests.cfg create mode 100644 services/horizon/internal/test/integration/core_config.go diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg deleted file mode 100644 index 296fc3f819..0000000000 --- a/services/horizon/docker/captive-core-integration-tests.cfg +++ /dev/null @@ -1,19 +0,0 @@ -PEER_PORT=11725 -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true -# Lower the TTL of persistent ledger entries -# so that ledger entry extension/restoring becomes testeable -# TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -# TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="localhost" -QUALITY="MEDIUM" diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg deleted file mode 100644 index 744cb07985..0000000000 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ /dev/null @@ -1,20 +0,0 @@ -DEPRECATED_SQL_LEDGER_STATE=false -PEER_PORT=11725 -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true -# Lower the TTL of persistent ledger entries -# so that ledger entry extension/restoring becomes testeable -#TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -#TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="core" -QUALITY="MEDIUM" diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg deleted file mode 100644 index 44820f5933..0000000000 --- a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg +++ /dev/null @@ -1,12 +0,0 @@ -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true -TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true -TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="localhost" -QUALITY="MEDIUM" diff --git a/services/horizon/docker/docker-compose.integration-tests.soroban-rpc.yml b/services/horizon/docker/docker-compose.integration-tests.soroban-rpc.yml deleted file mode 100644 index 940c340a40..0000000000 --- a/services/horizon/docker/docker-compose.integration-tests.soroban-rpc.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '3' -services: - soroban-rpc: - platform: linux/amd64 - image: ${SOROBAN_RPC_IMAGE:-stellar/soroban-rpc} - depends_on: - - core - restart: on-failure - ports: - - "8080:8080" - environment: - - ENDPOINT=:8080 - - NETWORK_PASSPHRASE=Standalone Network ; February 2017 - - CAPTIVE_CORE_CONFIG_PATH=/captive-core.cfg - - HISTORY_ARCHIVE_URLS=http://core:1570 - - CHECKPOINT_FREQUENCY=8 - - LOG_LEVEL=debug - volumes: - - ./captive-core-integration-tests.soroban-rpc.cfg:/captive-core.cfg - diff --git a/services/horizon/docker/docker-compose.integration-tests.yml b/services/horizon/docker/docker-compose.integration-tests.yml index 630e8b6b60..fb60f92eb0 100644 --- a/services/horizon/docker/docker-compose.integration-tests.yml +++ b/services/horizon/docker/docker-compose.integration-tests.yml @@ -1,23 +1,11 @@ version: '3' services: - core-postgres: - image: postgres:9.6.17-alpine - restart: on-failure - environment: - - POSTGRES_PASSWORD=mysecretpassword - - POSTGRES_DB=stellar - ports: - - "5641:5641" - command: ["-p", "5641"] core: platform: linux/amd64 # Note: Please keep the image pinned to an immutable tag matching the Captive Core version. # This avoid implicit updates which break compatibility between # the Core container and captive core. image: ${CORE_IMAGE:-stellar/stellar-core:19.13.1-1481.3acf6dd26.focal} - - depends_on: - - core-postgres restart: on-failure environment: - TRACY_NO_INVARIANT_CHECK=1 @@ -29,5 +17,20 @@ services: entrypoint: /usr/bin/env command: /start standalone volumes: - - ./${CORE_CONFIG_FILE:-stellar-core-integration-tests.cfg}:/stellar-core.cfg + - ${CORE_CONFIG_FILE:-stellar-core.cfg}:/stellar-core.cfg - ./core-start.sh:/start + soroban-rpc: + platform: linux/amd64 + image: ${SOROBAN_RPC_IMAGE:-stellar/soroban-rpc} + restart: on-failure + ports: + - "8080:8080" + environment: + - ENDPOINT=:8080 + - NETWORK_PASSPHRASE=Standalone Network ; February 2017 + - CAPTIVE_CORE_CONFIG_PATH=/captive-core.cfg + - HISTORY_ARCHIVE_URLS=http://core:1570 + - CHECKPOINT_FREQUENCY=8 + - LOG_LEVEL=debug + volumes: + - ${CAPTIVE_CORE_CONFIG_FILE:-stellar-core.cfg}:/captive-core.cfg diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg deleted file mode 100644 index fe23e94e8d..0000000000 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ /dev/null @@ -1,25 +0,0 @@ -DEPRECATED_SQL_LEDGER_STATE=false -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -NETWORK_PASSPHRASE="Standalone Network ; February 2017" - -PEER_PORT=11625 -HTTP_PORT=11626 -PUBLIC_HTTP_PORT=true - -NODE_SEED="SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" - -NODE_IS_VALIDATOR=true -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" - -[QUORUM_SET] -THRESHOLD_PERCENT=100 -VALIDATORS=["GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"] - -[HISTORY.vs] -get="cp history/vs/{0} {1}" -put="cp {0} history/vs/{1}" -mkdir="mkdir -p history/vs/{0}" \ No newline at end of file diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg deleted file mode 100644 index aa1b868471..0000000000 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ /dev/null @@ -1,30 +0,0 @@ -DEPRECATED_SQL_LEDGER_STATE=false -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -NETWORK_PASSPHRASE="Standalone Network ; February 2017" - -PEER_PORT=11625 -HTTP_PORT=11626 -PUBLIC_HTTP_PORT=true - -NODE_SEED="SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" - -NODE_IS_VALIDATOR=true -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" - -# Lower the TTL of persistent ledger entries -# so that ledger entry extension/restoring becomes testeable -# TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -# TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - -[QUORUM_SET] -THRESHOLD_PERCENT=100 -VALIDATORS=["GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"] - -[HISTORY.vs] -get="cp history/vs/{0} {1}" -put="cp {0} history/vs/{1}" -mkdir="mkdir -p history/vs/{0}" diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 7d85289852..2a60bcf660 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -531,11 +531,6 @@ func TestReingestDB(t *testing.T) { // subprocesses to conflict. itest.StopHorizon() - horizonConfig.CaptiveCoreConfigPath = filepath.Join( - filepath.Dir(horizonConfig.CaptiveCoreConfigPath), - "captive-core-reingest-range-integration-tests.cfg", - ) - var rootCmd = horizoncmd.NewRootCmd() rootCmd.SetArgs(command(t, horizonConfig, "db", "reingest", @@ -909,11 +904,6 @@ func TestFillGaps(t *testing.T) { _, err = historyQ.DeleteRangeAll(context.Background(), oldestLedger, latestLedger) tt.NoError(err) - horizonConfig.CaptiveCoreConfigPath = filepath.Join( - filepath.Dir(horizonConfig.CaptiveCoreConfigPath), - "captive-core-reingest-range-integration-tests.cfg", - ) - rootCmd := horizoncmd.NewRootCmd() rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "--parallel-workers=1")) tt.NoError(rootCmd.Execute()) diff --git a/services/horizon/internal/integration/extend_footprint_ttl_test.go b/services/horizon/internal/integration/extend_footprint_ttl_test.go index cc6f947a14..497b492bdd 100644 --- a/services/horizon/internal/integration/extend_footprint_ttl_test.go +++ b/services/horizon/internal/integration/extend_footprint_ttl_test.go @@ -18,6 +18,7 @@ func TestExtendFootprintTtl(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) // establish which account will be contract owner, and load it's current seq diff --git a/services/horizon/internal/integration/load_test.go b/services/horizon/internal/integration/load_test.go index deb497160c..69ddf0f79d 100644 --- a/services/horizon/internal/integration/load_test.go +++ b/services/horizon/internal/integration/load_test.go @@ -34,8 +34,8 @@ type sorobanTransaction struct { func TestLoad(t *testing.T) { var transactionsPerLedger, ledgers int - flag.IntVar(&transactionsPerLedger, "transactions-per-ledger", 4000, "number of transactions per ledger") - flag.IntVar(&ledgers, "ledgers", 10, "number of ledgers to generate") + flag.IntVar(&transactionsPerLedger, "transactions-per-ledger", 100, "number of transactions per ledger") + flag.IntVar(&ledgers, "ledgers", 2, "number of ledgers to generate") flag.Parse() itest := integration.NewTest(t, integration.Config{ diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 7ded1f9a88..25b2871db5 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -27,11 +27,6 @@ import ( "github.com/stretchr/testify/assert" ) -var defaultCaptiveCoreParameters = map[string]string{ - horizon.StellarCoreBinaryPathName: os.Getenv("CAPTIVE_CORE_BIN"), - horizon.StellarCoreURLFlagName: "", -} - var networkParamArgs = map[string]string{ horizon.CaptiveCoreConfigPathName: "", horizon.CaptiveCoreHTTPPortFlagName: "", @@ -41,29 +36,6 @@ var networkParamArgs = map[string]string{ horizon.NetworkPassphraseFlagName: "", } -var ( - CaptiveCoreConfigErrMsg = "error generating captive core configuration: invalid config: " -) - -// Ensures that BUCKET_DIR_PATH is not an allowed value for Captive Core. -func TestBucketDirDisallowed(t *testing.T) { - config := `BUCKET_DIR_PATH="/tmp" - ` + integration.SimpleCaptiveCoreToml - - confName, _, cleanup := integration.CreateCaptiveCoreConfig(config) - defer cleanup() - testConfig := integration.GetTestConfig() - testConfig.HorizonIngestParameters = map[string]string{ - horizon.CaptiveCoreConfigPathName: confName, - horizon.StellarCoreBinaryPathName: os.Getenv("CAPTIVE_CORE_BIN"), - } - test := integration.NewTest(t, *testConfig) - err := test.StartHorizon(true) - assert.Equal(t, err.Error(), integration.HorizonInitErrStr+": error generating captive core configuration:"+ - " invalid captive core toml file: could not unmarshal captive core toml: setting BUCKET_DIR_PATH is disallowed"+ - " for Captive Core, use CAPTIVE_CORE_STORAGE_PATH instead") -} - func TestEnvironmentPreserved(t *testing.T) { // Who tests the tests? This test. // @@ -234,28 +206,9 @@ func TestNetworkEnvironmentVariable(t *testing.T) { // Ensures that the filesystem ends up in the correct state with Captive Core. func TestCaptiveCoreConfigFilesystemState(t *testing.T) { - confName, storagePath, cleanup := integration.CreateCaptiveCoreConfig(integration.SimpleCaptiveCoreToml) - defer cleanup() - - localParams := integration.MergeMaps(defaultCaptiveCoreParameters, map[string]string{ - "captive-core-storage-path": storagePath, - horizon.CaptiveCoreConfigPathName: confName, - }) - testConfig := integration.GetTestConfig() - testConfig.HorizonIngestParameters = localParams - test := integration.NewTest(t, *testConfig) - - err := test.StartHorizon(true) - assert.NoError(t, err) - test.WaitForHorizonIngest() - - t.Run("disk state", func(t *testing.T) { - validateCaptiveCoreDiskState(test, storagePath) - }) - - t.Run("no bucket dir", func(t *testing.T) { - validateNoBucketDirPath(test, storagePath) - }) + test := integration.NewTest(t, integration.Config{}) + validateCaptiveCoreDiskState(test, test.CaptiveCoreStoragePath()) + validateNoBucketDirPath(test, test.CaptiveCoreStoragePath()) } func TestMaxAssetsForPathRequests(t *testing.T) { diff --git a/services/horizon/internal/integration/sac_test.go b/services/horizon/internal/integration/sac_test.go index 4a9db284bf..6fb5552d9c 100644 --- a/services/horizon/internal/integration/sac_test.go +++ b/services/horizon/internal/integration/sac_test.go @@ -42,6 +42,7 @@ func TestContractMintToAccount(t *testing.T) { itest := integration.NewTest(t, integration.Config{ HorizonEnvironment: map[string]string{"INGEST_DISABLE_STATE_VERIFICATION": "true", "CONNECTION_TIMEOUT": "360000"}, EnableSorobanRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -145,6 +146,7 @@ func TestContractMintToContract(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -230,6 +232,7 @@ func TestExpirationAndRestoration(t *testing.T) { // want state verification to detect this "ingest-disable-state-verification": "true", }, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -502,6 +505,7 @@ func TestContractTransferBetweenAccounts(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -576,6 +580,7 @@ func TestContractTransferBetweenAccountAndContract(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -696,6 +701,7 @@ func TestContractTransferBetweenContracts(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -777,6 +783,7 @@ func TestContractBurnFromAccount(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -852,6 +859,7 @@ func TestContractBurnFromContract(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -919,6 +927,7 @@ func TestContractClawbackFromAccount(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) // Give the master account the revocable flag (needed to set the clawback flag) @@ -996,6 +1005,7 @@ func TestContractClawbackFromContract(t *testing.T) { itest := integration.NewTest(t, integration.Config{ EnableSorobanRPC: true, + QuickExpiration: true, }) // Give the master account the revocable flag (needed to set the clawback flag) diff --git a/services/horizon/internal/test/integration/core_config.go b/services/horizon/internal/test/integration/core_config.go new file mode 100644 index 0000000000..2999ec1efe --- /dev/null +++ b/services/horizon/internal/test/integration/core_config.go @@ -0,0 +1,69 @@ +package integration + +type validatorCoreConfigTemplatePrams struct { + Accelerate bool + NetworkPassphrase string + TestingMinimumPersistentEntryLifetime int + TestingSorobanHighLimitOverride bool +} + +type captiveCoreConfigTemplatePrams struct { + validatorCoreConfigTemplatePrams + ValidatorAddress string +} + +const validatorCoreConfigTemplate = ` +DEPRECATED_SQL_LEDGER_STATE=false +ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING={{ .Accelerate }} + +NETWORK_PASSPHRASE="{{ .NetworkPassphrase }}" + +TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME={{ .TestingMinimumPersistentEntryLifetime }} +TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE={{ .TestingSorobanHighLimitOverride }} + +PEER_PORT=11625 +HTTP_PORT=11626 +PUBLIC_HTTP_PORT=true + +NODE_SEED="SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" + +NODE_IS_VALIDATOR=true +UNSAFE_QUORUM=true +FAILURE_SAFETY=0 + +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +DATABASE = "sqlite3://stellar.db" + +[QUORUM_SET] +THRESHOLD_PERCENT=100 +VALIDATORS=["GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"] + +[HISTORY.vs] +get="cp history/vs/{0} {1}" +put="cp {0} history/vs/{1}" +mkdir="mkdir -p history/vs/{0}" +` + +const captiveCoreConfigTemplate = ` +DEPRECATED_SQL_LEDGER_STATE=false +ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING={{ .Accelerate }} + +NETWORK_PASSPHRASE="{{ .NetworkPassphrase }}" + +TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME={{ .TestingMinimumPersistentEntryLifetime }} +TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE={{ .TestingSorobanHighLimitOverride }} +ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true + +PEER_PORT=11725 + +UNSAFE_QUORUM=true +FAILURE_SAFETY=0 + +[[VALIDATORS]] +NAME="local_core" +HOME_DOMAIN="core.local" +# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" +PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" +ADDRESS="{{ .ValidatorAddress }}" +QUALITY="MEDIUM" +` diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index c6899f6f10..0d76b9a756 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -14,6 +14,7 @@ import ( "sync" "syscall" "testing" + "text/template" "time" "github.com/stellar/go/historyarchive" @@ -54,27 +55,6 @@ const ( ) const ( - SimpleCaptiveCoreToml = ` - PEER_PORT=11725 - ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - NETWORK_PASSPHRASE = "Standalone Network ; February 2017" - UNSAFE_QUORUM=true - FAILURE_SAFETY=0 - RUN_STANDALONE=false - - # Lower the TTL of persistent ledger entries - # so that ledger entry extension/restoring becomes testeable - # These 2 settings need to be present in both places - stellar-core-integration-tests.cfg and here - TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 - TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - - [[VALIDATORS]] - NAME="local_core" - HOME_DOMAIN="core.local" - PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" - ADDRESS="localhost" - QUALITY="MEDIUM" -` StellarCoreURL = "http://localhost:11626" ) @@ -88,6 +68,7 @@ type Config struct { CoreDockerImage string SorobanRPCDockerImage string SkipProtocolUpgrade bool + QuickExpiration bool // Weird naming here because bools default to false, but we want to start // Horizon by default. @@ -111,13 +92,14 @@ type CaptiveConfig struct { binaryPath string configPath string storagePath string - useDB bool } type Test struct { t *testing.T - composePath string + composePath string + validatorConfPath string + rpcCoreConfPath string config Config coreConfig CaptiveConfig @@ -176,6 +158,16 @@ func NewTest(t *testing.T, config Config) *Test { config.ProtocolVersion = maxSupportedCoreProtocolFromEnv } } + validatorParams := validatorCoreConfigTemplatePrams{ + Accelerate: CheckpointFrequency < historyarchive.DefaultCheckpointFrequency, + NetworkPassphrase: StandaloneNetworkPassphrase, + TestingMinimumPersistentEntryLifetime: 65536, + TestingSorobanHighLimitOverride: false, + } + if config.QuickExpiration { + validatorParams.TestingSorobanHighLimitOverride = true + validatorParams.TestingMinimumPersistentEntryLifetime = 10 + } var i *Test if !config.SkipCoreContainerCreation { composePath := findDockerComposePath() @@ -186,9 +178,13 @@ func NewTest(t *testing.T, config Config) *Test { passPhrase: StandaloneNetworkPassphrase, environment: test.NewEnvironmentManager(), } - i.configureCaptiveCore() + i.validatorConfPath = i.createCoreValidatorConf(validatorParams) + i.rpcCoreConfPath = i.createCaptiveCoreConf(captiveCoreConfigTemplatePrams{ + validatorCoreConfigTemplatePrams: validatorParams, + ValidatorAddress: "core", + }) // Only run Stellar Core container and its dependencies. - i.runComposeCommand("up", "--detach", "--quiet-pull", "--no-color", "core") + i.startCoreValidator() } else { i = &Test{ t: t, @@ -196,13 +192,14 @@ func NewTest(t *testing.T, config Config) *Test { environment: test.NewEnvironmentManager(), } } + i.configureCaptiveCore(validatorParams) i.prepareShutdownHandlers() i.coreClient = &stellarcore.Client{URL: "http://localhost:" + strconv.Itoa(StellarCorePort)} if !config.SkipCoreContainerCreation { i.waitForCore() if i.config.EnableSorobanRPC { - i.runComposeCommand("up", "--detach", "--quiet-pull", "--no-color", "soroban-rpc") + i.startRPC() i.waitForSorobanRPC() } } @@ -218,12 +215,13 @@ func NewTest(t *testing.T, config Config) *Test { return i } -func (i *Test) configureCaptiveCore() { - composePath := findDockerComposePath() +func (i *Test) configureCaptiveCore(validatorParams validatorCoreConfigTemplatePrams) { i.coreConfig.binaryPath = os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN") - i.coreConfig.configPath = filepath.Join(composePath, "captive-core-integration-tests.cfg") + i.coreConfig.configPath = i.createCaptiveCoreConf(captiveCoreConfigTemplatePrams{ + validatorCoreConfigTemplatePrams: validatorParams, + ValidatorAddress: "localhost", + }) i.coreConfig.storagePath = i.CurrentTest().TempDir() - i.coreConfig.useDB = true if value := i.getIngestParameter( horizon.StellarCoreBinaryPathName, @@ -249,57 +247,121 @@ func (i *Test) getIngestParameter(argName, envName string) string { return "" } +func (i *Test) createCoreValidatorConf(params validatorCoreConfigTemplatePrams) string { + tomlFile, err := os.CreateTemp("", "stellar-core-integration-test-*.toml") + require.NoError(i.t, err) + + tmpl, err := template.New("core-validator").Parse(validatorCoreConfigTemplate) + require.NoError(i.t, err) + err = tmpl.Execute(tomlFile, params) + require.NoError(i.t, err) + + require.NoError(i.t, tomlFile.Close()) + + i.t.Cleanup(func() { + require.NoError(i.t, os.Remove(tomlFile.Name())) + }) + return tomlFile.Name() +} + +func (i *Test) createCaptiveCoreConf(params captiveCoreConfigTemplatePrams) string { + tomlFile, err := os.CreateTemp("", "captive-core-integration-test-*.toml") + require.NoError(i.t, err) + + tmpl, err := template.New("captive-core").Parse(captiveCoreConfigTemplate) + require.NoError(i.t, err) + err = tmpl.Execute(tomlFile, params) + require.NoError(i.t, err) + + require.NoError(i.t, tomlFile.Close()) + + i.t.Cleanup(func() { + require.NoError(i.t, os.Remove(tomlFile.Name())) + }) + return tomlFile.Name() +} + // Runs a docker-compose command applied to the above configs -func (i *Test) runComposeCommand(args ...string) { - integrationYaml := filepath.Join(i.composePath, "docker-compose.integration-tests.yml") - integrationSorobanRPCYaml := filepath.Join(i.composePath, "docker-compose.integration-tests.soroban-rpc.yml") +func (i *Test) runComposeCommand(envVars []string, args ...string) { + cmdline := append( + []string{ + "compose", + "-f", + filepath.Join(i.composePath, "docker-compose.integration-tests.yml"), + }, + args..., + ) + cmd := exec.Command("docker", cmdline...) + + cmd.Env = append( + envVars, + fmt.Sprintf("CAPTIVE_CORE_CONFIG_FILE=%s", i.rpcCoreConfPath), + fmt.Sprintf("CORE_CONFIG_FILE=%s", i.validatorConfPath), + ) + + i.t.Log("Running", cmd.Args) + out, innerErr := cmd.Output() + if len(out) > 0 { + fmt.Printf("stdout:\n%s\n", string(out)) + } + if exitErr, ok := innerErr.(*exec.ExitError); ok { + fmt.Printf("stderr:\n%s\n", string(exitErr.Stderr)) + } - cmdline := args - if i.config.EnableSorobanRPC { - cmdline = append([]string{"-f", integrationSorobanRPCYaml}, cmdline...) + if innerErr != nil { + i.t.Fatalf("Compose command failed: %v", innerErr) } - cmdline = append([]string{"-f", integrationYaml}, cmdline...) - cmdline = append([]string{"compose"}, cmdline...) - cmd := exec.Command("docker", cmdline...) - coreImageOverride := "" +} + +func (i *Test) startCoreValidator() { + var envVars []string + var coreImageOverride string + if i.config.CoreDockerImage != "" { coreImageOverride = i.config.CoreDockerImage } else if img := os.Getenv("HORIZON_INTEGRATION_TESTS_DOCKER_IMG"); img != "" { coreImageOverride = img } - - cmd.Env = os.Environ() if coreImageOverride != "" { - cmd.Env = append( - cmd.Environ(), + envVars = append( + envVars, fmt.Sprintf("CORE_IMAGE=%s", coreImageOverride), ) } - sorobanRPCOverride := "" + + i.runComposeCommand(envVars, "up", "--detach", "--quiet-pull", "--no-color", "core") +} + +func (i *Test) startRPC() { + var envVars []string + var sorobanRPCOverride string + if i.config.SorobanRPCDockerImage != "" { sorobanRPCOverride = i.config.CoreDockerImage } else if img := os.Getenv("HORIZON_INTEGRATION_TESTS_SOROBAN_RPC_DOCKER_IMG"); img != "" { sorobanRPCOverride = img } if sorobanRPCOverride != "" { - cmd.Env = append( - cmd.Environ(), + envVars = append( + envVars, fmt.Sprintf("SOROBAN_RPC_IMAGE=%s", sorobanRPCOverride), ) } - i.t.Log("Running", cmd.Args) - out, innerErr := cmd.Output() - if len(out) > 0 { - fmt.Printf("stdout:\n%s\n", string(out)) - } - if exitErr, ok := innerErr.(*exec.ExitError); ok { - fmt.Printf("stderr:\n%s\n", string(exitErr.Stderr)) - } + i.runComposeCommand(envVars, "up", "--detach", "--quiet-pull", "--no-color", "soroban-rpc") +} - if innerErr != nil { - i.t.Fatalf("Compose command failed: %v", innerErr) - } +func (i *Test) removeContainers(containers ...string) { + i.runComposeCommand( + nil, + append( + []string{ + "rm", + "-fvs", + }, + containers..., + )..., + ) } func (i *Test) prepareShutdownHandlers() { @@ -314,14 +376,13 @@ func (i *Test) prepareShutdownHandlers() { if !i.config.SkipCoreContainerCreation { if !i.config.SkipCoreContainerDeletion { i.t.Log("Removing core docker containers...") - i.runComposeCommand("rm", "-fvs", "core") - i.runComposeCommand("rm", "-fvs", "core-postgres") + i.removeContainers("core") } else { i.t.Log("Skip core docker container removal for debugging...") } if i.config.EnableSorobanRPC { - i.runComposeCommand("logs", "soroban-rpc") - i.runComposeCommand("rm", "-fvs", "soroban-rpc") + i.runComposeCommand(nil, "logs", "soroban-rpc") + i.removeContainers("soroban-rpc") } } }, @@ -588,45 +649,14 @@ func (i *Test) setupHorizonClient(webArgs map[string]string) { } } -// CreateCaptiveCoreConfig will create a temporary TOML config with the -// specified contents as well as a temporary storage directory. You should -// `defer` the returned function to clean these up when you're done. -func CreateCaptiveCoreConfig(contents string) (string, string, func()) { - tomlFile, err := ioutil.TempFile("", "captive-core-test-*.toml") - if err != nil { - panic(err) - } - defer tomlFile.Close() - - _, err = tomlFile.WriteString(contents) - if err != nil { - panic(err) - } - - storagePath, err := os.MkdirTemp("", "captive-core-test-*-storage") - if err != nil { - panic(err) - } - - filename := tomlFile.Name() - return filename, storagePath, func() { - os.Remove(filename) - os.RemoveAll(storagePath) - } -} - func (i *Test) CreateCaptiveCoreConfig() (*ledgerbackend.CaptiveCoreConfig, error) { - confName, storagePath, cleanupFn := CreateCaptiveCoreConfig(SimpleCaptiveCoreToml) - i.t.Cleanup(cleanupFn) - i.t.Logf("Creating Captive Core config files, ConfName: %v, storagePath: %v", confName, storagePath) - captiveCoreConfig := ledgerbackend.CaptiveCoreConfig{ BinaryPath: i.CoreBinaryPath(), HistoryArchiveURLs: []string{HistoryArchiveUrl}, NetworkPassphrase: StandaloneNetworkPassphrase, CheckpointFrequency: CheckpointFrequency, // This is required for accelerated archive creation for integration test UseDB: true, - StoragePath: storagePath, + StoragePath: i.CurrentTest().TempDir(), } tomlParams := ledgerbackend.CaptiveCoreTomlParams{ @@ -635,7 +665,7 @@ func (i *Test) CreateCaptiveCoreConfig() (*ledgerbackend.CaptiveCoreConfig, erro UseDB: true, } - toml, err := ledgerbackend.NewCaptiveCoreTomlFromData([]byte(SimpleCaptiveCoreToml), tomlParams) + toml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(i.coreConfig.configPath, tomlParams) if err != nil { return nil, err } @@ -1008,6 +1038,10 @@ func (i *Test) CoreBinaryPath() string { return corePath } +func (i *Test) CaptiveCoreStoragePath() string { + return i.coreConfig.storagePath +} + // Client returns horizon.Client connected to started Horizon instance. func (i *Test) Client() *sdk.Client { return i.horizonClient