Run system tests in parallel

go.mod

@@ -184,3 +184,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/elastic/package-spec/v3 => github.com/mrodm/package-spec/v3 v3.0.0-20240613145150-8d065e83d217

go.sum

@@ -102,8 +102,6 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul
 github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg=
 github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU=
 github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI=
-github.com/elastic/package-spec/v3 v3.1.5 h1:BoUvKZI3WhbxsWAvsOqIasgpCo8fmjmj7lZgO1JyB2s=
-github.com/elastic/package-spec/v3 v3.1.5/go.mod h1:kHltfTMhDn9Whp3LZluVwI0WFW5uHwqtlByx6+c8ZHE=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
@@ -316,6 +314,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mrodm/package-spec/v3 v3.0.0-20240613145150-8d065e83d217 h1:Ywx/eMe8dX0yx3xf7nWwlHMp6Bp+NyBqXtl95KvNpXU=
+github.com/mrodm/package-spec/v3 v3.0.0-20240613145150-8d065e83d217/go.mod h1:Qdq/tqETSbbTpCfUAlVTtIvO2RiAdXp6aXTbqeGdHfE=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=

internal/compose/compose.go

@@ -370,29 +370,33 @@ func (p *Project) WaitForHealthy(ctx context.Context, opts CommandOptions) error
 		}
 
 		for _, containerDescription := range descriptions {
-			logger.Debugf("Container status: %s", containerDescription.String())
 
 			// No healthcheck defined for service
 			if containerDescription.State.Status == "running" && containerDescription.State.Health == nil {
+				logger.Debugf("Container %s status: %s (no health status)", containerDescription.ID, containerDescription.State.Status)
 				continue
 			}
 
 			// Service is up and running and it's healthy
 			if containerDescription.State.Status == "running" && containerDescription.State.Health.Status == "healthy" {
+				logger.Debugf("Container %s status: %s (health: %s)", containerDescription.ID, containerDescription.State.Status, containerDescription.State.Health.Status)
 				continue
 			}
 
 			// Container started and finished with exit code 0
 			if containerDescription.State.Status == "exited" && containerDescription.State.ExitCode == 0 {
+				logger.Debugf("Container %s status: %s (exit code: %d)", containerDescription.ID, containerDescription.State.Status, containerDescription.State.ExitCode)
 				continue
 			}
 
 			// Container exited with code > 0
 			if containerDescription.State.Status == "exited" && containerDescription.State.ExitCode > 0 {
+				logger.Debugf("Container %s status: %s (exit code: %d)", containerDescription.ID, containerDescription.State.Status, containerDescription.State.ExitCode)
 				return fmt.Errorf("container (ID: %s) exited with code %d", containerDescription.ID, containerDescription.State.ExitCode)
 			}
 
 			// Any different status is considered unhealthy
+			logger.Debugf("Container %s status: unhealthy", containerDescription.ID)
 			healthy = false
 		}
 

internal/kibana/agents.go

@@ -118,7 +118,8 @@ func (c *Client) waitUntilPolicyAssigned(ctx context.Context, a Agent, p Policy)
 		if err != nil {
 			return fmt.Errorf("can't get the agent: %w", err)
 		}
-		logger.Debugf("Agent data: %s", agent.String())
+		logger.Debugf("Agent %s (Host: %s): Policy ID %s LogLevel: %s Status: %s",
+			agent.ID, agent.LocalMetadata.Host.Name, agent.PolicyID, agent.LocalMetadata.Elastic.Agent.LogLevel, agent.Status)
 
 		if agent.PolicyID == p.ID && agent.PolicyRevision >= p.Revision {
 			logger.Debugf("Policy revision assigned to the agent (ID: %s)...", a.ID)

internal/servicedeployer/compose.go

@@ -124,7 +124,7 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic
 	} else {
 		err = p.Up(ctx, opts)
 		if err != nil {
-			return nil, fmt.Errorf("could not boot up service using Docker Compose: %w", err)
+			return &service, fmt.Errorf("could not boot up service using Docker Compose: %w", err)
 		}
 	}
 
@@ -133,7 +133,7 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic
 		processServiceContainerLogs(context.WithoutCancel(ctx), p, compose.CommandOptions{
 			Env: opts.Env,
 		}, svcInfo.Name)
-		return nil, fmt.Errorf("service is unhealthy: %w", err)
+		return &service, fmt.Errorf("service is unhealthy: %w", err)
 	}
 
 	// Added a specific alias when connecting the service to the network.

internal/servicedeployer/custom_agent.go

@@ -169,7 +169,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D
 		processServiceContainerLogs(ctx, p, compose.CommandOptions{
 			Env: opts.Env,
 		}, svcInfo.Name)
-		return nil, fmt.Errorf("service is unhealthy: %w", err)
+		return &service, fmt.Errorf("service is unhealthy: %w", err)
 	}
 
 	logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName))

internal/servicedeployer/terraform.go

@@ -149,7 +149,7 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn
 	}
 	err = p.Up(ctx, opts)
 	if err != nil {
-		return nil, fmt.Errorf("could not boot up service using Docker Compose: %w", err)
+		return &service, fmt.Errorf("could not boot up service using Docker Compose: %w", err)
 	}
 
 	err = p.WaitForHealthy(ctx, opts)
@@ -158,7 +158,7 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn
 			Env: opts.Env,
 		}, svcInfo.Name)
 		//lint:ignore ST1005 error starting with product name can be capitalized
-		return nil, fmt.Errorf("Terraform deployer is unhealthy: %w", err)
+		return &service, fmt.Errorf("Terraform deployer is unhealthy: %w", err)
 	}
 
 	svcInfo.Agent.Host.NamePrefix = "docker-fleet-agent"

internal/testrunner/globaltestconfig.go

@@ -0,0 +1,47 @@
+package testrunner
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/elastic/go-ucfg"
+	"github.com/elastic/go-ucfg/yaml"
+)
+
+type GlobalTestConfig struct {
+	Asset    GlobalRunnerTestConfig `config:"asset"`
+	Pipeline GlobalRunnerTestConfig `config:"pipeline"`
+	Policy   GlobalRunnerTestConfig `config:"policy"`
+	Static   GlobalRunnerTestConfig `config:"static"`
+	System   GlobalRunnerTestConfig `config:"system"`
+}
+
+type GlobalRunnerTestConfig struct {
+	Parallel        bool `config:"parallel"`
+	SkippableConfig `config:",inline"`
+}
+
+func AGlobalTestConfig(packageRootPath string) (*GlobalTestConfig, error) {
+	configFilePath := filepath.Join(packageRootPath, "_dev", "test", "config.yml")
+
+	data, err := os.ReadFile(configFilePath)
+	if errors.Is(err, os.ErrNotExist) {
+		return &GlobalTestConfig{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s: %w", configFilePath, err)
+	}
+
+	var c GlobalTestConfig
+	cfg, err := yaml.NewConfig(data, ucfg.PathSep("."))
+	if err != nil {
+		return nil, fmt.Errorf("unable to load global test configuration file: %s: %w", configFilePath, err)
+	}
+	if err := cfg.Unpack(&c); err != nil {
+		return nil, fmt.Errorf("unable to unpack global test configuration file: %s: %w", configFilePath, err)
+	}
+
+	return &c, nil
+}

internal/testrunner/runners/asset/tester.go

@@ -57,6 +57,11 @@ func (r tester) String() string {
 	return "asset loading"
 }
 
+// Parallel indicates if this tester can run in parallel or not.
+func (r tester) Parallel() bool {
+	return false
+}
+
 // Run runs the asset loading tests
 func (r *tester) Run(ctx context.Context) ([]testrunner.TestResult, error) {
 	return r.run(ctx)

internal/testrunner/runners/pipeline/tester.go

@@ -127,6 +127,11 @@ func (r *tester) String() string {
 	return "pipeline"
 }
 
+// Parallel indicates if this tester can run in parallel or not.
+func (r tester) Parallel() bool {
+	return false
+}
+
 // Run runs the pipeline tests defined under the given folder
 func (r *tester) Run(ctx context.Context) ([]testrunner.TestResult, error) {
 	return r.run(ctx)

internal/testrunner/runners/policy/tester.go

@@ -59,6 +59,11 @@ func (r *tester) String() string {
 	return string(TestType)
 }
 
+// Parallel indicates if this tester can run in parallel or not.
+func (r tester) Parallel() bool {
+	return false
+}
+
 func (r *tester) Run(ctx context.Context) ([]testrunner.TestResult, error) {
 	var results []testrunner.TestResult
 

internal/testrunner/runners/static/tester.go

@@ -47,6 +47,11 @@ func (r tester) String() string {
 	return "static files"
 }
 
+// Parallel indicates if this tester can run in parallel or not.
+func (r tester) Parallel() bool {
+	return false
+}
+
 func (r tester) Run(ctx context.Context) ([]testrunner.TestResult, error) {
 	return r.run(ctx)
 }