Move installation package in policy and system tests

cmd/testrunner.go

@@ -168,7 +168,7 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error {
 
 	_, pkg := filepath.Split(packageRootPath)
 
-	runner := asset.NewAssetRunner(asset.AssetRunnerOptions{
+	runner := asset.NewAssetTester(asset.AssetTesterOptions{
 		TestFolder:      testrunner.TestFolder{Package: pkg},
 		PackageRootPath: packageRootPath,
 		KibanaClient:    kibanaClient,
@@ -289,17 +289,18 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error {
 	ctx, stop := signal.Enable(cmd.Context(), logger.Info)
 	defer stop()
 
-	var results []testrunner.TestResult
-	for _, folder := range testFolders {
-		runner := static.NewStaticRunner(static.StaticRunnerOptions{
+	factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
+		runner := static.NewStaticTester(static.StaticTesterOptions{
 			TestFolder:      folder,
 			PackageRootPath: packageRootPath,
 		})
-		r, err := testrunner.Run(ctx, runner)
-		if err != nil {
-			return fmt.Errorf("error running package %s tests: %w", testType, err)
-		}
-		results = append(results, r...)
+
+		return runner, nil
+	}
+
+	results, err := testrunner.RunWithFactory(ctx, testFolders, factory)
+	if err != nil {
+		return err
 	}
 
 	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
@@ -438,9 +439,8 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	var results []testrunner.TestResult
-	for _, folder := range testFolders {
-		runner, err := pipeline.NewPipelineRunner(pipeline.PipelineRunnerOptions{
+	factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
+		runner, err := pipeline.NewPipelineTester(pipeline.PipelineTesterOptions{
 			Profile:            profile,
 			TestFolder:         folder,
 			PackageRootPath:    packageRootPath,
@@ -451,14 +451,14 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error {
 			DeferCleanup:       deferCleanup,
 		})
 		if err != nil {
-			return fmt.Errorf("failed to create pipeline runner: %w", err)
+			return nil, fmt.Errorf("failed to create pipeline runner: %w", err)
 		}
+		return runner, nil
+	}
 
-		r, err := testrunner.Run(ctx, runner)
-		if err != nil {
-			return fmt.Errorf("error running package %s tests: %w", testType, err)
-		}
-		results = append(results, r...)
+	results, err := testrunner.RunWithFactory(ctx, testFolders, factory)
+	if err != nil {
+		return err
 	}
 
 	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
@@ -680,9 +680,16 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		cmd.Printf("Running tests per stages (technical preview)\n")
 	}
 
-	var results []testrunner.TestResult
-	for _, folder := range testFolders {
-		runner := system.NewSystemRunner(system.SystemRunnerOptions{
+	runner := system.NewSystemTestRunner(system.SystemTestRunnerOptions{
+		PackageRootPath: packageRootPath,
+		KibanaClient:    kibanaClient,
+		RunSetup:        runSetup,
+		RunTearDown:     runTearDown,
+		RunTestsOnly:    runTestsOnly,
+	})
+
+	factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
+		runner := system.NewSystemTester(system.SystemTesterOptions{
 			Profile:                    profile,
 			TestFolder:                 folder,
 			PackageRootPath:            packageRootPath,
@@ -698,14 +705,19 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 			RunIndependentElasticAgent: false,
 		})
 
-		r, err := testrunner.Run(ctx, runner)
-		if err != nil {
-			return fmt.Errorf("error running package %s tests: %w", testType, err)
-		}
-		results = append(results, r...)
+		return runner, nil
 	}
 
-	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
+	results, err := testrunner.RunSuite(ctx, testFolders, runner, factory)
+	if err != nil {
+		return err
+	}
+
+	err = processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
+	if err != nil {
+		return fmt.Errorf("failed to process results: %w", err)
+	}
+	return nil
 }
 
 func getTestRunnerPolicyCommand() *cobra.Command {
@@ -832,19 +844,24 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
 
-	var results []testrunner.TestResult
-	for _, folder := range testFolders {
-		runner := policy.NewPolicyRunner(policy.PolicyRunnerOptions{
+	runner := policy.NewPolicyTestRunner(policy.PolicyTestRunnerOptions{
+		PackageRootPath: packageRootPath,
+		KibanaClient:    kibanaClient,
+	})
+
+	factory := func(folder testrunner.TestFolder) (testrunner.Tester, error) {
+		runner := policy.NewPolicyTester(policy.PolicyTesterOptions{
 			TestFolder:         folder,
 			PackageRootPath:    packageRootPath,
 			GenerateTestResult: generateTestResult,
 			KibanaClient:       kibanaClient,
 		})
-		r, err := testrunner.Run(ctx, runner)
-		if err != nil {
-			return fmt.Errorf("error running package %s tests: %w", testType, err)
-		}
-		results = append(results, r...)
+		return runner, nil
+	}
+
+	results, err := testrunner.RunSuite(ctx, testFolders, runner, factory)
+	if err != nil {
+		return err
 	}
 
 	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)

internal/testrunner/runners/asset/runner.go

@@ -29,13 +29,13 @@ type runner struct {
 	resourcesManager *resources.Manager
 }
 
-type AssetRunnerOptions struct {
+type AssetTesterOptions struct {
 	TestFolder      testrunner.TestFolder
 	PackageRootPath string
 	KibanaClient    *kibana.Client
 }
 
-func NewAssetRunner(options AssetRunnerOptions) *runner {
+func NewAssetTester(options AssetTesterOptions) *runner {
 	runner := runner{
 		testFolder:      options.TestFolder,
 		packageRootPath: options.PackageRootPath,
@@ -49,8 +49,8 @@ func NewAssetRunner(options AssetRunnerOptions) *runner {
 	return &runner
 }
 
-// Ensures that runner implements testrunner.TestRunner interface
-var _ testrunner.TestRunner = new(runner)
+// Ensures that runner implements testrunner.Tester interface
+var _ testrunner.Tester = new(runner)
 
 // Type returns the type of test that can be run by this test runner.
 func (r *runner) Type() testrunner.TestType {

internal/testrunner/runners/pipeline/coverage.go

@@ -18,7 +18,7 @@ import (
 )
 
 // getPipelineCoverage returns a coverage report for the provided set of ingest pipelines.
-func getPipelineCoverage(options PipelineRunnerOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) {
+func getPipelineCoverage(options PipelineTesterOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) {
 	dataStreamPath, found, err := packages.FindDataStreamRootForPath(options.TestFolder.Path)
 	if err != nil {
 		return nil, fmt.Errorf("locating data_stream root failed: %w", err)

internal/testrunner/runners/pipeline/runner.go

@@ -58,7 +58,7 @@ type runner struct {
 	provider stack.Provider
 }
 
-type PipelineRunnerOptions struct {
+type PipelineTesterOptions struct {
 	Profile            *profile.Profile
 	DeferCleanup       time.Duration
 	API                *elasticsearch.API
@@ -69,7 +69,7 @@ type PipelineRunnerOptions struct {
 	CoverageType       string
 }
 
-func NewPipelineRunner(options PipelineRunnerOptions) (*runner, error) {
+func NewPipelineTester(options PipelineTesterOptions) (*runner, error) {
 	r := runner{
 		profile:            options.Profile,
 		deferCleanup:       options.DeferCleanup,
@@ -110,8 +110,8 @@ type IngestPipelineReroute struct {
 	AdditionalFields map[string]interface{}               `yaml:",inline"`
 }
 
-// Ensures that runner implements testrunner.TestRunner interface
-var _ testrunner.TestRunner = new(runner)
+// Ensures that runner implements testrunner.Tester interface
+var _ testrunner.Tester = new(runner)
 
 // Type returns the type of test that can be run by this test runner.
 func (r *runner) Type() testrunner.TestType {
@@ -355,7 +355,7 @@ func (r *runner) runTestCase(ctx context.Context, testCaseFile string, dsPath st
 	}
 
 	if r.withCoverage {
-		tr.Coverage, err = getPipelineCoverage(PipelineRunnerOptions{
+		tr.Coverage, err = getPipelineCoverage(PipelineTesterOptions{
 			TestFolder:      r.testFolder,
 			API:             r.esAPI,
 			PackageRootPath: r.packageRootPath,

internal/testrunner/runners/policy/runner.go

@@ -27,22 +27,72 @@ type runner struct {
 	packageRootPath    string
 	generateTestResult bool
 	kibanaClient       *kibana.Client
+
+	resourcesManager *resources.Manager
+	cleanup          func(context.Context) error
 }
 
-type PolicyRunnerOptions struct {
+// Ensures that runner implements testrunner.Tester interface
+var _ testrunner.Tester = new(runner)
+
+// Ensures that runner implements testrunner.TestRunner interface
+var _ testrunner.TestRunner = new(runner)
+
+type PolicyTestRunnerOptions struct {
+	KibanaClient    *kibana.Client
+	PackageRootPath string
+}
+
+type PolicyTesterOptions struct {
 	TestFolder         testrunner.TestFolder
 	KibanaClient       *kibana.Client
 	PackageRootPath    string
 	GenerateTestResult bool
 }
 
-func NewPolicyRunner(options PolicyRunnerOptions) *runner {
-	return &runner{
+func NewPolicyTestRunner(options PolicyTestRunnerOptions) *runner {
+	runner := runner{
+		kibanaClient:    options.KibanaClient,
+		packageRootPath: options.PackageRootPath,
+	}
+
+	runner.resourcesManager = resources.NewManager()
+	runner.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: runner.kibanaClient})
+	return &runner
+}
+
+func NewPolicyTester(options PolicyTesterOptions) *runner {
+	runner := runner{
 		kibanaClient:       options.KibanaClient,
 		testFolder:         options.TestFolder,
 		packageRootPath:    options.PackageRootPath,
 		generateTestResult: options.GenerateTestResult,
 	}
+	runner.resourcesManager = resources.NewManager()
+	runner.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: runner.kibanaClient})
+	return &runner
+}
+
+// SetupRunner prepares global resources required by the test runner.
+func (r *runner) SetupRunner(ctx context.Context) error {
+	cleanup, err := r.setupSuite(ctx, r.resourcesManager)
+	if err != nil {
+		return fmt.Errorf("failed to setup test runner: %w", err)
+	}
+	r.cleanup = cleanup
+
+	return nil
+}
+
+// TearDownRunner cleans up any global test runner resources. It must be called
+// after the test runner has finished executing all its tests.
+func (r *runner) TearDownRunner(ctx context.Context) error {
+	logger.Debug("Uninstalling package...")
+	err := r.cleanup(context.WithoutCancel(ctx))
+	if err != nil {
+		return fmt.Errorf("failed to clean up test runner: %w", err)
+	}
+	return nil
 }
 
 func (r *runner) Type() testrunner.TestType {
@@ -54,34 +104,19 @@ func (r *runner) String() string {
 }
 
 func (r *runner) Run(ctx context.Context) ([]testrunner.TestResult, error) {
-	manager := resources.NewManager()
-	manager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{
-		Client: r.kibanaClient,
-	})
-
-	cleanup, err := r.setupSuite(ctx, manager)
-	if err != nil {
-		return nil, fmt.Errorf("failed to setup test runner: %w", err)
-	}
-
 	var results []testrunner.TestResult
 	tests, err := filepath.Glob(filepath.Join(r.testFolder.Path, "test-*.yml"))
 	if err != nil {
 		return nil, fmt.Errorf("failed to look for test files in %s: %w", r.testFolder.Path, err)
 	}
 	for _, test := range tests {
-		result, err := r.runTest(ctx, manager, test)
+		result, err := r.runTest(ctx, r.resourcesManager, test)
 		if err != nil {
 			logger.Error(err)
 		}
 		results = append(results, result...)
 	}
 
-	err = cleanup(context.WithoutCancel(ctx))
-	if err != nil {
-		return nil, fmt.Errorf("failed to clean up test runner: %w", err)
-	}
-
 	return results, nil
 }
 
@@ -158,6 +193,7 @@ func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cl
 		return err
 	}
 
+	logger.Debugf("Installing package...")
 	_, err = manager.ApplyCtx(ctx, setupResources)
 	if err != nil {
 		if ctx.Err() == nil {

internal/testrunner/runners/static/runner.go

@@ -26,21 +26,21 @@ type runner struct {
 	packageRootPath string
 }
 
-type StaticRunnerOptions struct {
+type StaticTesterOptions struct {
 	TestFolder      testrunner.TestFolder
 	PackageRootPath string
 }
 
-func NewStaticRunner(options StaticRunnerOptions) *runner {
+func NewStaticTester(options StaticTesterOptions) *runner {
 	runner := runner{
 		testFolder:      options.TestFolder,
 		packageRootPath: options.PackageRootPath,
 	}
 	return &runner
 }
 
-// Ensures that runner implements testrunner.TestRunner interface
-var _ testrunner.TestRunner = new(runner)
+// Ensures that runner implements testrunner.Tester interface
+var _ testrunner.Tester = new(runner)
 
 const (
 	// TestType defining asset loading tests