Update Tester instances to trigger just one test

internal/testrunner/runners/pipeline/runner.go

@@ -7,6 +7,7 @@ package pipeline
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -115,27 +116,51 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
 		}
 	}
 
-	// TODO: Return a tester per each configuration file defined in the data stream.
 	var testers []testrunner.Tester
-	for _, t := range folders {
-		t, err := NewPipelineTester(PipelineTesterOptions{
-			TestFolder:         t,
-			PackageRootPath:    r.packageRootPath,
-			GenerateTestResult: r.generateTestResult,
-			WithCoverage:       r.withCoverage,
-			CoverageType:       r.coverageType,
-			DeferCleanup:       r.deferCleanup,
-			Profile:            r.profile,
-			API:                r.esAPI,
-		})
+	for _, folder := range folders {
+		testCaseFiles, err := r.listTestCaseFiles(folder)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create pipeline tester: %w", err)
+			return nil, fmt.Errorf("listing test case definitions failed: %w", err)
+		}
+
+		for _, caseFile := range testCaseFiles {
+			t, err := NewPipelineTester(PipelineTesterOptions{
+				TestFolder:         folder,
+				PackageRootPath:    r.packageRootPath,
+				GenerateTestResult: r.generateTestResult,
+				WithCoverage:       r.withCoverage,
+				CoverageType:       r.coverageType,
+				DeferCleanup:       r.deferCleanup,
+				Profile:            r.profile,
+				API:                r.esAPI,
+				TestCaseFile:       caseFile,
+			})
+			if err != nil {
+				return nil, fmt.Errorf("failed to create pipeline tester: %w", err)
+			}
+			testers = append(testers, t)
 		}
-		testers = append(testers, t)
 	}
 	return testers, nil
 }
 
 func (r *runner) Type() testrunner.TestType {
 	return TestType
 }
+
+func (r *runner) listTestCaseFiles(folder testrunner.TestFolder) ([]string, error) {
+	fis, err := os.ReadDir(folder.Path)
+	if err != nil {
+		return nil, fmt.Errorf("reading pipeline tests failed (path: %s): %w", folder.Path, err)
+	}
+
+	var files []string
+	for _, fi := range fis {
+		if strings.HasSuffix(fi.Name(), expectedTestResultSuffix) ||
+			strings.HasSuffix(fi.Name(), configTestSuffixYAML) {
+			continue
+		}
+		files = append(files, fi.Name())
+	}
+	return files, nil
+}

internal/testrunner/runners/pipeline/tester.go

@@ -46,6 +46,8 @@ type tester struct {
 	withCoverage       bool
 	coverageType       string
 
+	testCaseFile string
+
 	pipelines []ingest.Pipeline
 
 	runCompareResults bool
@@ -62,6 +64,7 @@ type PipelineTesterOptions struct {
 	GenerateTestResult bool
 	WithCoverage       bool
 	CoverageType       string
+	TestCaseFile       string
 }
 
 func NewPipelineTester(options PipelineTesterOptions) (*tester, error) {
@@ -71,6 +74,7 @@ func NewPipelineTester(options PipelineTesterOptions) (*tester, error) {
 		esAPI:              options.API,
 		deferCleanup:       options.DeferCleanup,
 		testFolder:         options.TestFolder,
+		testCaseFile:       options.TestCaseFile,
 		generateTestResult: options.GenerateTestResult,
 		withCoverage:       options.WithCoverage,
 		coverageType:       options.CoverageType,
@@ -96,6 +100,11 @@ func NewPipelineTester(options PipelineTesterOptions) (*tester, error) {
 			r.runCompareResults = false
 		}
 	}
+
+	if r.esAPI == nil {
+		return nil, errors.New("missing Elasticsearch client")
+	}
+
 	return &r, nil
 }
 
@@ -140,15 +149,6 @@ func (r *tester) TearDown(ctx context.Context) error {
 }
 
 func (r *tester) run(ctx context.Context) ([]testrunner.TestResult, error) {
-	testCaseFiles, err := r.listTestCaseFiles()
-	if err != nil {
-		return nil, fmt.Errorf("listing test case definitions failed: %w", err)
-	}
-
-	if r.esAPI == nil {
-		return nil, errors.New("missing Elasticsearch client")
-	}
-
 	dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.testFolder.Path)
 	if err != nil {
 		return nil, fmt.Errorf("locating data_stream root failed: %w", err)
@@ -157,7 +157,7 @@ func (r *tester) run(ctx context.Context) ([]testrunner.TestResult, error) {
 		return nil, errors.New("data stream root not found")
 	}
 
-	startTime := time.Now()
+	startTesting := time.Now()
 	var entryPipeline string
 	entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(r.esAPI, dataStreamPath)
 	if err != nil {
@@ -198,23 +198,21 @@ func (r *tester) run(ctx context.Context) ([]testrunner.TestResult, error) {
 	}
 
 	results := make([]testrunner.TestResult, 0)
-	for _, testCaseFile := range testCaseFiles {
-		validatorOptions := []fields.ValidatorOption{
-			fields.WithSpecVersion(pkgManifest.SpecVersion),
-			// explicitly enabled for pipeline tests only
-			// since system tests can have dynamic public IPs
-			fields.WithEnabledAllowedIPCheck(),
-			fields.WithExpectedDatasets(expectedDatasets),
-			fields.WithEnabledImportAllECSSChema(true),
-		}
-		result, err := r.runTestCase(ctx, testCaseFile, dataStreamPath, dsManifest.Type, entryPipeline, validatorOptions)
-		if err != nil {
-			return nil, err
-		}
-		results = append(results, result)
+	validatorOptions := []fields.ValidatorOption{
+		fields.WithSpecVersion(pkgManifest.SpecVersion),
+		// explicitly enabled for pipeline tests only
+		// since system tests can have dynamic public IPs
+		fields.WithEnabledAllowedIPCheck(),
+		fields.WithExpectedDatasets(expectedDatasets),
+		fields.WithEnabledImportAllECSSChema(true),
+	}
+	result, err := r.runTestCase(ctx, r.testCaseFile, dataStreamPath, dsManifest.Type, entryPipeline, validatorOptions)
+	if err != nil {
+		return nil, err
 	}
+	results = append(results, result)
 
-	esLogs, err := r.checkElasticsearchLogs(ctx, startTime)
+	esLogs, err := r.checkElasticsearchLogs(ctx, startTesting)
 	if err != nil {
 		return nil, err
 	}
@@ -278,7 +276,7 @@ func (r *tester) checkElasticsearchLogs(ctx context.Context, startTesting time.T
 
 	tr := testrunner.TestResult{
 		TestType:    TestType,
-		Name:        "(ingest pipeline warnings)",
+		Name:        fmt.Sprintf("(ingest pipeline warnings %s)", r.testCaseFile),
 		Package:     r.testFolder.Package,
 		DataStream:  r.testFolder.DataStream,
 		TimeElapsed: time.Since(startTime),
@@ -364,23 +362,6 @@ func (r *tester) runTestCase(ctx context.Context, testCaseFile string, dsPath st
 	return tr, nil
 }
 
-func (r *tester) listTestCaseFiles() ([]string, error) {
-	fis, err := os.ReadDir(r.testFolder.Path)
-	if err != nil {
-		return nil, fmt.Errorf("reading pipeline tests failed (path: %s): %w", r.testFolder.Path, err)
-	}
-
-	var files []string
-	for _, fi := range fis {
-		if strings.HasSuffix(fi.Name(), expectedTestResultSuffix) ||
-			strings.HasSuffix(fi.Name(), configTestSuffixYAML) {
-			continue
-		}
-		files = append(files, fi.Name())
-	}
-	return files, nil
-}
-
 func loadTestCaseFile(testFolderPath, testCaseFile string) (*testCase, error) {
 	testCasePath := filepath.Join(testFolderPath, testCaseFile)
 	testCaseData, err := os.ReadFile(testCasePath)

internal/testrunner/runners/policy/runner.go

@@ -7,6 +7,7 @@ package policy
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/elastic/elastic-package/internal/kibana"
@@ -117,19 +118,55 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
 		}
 	}
 
-	// TODO: Return a tester per each configuration file defined in the data stream folder
 	var testers []testrunner.Tester
-	for _, t := range folders {
-		testers = append(testers, NewPolicyTester(PolicyTesterOptions{
-			PackageRootPath:    r.packageRootPath,
-			TestFolder:         t,
-			KibanaClient:       r.kibanaClient,
-			GenerateTestResult: r.generateTestResult,
-		}))
+	for _, folder := range folders {
+		tests, err := filepath.Glob(filepath.Join(folder.Path, "test-*.yml"))
+		if err != nil {
+			return nil, fmt.Errorf("failed to look for test files in %s: %w", folder.Path, err)
+		}
+		for _, test := range tests {
+			testers = append(testers, NewPolicyTester(PolicyTesterOptions{
+				PackageRootPath:    r.packageRootPath,
+				TestFolder:         folder,
+				KibanaClient:       r.kibanaClient,
+				GenerateTestResult: r.generateTestResult,
+				TestPath:           test,
+			}))
+
+		}
 	}
 	return testers, nil
 }
 
 func (r *runner) Type() testrunner.TestType {
 	return TestType
 }
+
+func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cleanup func(ctx context.Context) error, err error) {
+	packageResource := resources.FleetPackage{
+		RootPath: r.packageRootPath,
+	}
+	setupResources := resources.Resources{
+		&packageResource,
+	}
+
+	cleanup = func(ctx context.Context) error {
+		packageResource.Absent = true
+		_, err := manager.ApplyCtx(ctx, setupResources)
+		return err
+	}
+
+	logger.Debugf("Installing package...")
+	_, err = manager.ApplyCtx(ctx, setupResources)
+	if err != nil {
+		if ctx.Err() == nil {
+			cleanupErr := cleanup(ctx)
+			if cleanupErr != nil {
+				return nil, fmt.Errorf("setup failed: %w (with cleanup error: %w)", err, cleanupErr)
+			}
+		}
+		return nil, fmt.Errorf("setup failed: %w", err)
+	}
+
+	return cleanup, err
+}

internal/testrunner/runners/policy/tester.go

@@ -6,7 +6,6 @@ package policy
 
 import (
 	"context"
-	"fmt"
 	"path/filepath"
 	"strings"
 
@@ -23,6 +22,7 @@ type tester struct {
 	packageRootPath    string
 	generateTestResult bool
 	kibanaClient       *kibana.Client
+	testPath           string
 
 	resourcesManager *resources.Manager
 }
@@ -32,6 +32,7 @@ var _ testrunner.Tester = new(tester)
 
 type PolicyTesterOptions struct {
 	TestFolder         testrunner.TestFolder
+	TestPath           string
 	KibanaClient       *kibana.Client
 	PackageRootPath    string
 	GenerateTestResult bool
@@ -43,6 +44,7 @@ func NewPolicyTester(options PolicyTesterOptions) *tester {
 		testFolder:         options.TestFolder,
 		packageRootPath:    options.PackageRootPath,
 		generateTestResult: options.GenerateTestResult,
+		testPath:           options.TestPath,
 	}
 	tester.resourcesManager = resources.NewManager()
 	tester.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: tester.kibanaClient})
@@ -59,18 +61,13 @@ func (r *tester) String() string {
 
 func (r *tester) Run(ctx context.Context) ([]testrunner.TestResult, error) {
 	var results []testrunner.TestResult
-	tests, err := filepath.Glob(filepath.Join(r.testFolder.Path, "test-*.yml"))
+
+	result, err := r.runTest(ctx, r.resourcesManager, r.testPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to look for test files in %s: %w", r.testFolder.Path, err)
-	}
-	for _, test := range tests {
-		result, err := r.runTest(ctx, r.resourcesManager, test)
-		if err != nil {
-			logger.Error(err)
-		}
-		results = append(results, result...)
+		logger.Error(err)
 	}
 
+	results = append(results, result...)
 	return results, nil
 }
 
@@ -133,35 +130,6 @@ func testNameFromPath(path string) string {
 	return strings.TrimSuffix(filepath.Base(path), ext)
 }
 
-func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cleanup func(ctx context.Context) error, err error) {
-	packageResource := resources.FleetPackage{
-		RootPath: r.packageRootPath,
-	}
-	setupResources := resources.Resources{
-		&packageResource,
-	}
-
-	cleanup = func(ctx context.Context) error {
-		packageResource.Absent = true
-		_, err := manager.ApplyCtx(ctx, setupResources)
-		return err
-	}
-
-	logger.Debugf("Installing package...")
-	_, err = manager.ApplyCtx(ctx, setupResources)
-	if err != nil {
-		if ctx.Err() == nil {
-			cleanupErr := cleanup(ctx)
-			if cleanupErr != nil {
-				return nil, fmt.Errorf("setup failed: %w (with cleanup error: %w)", err, cleanupErr)
-			}
-		}
-		return nil, fmt.Errorf("setup failed: %w", err)
-	}
-
-	return cleanup, err
-}
-
 func (r *tester) TearDown(ctx context.Context) error {
 	return nil
 }