Skip to content

Commit

Permalink
Update Tester instances to trigger just one test (#1898)
Browse files Browse the repository at this point in the history 
Update how tests are triggered. Now each test runner needs to define
its own Testers. Each instance of a Tester should contain all the required
information to run a test.
  • Loading branch information
@mrodm
mrodm authored Jun 13, 2024
1 parent 2dbab19 commit 3d8ed71
Show file tree
Hide file tree
Showing 9 changed files with 416 additions and 362 deletions.
53 changes: 39 additions & 14 deletions internal/testrunner/runners/pipeline/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package pipeline
import (
"context"
"fmt"
"os"
"strings"
"time"

Expand Down Expand Up @@ -115,27 +116,51 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
}
}

// TODO: Return a tester per each configuration file defined in the data stream.
var testers []testrunner.Tester
for _, t := range folders {
t, err := NewPipelineTester(PipelineTesterOptions{
TestFolder: t,
PackageRootPath: r.packageRootPath,
GenerateTestResult: r.generateTestResult,
WithCoverage: r.withCoverage,
CoverageType: r.coverageType,
DeferCleanup: r.deferCleanup,
Profile: r.profile,
API: r.esAPI,
})
for _, folder := range folders {
testCaseFiles, err := r.listTestCaseFiles(folder)
if err != nil {
return nil, fmt.Errorf("failed to create pipeline tester: %w", err)
return nil, fmt.Errorf("listing test case definitions failed: %w", err)
}

for _, caseFile := range testCaseFiles {
t, err := NewPipelineTester(PipelineTesterOptions{
TestFolder: folder,
PackageRootPath: r.packageRootPath,
GenerateTestResult: r.generateTestResult,
WithCoverage: r.withCoverage,
CoverageType: r.coverageType,
DeferCleanup: r.deferCleanup,
Profile: r.profile,
API: r.esAPI,
TestCaseFile: caseFile,
})
if err != nil {
return nil, fmt.Errorf("failed to create pipeline tester: %w", err)
}
testers = append(testers, t)
}
testers = append(testers, t)
}
return testers, nil
}

func (r *runner) Type() testrunner.TestType {
return TestType
}

func (r *runner) listTestCaseFiles(folder testrunner.TestFolder) ([]string, error) {
fis, err := os.ReadDir(folder.Path)
if err != nil {
return nil, fmt.Errorf("reading pipeline tests failed (path: %s): %w", folder.Path, err)
}

var files []string
for _, fi := range fis {
if strings.HasSuffix(fi.Name(), expectedTestResultSuffix) ||
strings.HasSuffix(fi.Name(), configTestSuffixYAML) {
continue
}
files = append(files, fi.Name())
}
return files, nil
}
67 changes: 24 additions & 43 deletions internal/testrunner/runners/pipeline/tester.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ type tester struct {
withCoverage bool
coverageType string

testCaseFile string

pipelines []ingest.Pipeline

runCompareResults bool
Expand All @@ -62,6 +64,7 @@ type PipelineTesterOptions struct {
GenerateTestResult bool
WithCoverage bool
CoverageType string
TestCaseFile string
}

func NewPipelineTester(options PipelineTesterOptions) (*tester, error) {
Expand All @@ -71,6 +74,7 @@ func NewPipelineTester(options PipelineTesterOptions) (*tester, error) {
esAPI: options.API,
deferCleanup: options.DeferCleanup,
testFolder: options.TestFolder,
testCaseFile: options.TestCaseFile,
generateTestResult: options.GenerateTestResult,
withCoverage: options.WithCoverage,
coverageType: options.CoverageType,
Expand All @@ -96,6 +100,11 @@ func NewPipelineTester(options PipelineTesterOptions) (*tester, error) {
r.runCompareResults = false
}
}

if r.esAPI == nil {
return nil, errors.New("missing Elasticsearch client")
}

return &r, nil
}

Expand Down Expand Up @@ -140,15 +149,6 @@ func (r *tester) TearDown(ctx context.Context) error {
}

func (r *tester) run(ctx context.Context) ([]testrunner.TestResult, error) {
testCaseFiles, err := r.listTestCaseFiles()
if err != nil {
return nil, fmt.Errorf("listing test case definitions failed: %w", err)
}

if r.esAPI == nil {
return nil, errors.New("missing Elasticsearch client")
}

dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.testFolder.Path)
if err != nil {
return nil, fmt.Errorf("locating data_stream root failed: %w", err)
Expand All @@ -157,7 +157,7 @@ func (r *tester) run(ctx context.Context) ([]testrunner.TestResult, error) {
return nil, errors.New("data stream root not found")
}

startTime := time.Now()
startTesting := time.Now()
var entryPipeline string
entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(r.esAPI, dataStreamPath)
if err != nil {
Expand Down Expand Up @@ -198,23 +198,21 @@ func (r *tester) run(ctx context.Context) ([]testrunner.TestResult, error) {
}

results := make([]testrunner.TestResult, 0)
for _, testCaseFile := range testCaseFiles {
validatorOptions := []fields.ValidatorOption{
fields.WithSpecVersion(pkgManifest.SpecVersion),
// explicitly enabled for pipeline tests only
// since system tests can have dynamic public IPs
fields.WithEnabledAllowedIPCheck(),
fields.WithExpectedDatasets(expectedDatasets),
fields.WithEnabledImportAllECSSChema(true),
}
result, err := r.runTestCase(ctx, testCaseFile, dataStreamPath, dsManifest.Type, entryPipeline, validatorOptions)
if err != nil {
return nil, err
}
results = append(results, result)
validatorOptions := []fields.ValidatorOption{
fields.WithSpecVersion(pkgManifest.SpecVersion),
// explicitly enabled for pipeline tests only
// since system tests can have dynamic public IPs
fields.WithEnabledAllowedIPCheck(),
fields.WithExpectedDatasets(expectedDatasets),
fields.WithEnabledImportAllECSSChema(true),
}
result, err := r.runTestCase(ctx, r.testCaseFile, dataStreamPath, dsManifest.Type, entryPipeline, validatorOptions)
if err != nil {
return nil, err
}
results = append(results, result)

esLogs, err := r.checkElasticsearchLogs(ctx, startTime)
esLogs, err := r.checkElasticsearchLogs(ctx, startTesting)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -278,7 +276,7 @@ func (r *tester) checkElasticsearchLogs(ctx context.Context, startTesting time.T

tr := testrunner.TestResult{
TestType: TestType,
Name: "(ingest pipeline warnings)",
Name: fmt.Sprintf("(ingest pipeline warnings %s)", r.testCaseFile),
Package: r.testFolder.Package,
DataStream: r.testFolder.DataStream,
TimeElapsed: time.Since(startTime),
Expand Down Expand Up @@ -364,23 +362,6 @@ func (r *tester) runTestCase(ctx context.Context, testCaseFile string, dsPath st
return tr, nil
}

func (r *tester) listTestCaseFiles() ([]string, error) {
fis, err := os.ReadDir(r.testFolder.Path)
if err != nil {
return nil, fmt.Errorf("reading pipeline tests failed (path: %s): %w", r.testFolder.Path, err)
}

var files []string
for _, fi := range fis {
if strings.HasSuffix(fi.Name(), expectedTestResultSuffix) ||
strings.HasSuffix(fi.Name(), configTestSuffixYAML) {
continue
}
files = append(files, fi.Name())
}
return files, nil
}

func loadTestCaseFile(testFolderPath, testCaseFile string) (*testCase, error) {
testCasePath := filepath.Join(testFolderPath, testCaseFile)
testCaseData, err := os.ReadFile(testCasePath)
Expand Down
53 changes: 45 additions & 8 deletions internal/testrunner/runners/policy/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package policy
import (
"context"
"fmt"
"path/filepath"
"strings"

"github.com/elastic/elastic-package/internal/kibana"
Expand Down Expand Up @@ -117,19 +118,55 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
}
}

// TODO: Return a tester per each configuration file defined in the data stream folder
var testers []testrunner.Tester
for _, t := range folders {
testers = append(testers, NewPolicyTester(PolicyTesterOptions{
PackageRootPath: r.packageRootPath,
TestFolder: t,
KibanaClient: r.kibanaClient,
GenerateTestResult: r.generateTestResult,
}))
for _, folder := range folders {
tests, err := filepath.Glob(filepath.Join(folder.Path, "test-*.yml"))
if err != nil {
return nil, fmt.Errorf("failed to look for test files in %s: %w", folder.Path, err)
}
for _, test := range tests {
testers = append(testers, NewPolicyTester(PolicyTesterOptions{
PackageRootPath: r.packageRootPath,
TestFolder: folder,
KibanaClient: r.kibanaClient,
GenerateTestResult: r.generateTestResult,
TestPath: test,
}))

}
}
return testers, nil
}

func (r *runner) Type() testrunner.TestType {
return TestType
}

func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cleanup func(ctx context.Context) error, err error) {
packageResource := resources.FleetPackage{
RootPath: r.packageRootPath,
}
setupResources := resources.Resources{
&packageResource,
}

cleanup = func(ctx context.Context) error {
packageResource.Absent = true
_, err := manager.ApplyCtx(ctx, setupResources)
return err
}

logger.Debugf("Installing package...")
_, err = manager.ApplyCtx(ctx, setupResources)
if err != nil {
if ctx.Err() == nil {
cleanupErr := cleanup(ctx)
if cleanupErr != nil {
return nil, fmt.Errorf("setup failed: %w (with cleanup error: %w)", err, cleanupErr)
}
}
return nil, fmt.Errorf("setup failed: %w", err)
}

return cleanup, err
}
46 changes: 7 additions & 39 deletions internal/testrunner/runners/policy/tester.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ package policy

import (
"context"
"fmt"
"path/filepath"
"strings"

Expand All @@ -23,6 +22,7 @@ type tester struct {
packageRootPath string
generateTestResult bool
kibanaClient *kibana.Client
testPath string

resourcesManager *resources.Manager
}
Expand All @@ -32,6 +32,7 @@ var _ testrunner.Tester = new(tester)

type PolicyTesterOptions struct {
TestFolder testrunner.TestFolder
TestPath string
KibanaClient *kibana.Client
PackageRootPath string
GenerateTestResult bool
Expand All @@ -43,6 +44,7 @@ func NewPolicyTester(options PolicyTesterOptions) *tester {
testFolder: options.TestFolder,
packageRootPath: options.PackageRootPath,
generateTestResult: options.GenerateTestResult,
testPath: options.TestPath,
}
tester.resourcesManager = resources.NewManager()
tester.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: tester.kibanaClient})
Expand All @@ -59,18 +61,13 @@ func (r *tester) String() string {

func (r *tester) Run(ctx context.Context) ([]testrunner.TestResult, error) {
var results []testrunner.TestResult
tests, err := filepath.Glob(filepath.Join(r.testFolder.Path, "test-*.yml"))

result, err := r.runTest(ctx, r.resourcesManager, r.testPath)
if err != nil {
return nil, fmt.Errorf("failed to look for test files in %s: %w", r.testFolder.Path, err)
}
for _, test := range tests {
result, err := r.runTest(ctx, r.resourcesManager, test)
if err != nil {
logger.Error(err)
}
results = append(results, result...)
logger.Error(err)
}

results = append(results, result...)
return results, nil
}

Expand Down Expand Up @@ -133,35 +130,6 @@ func testNameFromPath(path string) string {
return strings.TrimSuffix(filepath.Base(path), ext)
}

func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cleanup func(ctx context.Context) error, err error) {
packageResource := resources.FleetPackage{
RootPath: r.packageRootPath,
}
setupResources := resources.Resources{
&packageResource,
}

cleanup = func(ctx context.Context) error {
packageResource.Absent = true
_, err := manager.ApplyCtx(ctx, setupResources)
return err
}

logger.Debugf("Installing package...")
_, err = manager.ApplyCtx(ctx, setupResources)
if err != nil {
if ctx.Err() == nil {
cleanupErr := cleanup(ctx)
if cleanupErr != nil {
return nil, fmt.Errorf("setup failed: %w (with cleanup error: %w)", err, cleanupErr)
}
}
return nil, fmt.Errorf("setup failed: %w", err)
}

return cleanup, err
}

func (r *tester) TearDown(ctx context.Context) error {
return nil
}
Loading

0 comments on commit 3d8ed71

Please sign in to comment.