addded ability to configure multiple tests

example/config/basic.yaml

@@ -3,8 +3,8 @@ endpoints:
     executionUrl: http://localhost:8545
     consensusUrl: http://localhost:5052
 
-test:
-  name: "basic"
+tests:
+- name: "basic"
   timeout: 48h
   tasks:
   - name: sleep

example/config/basic_with_cleanup.yaml

@@ -3,8 +3,8 @@ endpoints:
     executionUrl: http://localhost:8545
     consensusUrl: http://localhost:5052
 
-test:
-  name: basic
+tests:
+- name: basic
   timeout: 48h
   tasks:
   - name: check_clients_are_healthy

example/config/consensus_synced.yaml

@@ -3,8 +3,8 @@ endpoints:
     executionUrl: http://localhost:8545
     consensusUrl: http://localhost:5052
 
-test:
-  name: "basic"
+tests:
+- name: "basic"
   timeout: 48h
   tasks:
   - name: check_clients_are_healthy

example/config/test.yaml

@@ -3,8 +3,8 @@ endpoints:
     executionUrl: http://localhost:8545
     consensusUrl: http://localhost:5052
 
-test:
-  name: "basic"
+tests:
+- name: "basic"
   timeout: 48h
   tasks:
   - name: check_clients_are_healthy

pkg/coordinator/config.go

@@ -16,8 +16,8 @@ type Config struct {
 	// WebServer config
 	Web *web_types.WebConfig `yaml:"web" json:"web"`
 
-	// Test is the test configuration.
-	Test test.Config `yaml:"test" json:"test"`
+	// List of Test configurations.
+	Tests []*test.Config `yaml:"tests" json:"tests"`
 }
 
 // DefaultConfig represents a sane-default configuration.
@@ -30,7 +30,9 @@ func DefaultConfig() *Config {
 				ConsensusUrl: "http://localhost:5052",
 			},
 		},
-		Test: test.BasicSynced(),
+		Tests: []*test.Config{
+			test.BasicSynced(),
+		},
 	}
 }
 

pkg/coordinator/coordinator.go

@@ -68,27 +68,19 @@ func (c *Coordinator) Run(ctx context.Context) error {
 		}
 	}
 
-	// run test
-	testToRun, err := test.CreateRunnable(ctx, c, c.Config.Test)
-	if err != nil {
-		return err
-	}
-	c.tests = append(c.tests, testToRun)
-
-	if err := testToRun.Validate(); err != nil {
-		return err
-	}
-
-	c.log.Info(fmt.Sprintf("starting test '%s'", testToRun.Name()))
-
 	//nolint:errcheck // ignore
 	go c.startMetrics()
 
-	if err := testToRun.Run(ctx); err != nil {
-		return err
+	// initialize tests
+	for _, testCfg := range c.Config.Tests {
+		test, err := test.CreateTest(ctx, c, testCfg)
+		if err != nil {
+			return fmt.Errorf("failed initializing test '%v': %w", testCfg.Name, err)
+		}
+		c.tests = append(c.tests, test)
 	}
 
-	c.log.WithField("test", c.Config.Test).Info("test completed!")
+	c.runTests(ctx)
 
 	if c.webserver == nil {
 		c.log.WithField("seconds", c.lameDuckSeconds).Info("Initiating lame duck")
@@ -127,3 +119,16 @@ func (c *Coordinator) startMetrics() error {
 
 	return err
 }
+
+func (c *Coordinator) runTests(ctx context.Context) {
+	for _, test := range c.tests {
+		if err := test.Validate(); err != nil {
+			test.Logger().Errorf("test validation failed: %v", err)
+			continue
+		}
+		if err := test.Run(ctx); err != nil {
+			test.Logger().Errorf("test execution failed: %v", err)
+			continue
+		}
+	}
+}

pkg/coordinator/test/basic.go

@@ -1,7 +1,7 @@
 package test
 
-func BasicSynced() Config {
-	return Config{
+func BasicSynced() *Config {
+	return &Config{
 		Name: "basic",
 		// Tasks: []TaskConfig{
 		// 	{

pkg/coordinator/test/config.go

@@ -7,6 +7,7 @@ import (
 
 type Config struct {
 	Name         string              `yaml:"name" json:"name"`
+	Disable      bool                `yaml:"disable" json:"disable"`
 	Timeout      human.Duration      `yaml:"timeout" json:"timeout"`
 	Tasks        []helper.RawMessage `yaml:"tasks" json:"tasks"`
 	CleanupTasks []helper.RawMessage `yaml:"cleanupTasks" json:"cleanupTasks"`

pkg/coordinator/test/test.go

@@ -13,7 +13,7 @@ type Test struct {
 	name          string
 	taskScheduler *TaskScheduler
 	log           logrus.FieldLogger
-	config        Config
+	config        *Config
 	metrics       Metrics
 
 	status    types.TestStatus
@@ -22,7 +22,7 @@ type Test struct {
 	timeout   time.Duration
 }
 
-func CreateRunnable(ctx context.Context, coordinator types.Coordinator, config Config) (types.Test, error) {
+func CreateTest(ctx context.Context, coordinator types.Coordinator, config *Config) (types.Test, error) {
 	test := &Test{
 		name:    config.Name,
 		log:     coordinator.Logger().WithField("component", "test").WithField("test", config.Name),
@@ -33,36 +33,40 @@ func CreateRunnable(ctx context.Context, coordinator types.Coordinator, config C
 		test.timeout = test.config.Timeout.Duration
 	}
 
-	// parse tasks
-	test.taskScheduler = NewTaskScheduler(test.log, coordinator)
-	for _, rawtask := range config.Tasks {
-		taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask)
-		if err != nil {
-			return nil, err
+	if config.Disable {
+		test.status = types.TestStatusSkipped
+	} else {
+
+		// parse tasks
+		test.taskScheduler = NewTaskScheduler(test.log, coordinator)
+		for _, rawtask := range config.Tasks {
+			taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask)
+			if err != nil {
+				return nil, err
+			}
+			_, err = test.taskScheduler.AddRootTask(taskOptions)
+			if err != nil {
+				return nil, err
+			}
 		}
-		_, err = test.taskScheduler.AddRootTask(taskOptions)
-		if err != nil {
-			return nil, err
-		}
-	}
 
-	for _, rawtask := range config.CleanupTasks {
-		taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask)
-		if err != nil {
-			return nil, err
-		}
-		_, err = test.taskScheduler.AddCleanupTask(taskOptions)
-		if err != nil {
-			return nil, err
+		for _, rawtask := range config.CleanupTasks {
+			taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask)
+			if err != nil {
+				return nil, err
+			}
+			_, err = test.taskScheduler.AddCleanupTask(taskOptions)
+			if err != nil {
+				return nil, err
+			}
 		}
-	}
-
-	// setup metrics
-	test.metrics.Register()
 
-	test.metrics.SetTestInfo(config.Name)
-	test.metrics.SetTotalTasks(float64(len(config.Tasks)))
+		// setup metrics
+		test.metrics.Register()
 
+		test.metrics.SetTestInfo(config.Name)
+		test.metrics.SetTotalTasks(float64(len(config.Tasks)))
+	}
 	return test, nil
 }
 
@@ -86,20 +90,32 @@ func (t *Test) Status() types.TestStatus {
 	return t.status
 }
 
+func (t *Test) Logger() logrus.FieldLogger {
+	return t.log
+}
+
 func (t *Test) Validate() error {
+	if t.taskScheduler == nil {
+		return nil
+	}
 	err := t.taskScheduler.ValidateTaskConfigs()
 	if err != nil {
+		t.status = types.TestStatusFailure
 		return fmt.Errorf("test %s config validation failed: %w", t.name, err)
 	}
 
 	if t.taskScheduler.GetTaskCount() == 0 {
+		t.status = types.TestStatusFailure
 		return fmt.Errorf("test %s has no tasks", t.name)
 	}
 
 	return nil
 }
 
 func (t *Test) Run(ctx context.Context) error {
+	if t.taskScheduler == nil {
+		return nil
+	}
 	if t.status != types.TestStatusPending {
 		return fmt.Errorf("test has already been started")
 	}

pkg/coordinator/types/test.go

@@ -3,6 +3,8 @@ package types
 import (
 	"context"
 	"time"
+
+	"github.com/sirupsen/logrus"
 )
 
 type TestStatus uint8
@@ -12,6 +14,7 @@ const (
 	TestStatusRunning TestStatus = 1
 	TestStatusSuccess TestStatus = 2
 	TestStatusFailure TestStatus = 3
+	TestStatusSkipped TestStatus = 4
 )
 
 type Test interface {
@@ -23,5 +26,6 @@ type Test interface {
 	Timeout() time.Duration
 	Percent() float64
 	Status() TestStatus
+	Logger() logrus.FieldLogger
 	GetTaskScheduler() TaskScheduler
 }

pkg/coordinator/web/handlers/index.go

@@ -102,32 +102,40 @@ func (fh *FrontendHandler) getIndexPageData() (*IndexPage, error) {
 	pageData.Tests = []*IndexPageTest{}
 	for idx, test := range fh.coordinator.GetTests() {
 		testData := &IndexPageTest{
-			Index:       uint64(idx),
-			Name:        test.Name(),
-			IsStarted:   test.Status() != types.TestStatusPending,
-			IsCompleted: test.Status() > types.TestStatusRunning,
-			StartTime:   test.StartTime(),
-			StopTime:    test.StopTime(),
-			Timeout:     test.Timeout(),
-			HasTimeout:  test.Timeout() > 0,
-			TaskCount:   uint64(test.GetTaskScheduler().GetTaskCount()),
-		}
-		if testData.IsCompleted {
-			testData.RunTime = testData.StopTime.Sub(testData.StartTime)
-			testData.HasRunTime = true
-		} else if testData.IsStarted {
-			testData.RunTime = time.Since(testData.StartTime)
-			testData.HasRunTime = true
+			Index:      uint64(idx),
+			Name:       test.Name(),
+			StartTime:  test.StartTime(),
+			StopTime:   test.StopTime(),
+			Timeout:    test.Timeout(),
+			HasTimeout: test.Timeout() > 0,
 		}
+
 		switch test.Status() {
 		case types.TestStatusPending:
 			testData.Status = "pending"
 		case types.TestStatusRunning:
 			testData.Status = "running"
+			testData.IsStarted = true
 		case types.TestStatusSuccess:
 			testData.Status = "success"
+			testData.IsStarted = true
+			testData.IsCompleted = true
 		case types.TestStatusFailure:
 			testData.Status = "failure"
+			testData.IsStarted = true
+			testData.IsCompleted = true
+		case types.TestStatusSkipped:
+			testData.Status = "skipped"
+		}
+		if testData.IsCompleted {
+			testData.RunTime = testData.StopTime.Sub(testData.StartTime)
+			testData.HasRunTime = true
+		} else if testData.IsStarted {
+			testData.RunTime = time.Since(testData.StartTime)
+			testData.HasRunTime = true
+		}
+		if taskScheduler := test.GetTaskScheduler(); taskScheduler != nil {
+			testData.TaskCount = uint64(taskScheduler.GetTaskCount())
 		}
 		pageData.Tests = append(pageData.Tests, testData)
 	}