diff --git a/example/config/basic.yaml b/example/config/basic.yaml index 836e1c4..1d77c83 100644 --- a/example/config/basic.yaml +++ b/example/config/basic.yaml @@ -3,8 +3,8 @@ endpoints: executionUrl: http://localhost:8545 consensusUrl: http://localhost:5052 -test: - name: "basic" +tests: +- name: "basic" timeout: 48h tasks: - name: sleep diff --git a/example/config/basic_with_cleanup.yaml b/example/config/basic_with_cleanup.yaml index 1e21ccb..91f4d46 100644 --- a/example/config/basic_with_cleanup.yaml +++ b/example/config/basic_with_cleanup.yaml @@ -3,8 +3,8 @@ endpoints: executionUrl: http://localhost:8545 consensusUrl: http://localhost:5052 -test: - name: basic +tests: +- name: basic timeout: 48h tasks: - name: check_clients_are_healthy diff --git a/example/config/consensus_synced.yaml b/example/config/consensus_synced.yaml index 7bfc77a..c97d512 100644 --- a/example/config/consensus_synced.yaml +++ b/example/config/consensus_synced.yaml @@ -3,8 +3,8 @@ endpoints: executionUrl: http://localhost:8545 consensusUrl: http://localhost:5052 -test: - name: "basic" +tests: +- name: "basic" timeout: 48h tasks: - name: check_clients_are_healthy diff --git a/example/config/test.yaml b/example/config/test.yaml index 0c1611c..d992f87 100644 --- a/example/config/test.yaml +++ b/example/config/test.yaml @@ -3,8 +3,8 @@ endpoints: executionUrl: http://localhost:8545 consensusUrl: http://localhost:5052 -test: - name: "basic" +tests: +- name: "basic" timeout: 48h tasks: - name: check_clients_are_healthy diff --git a/pkg/coordinator/config.go b/pkg/coordinator/config.go index 7866253..4e32356 100644 --- a/pkg/coordinator/config.go +++ b/pkg/coordinator/config.go @@ -16,8 +16,8 @@ type Config struct { // WebServer config Web *web_types.WebConfig `yaml:"web" json:"web"` - // Test is the test configuration. - Test test.Config `yaml:"test" json:"test"` + // List of Test configurations. + Tests []*test.Config `yaml:"tests" json:"tests"` } // DefaultConfig represents a sane-default configuration. @@ -30,7 +30,9 @@ func DefaultConfig() *Config { ConsensusUrl: "http://localhost:5052", }, }, - Test: test.BasicSynced(), + Tests: []*test.Config{ + test.BasicSynced(), + }, } } diff --git a/pkg/coordinator/coordinator.go b/pkg/coordinator/coordinator.go index 0b1cc08..0c18541 100644 --- a/pkg/coordinator/coordinator.go +++ b/pkg/coordinator/coordinator.go @@ -68,27 +68,19 @@ func (c *Coordinator) Run(ctx context.Context) error { } } - // run test - testToRun, err := test.CreateRunnable(ctx, c, c.Config.Test) - if err != nil { - return err - } - c.tests = append(c.tests, testToRun) - - if err := testToRun.Validate(); err != nil { - return err - } - - c.log.Info(fmt.Sprintf("starting test '%s'", testToRun.Name())) - //nolint:errcheck // ignore go c.startMetrics() - if err := testToRun.Run(ctx); err != nil { - return err + // initialize tests + for _, testCfg := range c.Config.Tests { + test, err := test.CreateTest(ctx, c, testCfg) + if err != nil { + return fmt.Errorf("failed initializing test '%v': %w", testCfg.Name, err) + } + c.tests = append(c.tests, test) } - c.log.WithField("test", c.Config.Test).Info("test completed!") + c.runTests(ctx) if c.webserver == nil { c.log.WithField("seconds", c.lameDuckSeconds).Info("Initiating lame duck") @@ -127,3 +119,16 @@ func (c *Coordinator) startMetrics() error { return err } + +func (c *Coordinator) runTests(ctx context.Context) { + for _, test := range c.tests { + if err := test.Validate(); err != nil { + test.Logger().Errorf("test validation failed: %v", err) + continue + } + if err := test.Run(ctx); err != nil { + test.Logger().Errorf("test execution failed: %v", err) + continue + } + } +} diff --git a/pkg/coordinator/test/basic.go b/pkg/coordinator/test/basic.go index 5f10554..6c95cac 100644 --- a/pkg/coordinator/test/basic.go +++ b/pkg/coordinator/test/basic.go @@ -1,7 +1,7 @@ package test -func BasicSynced() Config { - return Config{ +func BasicSynced() *Config { + return &Config{ Name: "basic", // Tasks: []TaskConfig{ // { diff --git a/pkg/coordinator/test/config.go b/pkg/coordinator/test/config.go index 5ac66c2..f43c0ab 100644 --- a/pkg/coordinator/test/config.go +++ b/pkg/coordinator/test/config.go @@ -7,6 +7,7 @@ import ( type Config struct { Name string `yaml:"name" json:"name"` + Disable bool `yaml:"disable" json:"disable"` Timeout human.Duration `yaml:"timeout" json:"timeout"` Tasks []helper.RawMessage `yaml:"tasks" json:"tasks"` CleanupTasks []helper.RawMessage `yaml:"cleanupTasks" json:"cleanupTasks"` diff --git a/pkg/coordinator/test/test.go b/pkg/coordinator/test/test.go index 9f9730f..611a54e 100644 --- a/pkg/coordinator/test/test.go +++ b/pkg/coordinator/test/test.go @@ -13,7 +13,7 @@ type Test struct { name string taskScheduler *TaskScheduler log logrus.FieldLogger - config Config + config *Config metrics Metrics status types.TestStatus @@ -22,7 +22,7 @@ type Test struct { timeout time.Duration } -func CreateRunnable(ctx context.Context, coordinator types.Coordinator, config Config) (types.Test, error) { +func CreateTest(ctx context.Context, coordinator types.Coordinator, config *Config) (types.Test, error) { test := &Test{ name: config.Name, log: coordinator.Logger().WithField("component", "test").WithField("test", config.Name), @@ -33,36 +33,40 @@ func CreateRunnable(ctx context.Context, coordinator types.Coordinator, config C test.timeout = test.config.Timeout.Duration } - // parse tasks - test.taskScheduler = NewTaskScheduler(test.log, coordinator) - for _, rawtask := range config.Tasks { - taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask) - if err != nil { - return nil, err + if config.Disable { + test.status = types.TestStatusSkipped + } else { + + // parse tasks + test.taskScheduler = NewTaskScheduler(test.log, coordinator) + for _, rawtask := range config.Tasks { + taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask) + if err != nil { + return nil, err + } + _, err = test.taskScheduler.AddRootTask(taskOptions) + if err != nil { + return nil, err + } } - _, err = test.taskScheduler.AddRootTask(taskOptions) - if err != nil { - return nil, err - } - } - for _, rawtask := range config.CleanupTasks { - taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask) - if err != nil { - return nil, err - } - _, err = test.taskScheduler.AddCleanupTask(taskOptions) - if err != nil { - return nil, err + for _, rawtask := range config.CleanupTasks { + taskOptions, err := test.taskScheduler.ParseTaskOptions(&rawtask) + if err != nil { + return nil, err + } + _, err = test.taskScheduler.AddCleanupTask(taskOptions) + if err != nil { + return nil, err + } } - } - - // setup metrics - test.metrics.Register() - test.metrics.SetTestInfo(config.Name) - test.metrics.SetTotalTasks(float64(len(config.Tasks))) + // setup metrics + test.metrics.Register() + test.metrics.SetTestInfo(config.Name) + test.metrics.SetTotalTasks(float64(len(config.Tasks))) + } return test, nil } @@ -86,13 +90,22 @@ func (t *Test) Status() types.TestStatus { return t.status } +func (t *Test) Logger() logrus.FieldLogger { + return t.log +} + func (t *Test) Validate() error { + if t.taskScheduler == nil { + return nil + } err := t.taskScheduler.ValidateTaskConfigs() if err != nil { + t.status = types.TestStatusFailure return fmt.Errorf("test %s config validation failed: %w", t.name, err) } if t.taskScheduler.GetTaskCount() == 0 { + t.status = types.TestStatusFailure return fmt.Errorf("test %s has no tasks", t.name) } @@ -100,6 +113,9 @@ func (t *Test) Validate() error { } func (t *Test) Run(ctx context.Context) error { + if t.taskScheduler == nil { + return nil + } if t.status != types.TestStatusPending { return fmt.Errorf("test has already been started") } diff --git a/pkg/coordinator/types/test.go b/pkg/coordinator/types/test.go index b3f1a04..db4ec32 100644 --- a/pkg/coordinator/types/test.go +++ b/pkg/coordinator/types/test.go @@ -3,6 +3,8 @@ package types import ( "context" "time" + + "github.com/sirupsen/logrus" ) type TestStatus uint8 @@ -12,6 +14,7 @@ const ( TestStatusRunning TestStatus = 1 TestStatusSuccess TestStatus = 2 TestStatusFailure TestStatus = 3 + TestStatusSkipped TestStatus = 4 ) type Test interface { @@ -23,5 +26,6 @@ type Test interface { Timeout() time.Duration Percent() float64 Status() TestStatus + Logger() logrus.FieldLogger GetTaskScheduler() TaskScheduler } diff --git a/pkg/coordinator/web/handlers/index.go b/pkg/coordinator/web/handlers/index.go index 4fac451..1137b7c 100644 --- a/pkg/coordinator/web/handlers/index.go +++ b/pkg/coordinator/web/handlers/index.go @@ -102,32 +102,40 @@ func (fh *FrontendHandler) getIndexPageData() (*IndexPage, error) { pageData.Tests = []*IndexPageTest{} for idx, test := range fh.coordinator.GetTests() { testData := &IndexPageTest{ - Index: uint64(idx), - Name: test.Name(), - IsStarted: test.Status() != types.TestStatusPending, - IsCompleted: test.Status() > types.TestStatusRunning, - StartTime: test.StartTime(), - StopTime: test.StopTime(), - Timeout: test.Timeout(), - HasTimeout: test.Timeout() > 0, - TaskCount: uint64(test.GetTaskScheduler().GetTaskCount()), - } - if testData.IsCompleted { - testData.RunTime = testData.StopTime.Sub(testData.StartTime) - testData.HasRunTime = true - } else if testData.IsStarted { - testData.RunTime = time.Since(testData.StartTime) - testData.HasRunTime = true + Index: uint64(idx), + Name: test.Name(), + StartTime: test.StartTime(), + StopTime: test.StopTime(), + Timeout: test.Timeout(), + HasTimeout: test.Timeout() > 0, } + switch test.Status() { case types.TestStatusPending: testData.Status = "pending" case types.TestStatusRunning: testData.Status = "running" + testData.IsStarted = true case types.TestStatusSuccess: testData.Status = "success" + testData.IsStarted = true + testData.IsCompleted = true case types.TestStatusFailure: testData.Status = "failure" + testData.IsStarted = true + testData.IsCompleted = true + case types.TestStatusSkipped: + testData.Status = "skipped" + } + if testData.IsCompleted { + testData.RunTime = testData.StopTime.Sub(testData.StartTime) + testData.HasRunTime = true + } else if testData.IsStarted { + testData.RunTime = time.Since(testData.StartTime) + testData.HasRunTime = true + } + if taskScheduler := test.GetTaskScheduler(); taskScheduler != nil { + testData.TaskCount = uint64(taskScheduler.GetTaskCount()) } pageData.Tests = append(pageData.Tests, testData) } diff --git a/pkg/coordinator/web/handlers/test.go b/pkg/coordinator/web/handlers/test.go index d5fb54f..fd66622 100644 --- a/pkg/coordinator/web/handlers/test.go +++ b/pkg/coordinator/web/handlers/test.go @@ -104,65 +104,71 @@ func (fh *FrontendHandler) getTestPageData(testIdx int64) (*TestPage, error) { } pageData := &TestPage{ - Index: uint64(testIdx), - Name: test.Name(), - IsStarted: test.Status() != types.TestStatusPending, - IsCompleted: test.Status() > types.TestStatusRunning, - StartTime: test.StartTime(), - StopTime: test.StopTime(), - Timeout: test.Timeout(), + Index: uint64(testIdx), + Name: test.Name(), + StartTime: test.StartTime(), + StopTime: test.StopTime(), + Timeout: test.Timeout(), } switch test.Status() { case types.TestStatusPending: pageData.Status = "pending" case types.TestStatusRunning: pageData.Status = "running" + pageData.IsStarted = true case types.TestStatusSuccess: pageData.Status = "success" + pageData.IsStarted = true + pageData.IsCompleted = true case types.TestStatusFailure: pageData.Status = "failure" + pageData.IsStarted = true + pageData.IsCompleted = true + case types.TestStatusSkipped: + pageData.Status = "skipped" } taskScheduler := test.GetTaskScheduler() + if taskScheduler != nil { + for _, task := range taskScheduler.GetAllTasks() { + taskStatus := taskScheduler.GetTaskStatus(task) - for _, task := range taskScheduler.GetAllTasks() { - taskStatus := taskScheduler.GetTaskStatus(task) + taskData := &TestPageTask{ + Index: taskStatus.Index, + Name: task.Name(), + Title: task.Title(), + IsStarted: taskStatus.IsStarted, + IsCompleted: taskStatus.IsStarted && !taskStatus.IsRunning, + StartTime: taskStatus.StartTime, + StopTime: taskStatus.StopTime, + Timeout: task.Timeout(), + HasTimeout: task.Timeout() > 0, + } + if !taskStatus.IsStarted { + taskData.Status = "pending" + } else if taskStatus.IsRunning { + taskData.Status = "running" + taskData.HasRunTime = true + taskData.RunTime = time.Since(taskStatus.StartTime) + } else { + taskData.Status = "complete" + taskData.HasRunTime = true + taskData.RunTime = taskStatus.StopTime.Sub(taskStatus.StartTime) + } + switch taskStatus.Result { + case types.TaskResultNone: + taskData.Result = "none" + case types.TaskResultSuccess: + taskData.Result = "success" + case types.TaskResultFailure: + taskData.Result = "failure" + } + if taskStatus.Error != nil { + taskData.ResultError = taskStatus.Error.Error() + } - taskData := &TestPageTask{ - Index: taskStatus.Index, - Name: task.Name(), - Title: task.Title(), - IsStarted: taskStatus.IsStarted, - IsCompleted: taskStatus.IsStarted && !taskStatus.IsRunning, - StartTime: taskStatus.StartTime, - StopTime: taskStatus.StopTime, - Timeout: task.Timeout(), - HasTimeout: task.Timeout() > 0, + pageData.Tasks = append(pageData.Tasks, taskData) } - if !taskStatus.IsStarted { - taskData.Status = "pending" - } else if taskStatus.IsRunning { - taskData.Status = "running" - taskData.HasRunTime = true - taskData.RunTime = time.Since(taskStatus.StartTime) - } else { - taskData.Status = "complete" - taskData.HasRunTime = true - taskData.RunTime = taskStatus.StopTime.Sub(taskStatus.StartTime) - } - switch taskStatus.Result { - case types.TaskResultNone: - taskData.Result = "none" - case types.TaskResultSuccess: - taskData.Result = "success" - case types.TaskResultFailure: - taskData.Result = "failure" - } - if taskStatus.Error != nil { - taskData.ResultError = taskStatus.Error.Error() - } - - pageData.Tasks = append(pageData.Tasks, taskData) } return pageData, nil diff --git a/pkg/coordinator/web/templates/test/test.html b/pkg/coordinator/web/templates/test/test.html index 9b88ab6..31e5f9f 100644 --- a/pkg/coordinator/web/templates/test/test.html +++ b/pkg/coordinator/web/templates/test/test.html @@ -99,16 +99,6 @@

- {{ if eq $task.Status "pending" }} - - - - {{ else if eq $task.Status "running" }} - - - - {{ end }} - {{ if eq $task.Result "success" }} @@ -122,6 +112,16 @@

{{ end }} + + {{ if eq $task.Status "pending" }} + + + + {{ else if eq $task.Status "running" }} + + + + {{ end }}