Add config option to deploy custom elastic-agents as test services

.ci/Jenkinsfile

@@ -103,6 +103,7 @@ pipeline {
                     'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']),
                     'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
                     'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
+                    'check-packages-with-custom-agent': generateTestCommandStage(command: 'test-check-packages-with-custom-agent', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
                     'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']),
                     'profiles-command': generateTestCommandStage(command: 'test-profiles-command')
                   ]

Makefile

@@ -65,7 +65,7 @@ test-stack-command-8x:
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 
-test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel
+test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent
 
 test-check-packages-with-kind:
 	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
@@ -76,6 +76,9 @@ test-check-packages-other:
 test-check-packages-parallel:
 	PACKAGE_TEST_TYPE=parallel ./scripts/test-check-packages.sh
 
+test-check-packages-with-custom-agent:
+	PACKAGE_TEST_TYPE=with-custom-agent ./scripts/test-check-packages.sh
+
 test-build-zip:
 	./scripts/test-build-zip.sh
 

docs/howto/system_testing.md

@@ -64,6 +64,7 @@ or the data stream's level:
 
 `<service deployer>` - a name of the supported service deployer:
 * `docker` - Docker Compose
+* `agent` - Custom `elastic-agent` with Docker Compose
 * `k8s` - Kubernetes
 * `tf` - Terraform
 
@@ -106,6 +107,58 @@ volumes:
   mysqldata:
 ```
 
+### Agent service deployer
+
+When using the Agent service deployer, the `elastic-agent` provided by the stack
+will not be used. An agent will be deployed as a Docker compose service named `docker-custom-agent`
+which base configuration is provided [here](../../internal/install/_static/docker-custom-agent-base.yml).
+This configuration will be merged with the one provided in the `custom-agent.yml` file.
+This is useful if you need different capabilities than the provided by the
+`elastic-agent` used by the `elastic-package stack` command. 
+
+`custom-agent.yml`
+```
+version: '2.3'
+services:
+  docker-custom-agent:
+    pid: host
+    cap_add:
+      - AUDIT_CONTROL
+      - AUDIT_READ
+    user: root
+```
+
+This will result in an agent configuration such as:
+
+```
+version: '2.3'
+services:
+  docker-custom-agent:
+    hostname: docker-custom-agent
+    image: "docker.elastic.co/beats/elastic-agent-complete:8.2.0"
+    pid: host
+    cap_add:
+      - AUDIT_CONTROL
+      - AUDIT_READ
+    user: root
+    healthcheck:
+      test: "elastic-agent status"
+      retries: 180
+      interval: 1s
+    environment:
+      FLEET_ENROLL: "1"
+      FLEET_INSECURE: "1"
+      FLEET_URL: "http://fleet-server:8220"
+```
+
+And in the test config:
+
+```
+data_stream:
+  vars:
+  # ...
+```
+
 
 ### Terraform service deployer
 

internal/configuration/locations/locations.go

@@ -27,12 +27,15 @@ const (
 	fieldsCachedDir = "cache/fields"
 
 	terraformDeployerYmlFile = "terraform-deployer.yml"
+
+	dockerCustomAgentDeployerYmlFile = "docker-custom-agent-base.yml"
 )
 
 var (
-	serviceLogsDir        = filepath.Join(temporaryDir, "service_logs")
-	kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes")
-	terraformDeployerDir  = filepath.Join(deployerDir, "terraform")
+	serviceLogsDir               = filepath.Join(temporaryDir, "service_logs")
+	kubernetesDeployerDir        = filepath.Join(deployerDir, "kubernetes")
+	terraformDeployerDir         = filepath.Join(deployerDir, "terraform")
+	dockerCustomAgentDeployerDir = filepath.Join(deployerDir, "docker_custom_agent")
 )
 
 //LocationManager maintains an instance of a config path location
@@ -96,6 +99,16 @@ func (loc LocationManager) TerraformDeployerYml() string {
 	return filepath.Join(loc.stackPath, terraformDeployerDir, terraformDeployerYmlFile)
 }
 
+// DockerCustomAgentDeployerDir returns the DockerCustomAgent Directory
+func (loc LocationManager) DockerCustomAgentDeployerDir() string {
+	return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir)
+}
+
+// DockerCustomAgentDeployerYml returns the DockerCustomAgent deployer yml file
+func (loc LocationManager) DockerCustomAgentDeployerYml() string {
+	return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir, dockerCustomAgentDeployerYmlFile)
+}
+
 // ServiceLogDir returns the log directory
 func (loc LocationManager) ServiceLogDir() string {
 	return filepath.Join(loc.stackPath, serviceLogsDir)

internal/install/_static/docker-custom-agent-base.yml

@@ -0,0 +1,15 @@
+version: "2.3"
+services:
+  docker-custom-agent:
+    image: "${ELASTIC_AGENT_IMAGE_REF}"
+    healthcheck:
+      test: "elastic-agent status"
+      retries: 180
+      interval: 1s
+    hostname: docker-custom-agent
+    environment:
+      - FLEET_ENROLL=1
+      - FLEET_INSECURE=1
+      - FLEET_URL=http://fleet-server:8220
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/tmp/service_logs/

internal/install/install.go

@@ -67,6 +67,11 @@ func EnsureInstalled() error {
 		return errors.Wrap(err, "writing Terraform deployer resources failed")
 	}
 
+	err = writeDockerCustomAgentResources(elasticPackagePath)
+	if err != nil {
+		return errors.Wrap(err, "writing Terraform deployer resources failed")
+	}
+
 	if err := createServiceLogsDir(elasticPackagePath); err != nil {
 		return errors.Wrap(err, "creating service logs directory failed")
 	}
@@ -218,6 +223,17 @@ func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManag
 	return nil
 }
 
+func writeDockerCustomAgentResources(elasticPackagePath *locations.LocationManager) error {
+	dir := elasticPackagePath.DockerCustomAgentDeployerDir()
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return errors.Wrapf(err, "creating directory failed (path: %s)", dir)
+	}
+	if err := writeStaticResource(nil, elasticPackagePath.DockerCustomAgentDeployerYml(), dockerCustomAgentBaseYml); err != nil {
+		return errors.Wrap(err, "writing static resource failed")
+	}
+	return nil
+}
+
 func writeConfigFile(elasticPackagePath *locations.LocationManager) error {
 	var err error
 	err = writeStaticResource(err, filepath.Join(elasticPackagePath.RootDir(), applicationConfigurationYmlFile), applicationConfigurationYml)

internal/install/static.go

@@ -29,3 +29,6 @@ var geoIpCountryMmdb string
 
 //go:embed _static/service_tokens
 var serviceTokens string
+
+//go:embed _static/docker-custom-agent-base.yml
+var dockerCustomAgentBaseYml string

internal/testrunner/runners/system/servicedeployer/custom_agent.go

@@ -0,0 +1,151 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package servicedeployer
+
+import (
+	_ "embed"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/compose"
+	"github.com/elastic/elastic-package/internal/configuration/locations"
+	"github.com/elastic/elastic-package/internal/docker"
+	"github.com/elastic/elastic-package/internal/files"
+	"github.com/elastic/elastic-package/internal/install"
+	"github.com/elastic/elastic-package/internal/kibana"
+	"github.com/elastic/elastic-package/internal/logger"
+	"github.com/elastic/elastic-package/internal/stack"
+)
+
+const dockerCustomAgentName = "docker-custom-agent"
+
+// CustomAgentDeployer knows how to deploy a custom elastic-agent defined via
+// a Docker Compose file.
+type CustomAgentDeployer struct {
+	cfg string
+}
+
+// NewCustomAgentDeployer returns a new instance of a deployedCustomAgent.
+func NewCustomAgentDeployer(cfgPath string) (*CustomAgentDeployer, error) {
+	return &CustomAgentDeployer{
+		cfg: cfgPath,
+	}, nil
+}
+
+// SetUp sets up the service and returns any relevant information.
+func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) {
+	logger.Debug("setting up service using Docker Compose service deployer")
+
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't read application configuration")
+	}
+
+	kibanaClient, err := kibana.NewClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create Kibana client")
+	}
+
+	stackVersion, err := kibanaClient.Version()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't read Kibana injected metadata")
+	}
+
+	env := append(
+		appConfig.StackImageRefs(stackVersion).AsEnv(),
+		fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local),
+	)
+
+	ymlPaths, err := d.loadComposeDefinitions()
+	if err != nil {
+		return nil, err
+	}
+
+	service := dockerComposeDeployedService{
+		ymlPaths: ymlPaths,
+		project:  "elastic-package-service",
+		sv: ServiceVariant{
+			Name: dockerCustomAgentName,
+			Env:  env,
+		},
+	}
+
+	outCtxt := inCtxt
+
+	p, err := compose.NewProject(service.project, service.ymlPaths...)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create Docker Compose project for service")
+	}
+
+	// Verify the Elastic stack network
+	err = stack.EnsureStackNetworkUp()
+	if err != nil {
+		return nil, errors.Wrap(err, "Elastic stack network is not ready")
+	}
+
+	// Clean service logs
+	err = files.RemoveContent(outCtxt.Logs.Folder.Local)
+	if err != nil {
+		return nil, errors.Wrap(err, "removing service logs failed")
+	}
+
+	inCtxt.Name = dockerCustomAgentName
+	serviceName := inCtxt.Name
+	opts := compose.CommandOptions{
+		Env:       env,
+		ExtraArgs: []string{"--build", "-d"},
+	}
+	err = p.Up(opts)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not boot up service using Docker Compose")
+	}
+
+	// Connect service network with stack network (for the purpose of metrics collection)
+	err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network())
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't attach service container to the stack network")
+	}
+
+	err = p.WaitForHealthy(opts)
+	if err != nil {
+		processServiceContainerLogs(p, compose.CommandOptions{
+			Env: opts.Env,
+		}, outCtxt.Name)
+		return nil, errors.Wrap(err, "service is unhealthy")
+	}
+
+	// Build service container name
+	outCtxt.Hostname = p.ContainerName(serviceName)
+
+	logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName))
+	serviceComposeConfig, err := p.Config(compose.CommandOptions{Env: env})
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get Docker Compose configuration for service")
+	}
+
+	s := serviceComposeConfig.Services[serviceName]
+	outCtxt.Ports = make([]int, len(s.Ports))
+	for idx, port := range s.Ports {
+		outCtxt.Ports[idx] = port.InternalPort
+	}
+
+	// Shortcut to first port for convenience
+	if len(outCtxt.Ports) > 0 {
+		outCtxt.Port = outCtxt.Ports[0]
+	}
+
+	outCtxt.Agent.Host.NamePrefix = inCtxt.Name
+	service.ctxt = outCtxt
+	return &service, nil
+}
+
+func (d *CustomAgentDeployer) loadComposeDefinitions() ([]string, error) {
+	locationManager, err := locations.NewLocationManager()
+	if err != nil {
+		return nil, errors.Wrap(err, "can't locate Docker Compose file for Custom Agent deployer")
+	}
+	return []string{locationManager.DockerCustomAgentDeployerYml(), d.cfg}, nil
+}

internal/testrunner/runners/system/servicedeployer/factory.go

@@ -51,6 +51,13 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
 			}
 			return NewDockerComposeServiceDeployer([]string{dockerComposeYMLPath}, sv)
 		}
+	case "agent":
+		customAgentCfgYMLPath := filepath.Join(serviceDeployerPath, "custom-agent.yml")
+		if _, err := os.Stat(customAgentCfgYMLPath); err != nil {
+			return nil, errors.Wrap(err, "can't find expected file custom-agent.yml")
+		}
+		return NewCustomAgentDeployer(customAgentCfgYMLPath)
+
 	case "tf":
 		if _, err := os.Stat(serviceDeployerPath); err == nil {
 			return NewTerraformServiceDeployer(serviceDeployerPath)

test/packages/with-custom-agent/auditd_manager/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

test/packages/with-custom-agent/auditd_manager/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "999.999.999"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1

test/packages/with-custom-agent/auditd_manager/data_stream/auditd/_dev/deploy/agent/custom-agent.yml

@@ -0,0 +1,7 @@
+services:
+  docker-custom-agent:
+    pid: host
+    cap_add:
+      - AUDIT_CONTROL
+      - AUDIT_READ
+    user: root

test/packages/with-custom-agent/auditd_manager/data_stream/auditd/_dev/test/system/test-default-config.yml

@@ -0,0 +1,5 @@
+data_stream:
+  vars:
+    audit_rules:
+      - "-a always,exit -F arch=b64 -S execve,execveat -k exec"
+    preserve_original_event: true