Allow to define custom agents in system test configuration files

internal/agentdeployer/_static/docker-agent-base.yml → internal/agentdeployer/_static/docker-agent-base.yml.tmpl

@@ -7,6 +7,18 @@ services:
       test: "elastic-agent status"
       retries: 180
       interval: 1s
+    {{ if ne .pidMode "" }}
+    pid: {{ .pidMode }}
+    {{ end }}
+    {{ if ne .user "" }}
+    user: {{ .user }}
+    {{ end }}
+    {{ if .capabilities }}
+    cap_add:
+    {{- range .capabilities }}
+      - {{ . }}
+    {{- end }}
+    {{ end }}
     environment:
       - FLEET_ENROLL=1
       - FLEET_URL=https://fleet-server:8220
@@ -15,4 +27,3 @@ services:
     volumes:
       - ${SERVICE_LOGS_DIR}:/tmp/service_logs/
       - ${LOCAL_CA_CERT}:/etc/ssl/certs/elastic-package.pem
-

internal/agentdeployer/agent.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"text/template"
 	"time"
 
 	"github.com/elastic/elastic-package/internal/compose"
@@ -24,13 +25,12 @@ import (
 
 const (
 	dockerTestAgentNamePrefix    = "elastic-agent"
-	dockerTestgentDir            = "docker_test_agent"
 	dockerTestAgentDockerCompose = "docker-agent-base.yml"
 	defaultAgentPolicyName       = "Elastic-Agent (elastic-package)"
 )
 
-//go:embed _static/docker-agent-base.yml
-var dockerAgentDockerComposeContent []byte
+//go:embed _static/docker-agent-base.yml.tmpl
+var dockerTestAgentDockerComposeTemplate string
 
 // CustomAgentDeployer knows how to deploy a custom elastic-agent defined via
 // a Docker Compose file.
@@ -112,7 +112,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI
 		fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()),
 	)
 
-	configDir, err := d.installDockerfile()
+	configDir, err := d.installDockerfile(agentInfo)
 	if err != nil {
 		return nil, fmt.Errorf("could not create resources for custom agent: %w", err)
 	}
@@ -236,17 +236,29 @@ func (d *DockerComposeAgentDeployer) agentName() string {
 
 // installDockerfile creates the files needed to run the custom elastic agent and returns
 // the directory with these files.
-func (d *DockerComposeAgentDeployer) installDockerfile() (string, error) {
+func (d *DockerComposeAgentDeployer) installDockerfile(agentInfo AgentInfo) (string, error) {
 	customAgentDir := filepath.Join(d.profile.ProfilePath, fmt.Sprintf("agent-%s", d.agentName()))
 	err := os.MkdirAll(customAgentDir, 0755)
 	if err != nil {
 		return "", fmt.Errorf("failed to create directory for custom agent files: %w", err)
 	}
 
 	customAgentDockerfile := filepath.Join(customAgentDir, dockerTestAgentDockerCompose)
-	err = os.WriteFile(customAgentDockerfile, dockerAgentDockerComposeContent, 0644)
+	file, err := os.Create(customAgentDockerfile)
 	if err != nil {
-		return "", fmt.Errorf("failed to create docker compose file for custom agent: %w", err)
+		return "", fmt.Errorf("failed to create file (name %s): %w", customAgentDockerfile, err)
+	}
+	defer file.Close()
+
+	tmpl := template.Must(template.New(dockerTestAgentDockerCompose).Parse(dockerTestAgentDockerComposeTemplate))
+	err = tmpl.Execute(file, map[string]any{
+		"user":         agentInfo.Agent.User,
+		"capabilities": agentInfo.Agent.LinuxCapabilities,
+		"runtime":      agentInfo.Agent.Runtime,
+		"pidMode":      agentInfo.Agent.PidMode,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to create contents of the docker-compose file %q: %w", customAgentDockerfile, err)
 	}
 
 	return customAgentDir, nil

internal/agentdeployer/info.go

@@ -71,6 +71,17 @@ type AgentInfo struct {
 			// Name prefix for the host's name
 			NamePrefix string
 		}
+
+		// User user to run Elastic Agent process
+		User string
+		// PidMode selects the host PID mode
+		// (From docker-compose docs) Turns on sharing between container and the host
+		// operating system the PID address space
+		PidMode string
+		// Runtime is the selected runtime to run the Elastic Agent process
+		Runtime string
+		// LinuxCapabilities is a list of the capabilities needed to run the Elastic Agent process
+		LinuxCapabilities []string
 	}
 
 	// CustomProperties store additional data used to boot up the service, e.g. AWS credentials.

internal/testrunner/runners/system/runner.go

@@ -299,7 +299,7 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor
 	}
 }
 
-func (r *runner) createAgentInfo(policy *kibana.Policy) (agentdeployer.AgentInfo, error) {
+func (r *runner) createAgentInfo(policy *kibana.Policy, config *testConfig) (agentdeployer.AgentInfo, error) {
 	var info agentdeployer.AgentInfo
 
 	info.Name = r.options.TestFolder.Package
@@ -320,6 +320,11 @@ func (r *runner) createAgentInfo(policy *kibana.Policy) (agentdeployer.AgentInfo
 	info.Policy.Name = policy.Name
 	info.Policy.ID = policy.ID
 
+	info.Agent.User = config.Agent.User
+	info.Agent.LinuxCapabilities = config.Agent.LinuxCapabilities
+	info.Agent.Runtime = config.Agent.Runtime
+	info.Agent.PidMode = config.Agent.PidMode
+
 	return info, nil
 }
 
@@ -716,7 +721,7 @@ type scenarioTest struct {
 	syntheticEnabled   bool
 	docs               []common.MapStr
 	agent              agentdeployer.DeployedAgent
-	enrollingTime      time.Time
+	startTestTime      time.Time
 }
 
 func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInfo servicedeployer.ServiceInfo, serviceOptions servicedeployer.FactoryOptions) (*scenarioTest, error) {
@@ -792,17 +797,11 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		return nil
 	}
 
-	enrollingTime := time.Now()
-	if r.options.RunTearDown || r.options.RunTestsOnly {
-		enrollingTime = serviceStateData.EnrollingAgentTime
-	}
-
-	agentDeployed, agentInfo, err := r.setupAgent(ctx, serviceStateData, policy)
+	agentDeployed, agentInfo, err := r.setupAgent(ctx, config, serviceStateData, policy)
 	if err != nil {
 		return nil, err
 	}
 
-	scenario.enrollingTime = enrollingTime
 	scenario.agent = agentDeployed
 
 	service, svcInfo, err := r.setupService(ctx, config, serviceOptions, svcInfo, agentInfo, agentDeployed, policy, serviceStateData)
@@ -873,6 +872,10 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 	r.deletePackageHandler = deletePackageHandler
 
+	// store the time just before adding the Test Policy, this time will be used to check
+	// the agent logs from that time onwards to avoid possible previous errors present in logs
+	scenario.startTestTime = time.Now()
+
 	logger.Debug("adding package data stream to test policy...")
 	ds := createPackageDatastream(*policy, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config)
 	if r.options.RunTearDown || r.options.RunTestsOnly {
@@ -1083,7 +1086,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 			currentPolicy: policy,
 			config:        config,
 			agent:         origAgent,
-			enrollingTime: enrollingTime,
 			agentInfo:     agentInfo,
 			svcInfo:       svcInfo,
 		}
@@ -1143,12 +1145,12 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp
 	return service, service.Info(), nil
 }
 
-func (r *runner) setupAgent(ctx context.Context, state ServiceState, policy *kibana.Policy) (agentdeployer.DeployedAgent, agentdeployer.AgentInfo, error) {
+func (r *runner) setupAgent(ctx context.Context, config *testConfig, state ServiceState, policy *kibana.Policy) (agentdeployer.DeployedAgent, agentdeployer.AgentInfo, error) {
 	if !r.options.RunIndependentElasticAgent {
 		return nil, agentdeployer.AgentInfo{}, nil
 	}
 	logger.Warn("setting up agent (technical preview)...")
-	agentInfo, err := r.createAgentInfo(policy)
+	agentInfo, err := r.createAgentInfo(policy, config)
 	if err != nil {
 		return nil, agentdeployer.AgentInfo{}, err
 	}
@@ -1162,6 +1164,7 @@ func (r *runner) setupAgent(ctx context.Context, state ServiceState, policy *kib
 		return nil, agentInfo, fmt.Errorf("could not create agent runner: %w", err)
 	}
 	if agentDeployer == nil {
+		logger.Debug("Not found agent deployer. Agent will be created along with the service.")
 		return nil, agentInfo, nil
 	}
 
@@ -1215,38 +1218,35 @@ func (r *runner) readServiceStateData() (ServiceState, error) {
 }
 
 type ServiceState struct {
-	OrigPolicy         kibana.Policy `json:"orig_policy"`
-	CurrentPolicy      kibana.Policy `json:"current_policy"`
-	Agent              kibana.Agent  `json:"agent"`
-	ConfigFilePath     string        `json:"config_file_path"`
-	VariantName        string        `json:"variant_name"`
-	EnrollingAgentTime time.Time     `json:"enrolling_agent_time"`
-	ServiceRunID       string        `json:"service_info_run_id"`
-	AgentRunID         string        `json:"agent_info_run_id"`
-	ServiceOutputDir   string        `json:"service_output_dir"`
+	OrigPolicy       kibana.Policy `json:"orig_policy"`
+	CurrentPolicy    kibana.Policy `json:"current_policy"`
+	Agent            kibana.Agent  `json:"agent"`
+	ConfigFilePath   string        `json:"config_file_path"`
+	VariantName      string        `json:"variant_name"`
+	ServiceRunID     string        `json:"service_info_run_id"`
+	AgentRunID       string        `json:"agent_info_run_id"`
+	ServiceOutputDir string        `json:"service_output_dir"`
 }
 
 type scenarioStateOpts struct {
 	currentPolicy *kibana.Policy
 	origPolicy    *kibana.Policy
 	config        *testConfig
 	agent         kibana.Agent
-	enrollingTime time.Time
 	agentInfo     agentdeployer.AgentInfo
 	svcInfo       servicedeployer.ServiceInfo
 }
 
 func (r *runner) writeScenarioState(opts scenarioStateOpts) error {
 	data := ServiceState{
-		OrigPolicy:         *opts.origPolicy,
-		CurrentPolicy:      *opts.currentPolicy,
-		Agent:              opts.agent,
-		ConfigFilePath:     opts.config.Path,
-		VariantName:        opts.config.ServiceVariantName,
-		EnrollingAgentTime: opts.enrollingTime,
-		ServiceRunID:       opts.svcInfo.Test.RunID,
-		AgentRunID:         opts.agentInfo.Test.RunID,
-		ServiceOutputDir:   opts.svcInfo.OutputDir,
+		OrigPolicy:       *opts.origPolicy,
+		CurrentPolicy:    *opts.currentPolicy,
+		Agent:            opts.agent,
+		ConfigFilePath:   opts.config.Path,
+		VariantName:      opts.config.ServiceVariantName,
+		ServiceRunID:     opts.svcInfo.Test.RunID,
+		AgentRunID:       opts.agentInfo.Test.RunID,
+		ServiceOutputDir: opts.svcInfo.OutputDir,
 	}
 	dataBytes, err := json.Marshal(data)
 	if err != nil {
@@ -1366,7 +1366,7 @@ func (r *runner) validateTestScenario(ctx context.Context, result *testrunner.Re
 	}
 
 	if scenario.agent != nil {
-		logResults, err := r.checkNewAgentLogs(ctx, scenario.agent, scenario.enrollingTime, errorPatterns)
+		logResults, err := r.checkNewAgentLogs(ctx, scenario.agent, scenario.startTestTime, errorPatterns)
 		if err != nil {
 			return result.WithError(err)
 		}

internal/testrunner/runners/system/test_config.go

@@ -51,6 +51,14 @@ type testConfig struct {
 
 	Path               string `config:",ignore"` // Path of config file.
 	ServiceVariantName string `config:",ignore"` // Name of test variant when using variants.yml.
+
+	// Agent related properties
+	Agent struct {
+		User              string   `config:"user"`
+		PidMode           string   `config:"pid_mode"`
+		LinuxCapabilities []string `config:"linux_capabilities"`
+		Runtime           string   `config:"runtime"`
+	} `config:"agent"`
 }
 
 func (t testConfig) Name() string {

scripts/test-check-packages.sh

@@ -85,10 +85,10 @@ fi
 for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
   (
     cd "$d"
+    package_to_test=$(basename "${d}")
 
     if [ "${PACKAGE_TEST_TYPE:-other}" == "benchmarks" ]; then
       # It is not used PACKAGE_UNDER_TEST, so all benchmark packages are run in the same loop
-      package_to_test=$(basename "${d}")
       if [ "${package_to_test}" == "pipeline_benchmark" ]; then
         rm -rf "${OLDPWD}/build/benchmark-results"
         elastic-package benchmark pipeline -v --report-format xUnit --report-output file --fail-on-missing
@@ -109,6 +109,10 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
     elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then
         elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s
     else
+      if [[ "${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT}" == "false" && "${package_to_test}" == "auditd_manager_independent_agent" ]]; then
+          echo "Package \"${package_to_test}\" skipped: not supported with Elastic Agent running in the stack (missing capabilities)."
+          exit # as it is run in a subshell, it cannot be used "continue"
+      fi
       # defer-cleanup is set to a short period to verify that the option is available
       elastic-package test -v \
           --report-format xUnit \

test/packages/with-custom-agent/auditd_manager_independent_agent/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

test/packages/with-custom-agent/auditd_manager_independent_agent/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "999.999.999"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1

test/packages/with-custom-agent/auditd_manager_independent_agent/data_stream/auditd/_dev/test/system/test-default-config.yml

@@ -0,0 +1,12 @@
+data_stream:
+  vars:
+    audit_rules:
+      - "-a always,exit -F arch=b64 -S execve,execveat -k exec"
+    preserve_original_event: true
+agent:
+  runtime: docker
+  user: "root"
+  pid_mode: "host"
+  linux_capabilities:
+    - AUDIT_CONTROL
+    - AUDIT_READ

test/packages/with-custom-agent/auditd_manager_independent_agent/data_stream/auditd/agent/stream/auditd.yml.hbs

@@ -0,0 +1,34 @@
+# EQL expression evaluated by Elastic Agent to limit input to Linux.
+condition: ${host.platform} == 'linux'
+
+type: audit/auditd
+include_raw_message: true
+{{#if multicast}}
+socket_type: multicast
+{{else}}
+socket_type: unicast
+{{/if}}
+resolve_ids: {{resolve_ids}}
+failure_mode: {{failure_mode}}
+audit_rules: |
+{{#each audit_rules as |rule i|}}
+  {{rule}}
+{{/each}}
+backlog_limit: {{backlog_limit}}
+rate_limit: {{rate_limit}}
+include_warnings: {{include_warnings}}
+backpressure_strategy: {{backpressure_strategy}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}