Support multiple appsec configs (crowdsecurity#3314)

* support multiple appsec configs

pkg/acquisition/modules/appsec/appsec.go

@@ -41,6 +41,7 @@ type AppsecSourceConfig struct {
 	Path                              string         `yaml:"path"`
 	Routines                          int            `yaml:"routines"`
 	AppsecConfig                      string         `yaml:"appsec_config"`
+	AppsecConfigs                     []string       `yaml:"appsec_configs"`
 	AppsecConfigPath                  string         `yaml:"appsec_config_path"`
 	AuthCacheDuration                 *time.Duration `yaml:"auth_cache_duration"`
 	configuration.DataSourceCommonCfg `yaml:",inline"`
@@ -121,10 +122,14 @@ func (w *AppsecSource) UnmarshalConfig(yamlConfig []byte) error {
 		w.config.Routines = 1
 	}
 
-	if w.config.AppsecConfig == "" && w.config.AppsecConfigPath == "" {
+	if w.config.AppsecConfig == "" && w.config.AppsecConfigPath == "" && len(w.config.AppsecConfigs) == 0 {
 		return errors.New("appsec_config or appsec_config_path must be set")
 	}
 
+	if (w.config.AppsecConfig != "" || w.config.AppsecConfigPath != "") && len(w.config.AppsecConfigs) != 0 {
+		return errors.New("appsec_config and appsec_config_path are mutually exclusive with appsec_configs")
+	}
+
 	if w.config.Name == "" {
 		if w.config.ListenSocket != "" && w.config.ListenAddr == "" {
 			w.config.Name = w.config.ListenSocket
@@ -175,6 +180,9 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLe
 	w.InChan = make(chan appsec.ParsedRequest)
 	appsecCfg := appsec.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")}
 
+	//we keep the datasource name
+	appsecCfg.Name = w.config.Name
+
 	// let's load the associated appsec_config:
 	if w.config.AppsecConfigPath != "" {
 		err := appsecCfg.LoadByPath(w.config.AppsecConfigPath)
@@ -186,10 +194,20 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLe
 		if err != nil {
 			return fmt.Errorf("unable to load appsec_config: %w", err)
 		}
+	} else if len(w.config.AppsecConfigs) > 0 {
+		for _, appsecConfig := range w.config.AppsecConfigs {
+			err := appsecCfg.Load(appsecConfig)
+			if err != nil {
+				return fmt.Errorf("unable to load appsec_config: %w", err)
+			}
+		}
 	} else {
 		return errors.New("no appsec_config provided")
 	}
 
+	// Now we can set up the logger
+	appsecCfg.SetUpLogger()
+
 	w.AppsecRuntime, err = appsecCfg.Build()
 	if err != nil {
 		return fmt.Errorf("unable to build appsec_config: %w", err)

pkg/acquisition/modules/appsec/appsec_rules_test.go

@@ -373,7 +373,7 @@ toto
 		{
 			name:             "Basic matching IP address",
 			expected_load_ok: true,
-			seclang_rules: []string{
+			inband_native_rules: []string{
 				"SecRule REMOTE_ADDR \"@ipMatch 1.2.3.4\" \"id:1,phase:1,log,deny,msg: 'block ip'\"",
 			},
 			input_request: appsec.ParsedRequest{

pkg/acquisition/modules/appsec/appsec_runner.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -31,23 +32,38 @@ type AppsecRunner struct {
 	logger              *log.Entry
 }
 
+func (r *AppsecRunner) MergeDedupRules(collections []appsec.AppsecCollection, logger *log.Entry) string {
+	var rulesArr []string
+	dedupRules := make(map[string]struct{})
+
+	for _, collection := range collections {
+		for _, rule := range collection.Rules {
+			if _, ok := dedupRules[rule]; !ok {
+				rulesArr = append(rulesArr, rule)
+				dedupRules[rule] = struct{}{}
+			} else {
+				logger.Debugf("Discarding duplicate rule : %s", rule)
+			}
+		}
+	}
+	if len(rulesArr) != len(dedupRules) {
+		logger.Warningf("%d rules were discarded as they were duplicates", len(rulesArr)-len(dedupRules))
+	}
+
+	return strings.Join(rulesArr, "\n")
+}
+
 func (r *AppsecRunner) Init(datadir string) error {
 	var err error
 	fs := os.DirFS(datadir)
 
-	inBandRules := ""
-	outOfBandRules := ""
-
-	for _, collection := range r.AppsecRuntime.InBandRules {
-		inBandRules += collection.String()
-	}
-
-	for _, collection := range r.AppsecRuntime.OutOfBandRules {
-		outOfBandRules += collection.String()
-	}
 	inBandLogger := r.logger.Dup().WithField("band", "inband")
 	outBandLogger := r.logger.Dup().WithField("band", "outband")
 
+	//While loading rules, we dedup rules based on their content, while keeping the order
+	inBandRules := r.MergeDedupRules(r.AppsecRuntime.InBandRules, inBandLogger)
+	outOfBandRules := r.MergeDedupRules(r.AppsecRuntime.OutOfBandRules, outBandLogger)
+
 	//setting up inband engine
 	inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(inBandLogger))
 	if !r.AppsecRuntime.Config.InbandOptions.DisableBodyInspection {

pkg/acquisition/modules/appsec/appsec_runner_test.go

@@ -0,0 +1,139 @@
+package appsecacquisition
+
+import (
+	"testing"
+
+	"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppsecRuleLoad(t *testing.T) {
+	log.SetLevel(log.TraceLevel)
+	tests := []appsecRuleTest{
+		{
+			name:             "simple rule load",
+			expected_load_ok: true,
+			inband_rules: []appsec_rule.CustomRule{
+				{
+					Name:  "rule1",
+					Zones: []string{"ARGS"},
+					Match: appsec_rule.Match{Type: "equals", Value: "toto"},
+				},
+			},
+			afterload_asserts: func(runner AppsecRunner) {
+				require.Len(t, runner.AppsecInbandEngine.GetRuleGroup().GetRules(), 1)
+			},
+		},
+		{
+			name:             "simple native rule load",
+			expected_load_ok: true,
+			inband_native_rules: []string{
+				`Secrule REQUEST_HEADERS:Content-Type "@rx ^application/x-www-form-urlencoded" "id:100,phase:1,pass,nolog,noauditlog,ctl:requestBodyProcessor=URLENCODED"`,
+			},
+			afterload_asserts: func(runner AppsecRunner) {
+				require.Len(t, runner.AppsecInbandEngine.GetRuleGroup().GetRules(), 1)
+			},
+		},
+		{
+			name:             "simple native rule load (2)",
+			expected_load_ok: true,
+			inband_native_rules: []string{
+				`Secrule REQUEST_HEADERS:Content-Type "@rx ^application/x-www-form-urlencoded" "id:100,phase:1,pass,nolog,noauditlog,ctl:requestBodyProcessor=URLENCODED"`,
+				`Secrule REQUEST_HEADERS:Content-Type "@rx ^multipart/form-data" "id:101,phase:1,pass,nolog,noauditlog,ctl:requestBodyProcessor=MULTIPART"`,
+			},
+			afterload_asserts: func(runner AppsecRunner) {
+				require.Len(t, runner.AppsecInbandEngine.GetRuleGroup().GetRules(), 2)
+			},
+		},
+		{
+			name:             "simple native rule load + dedup",
+			expected_load_ok: true,
+			inband_native_rules: []string{
+				`Secrule REQUEST_HEADERS:Content-Type "@rx ^application/x-www-form-urlencoded" "id:100,phase:1,pass,nolog,noauditlog,ctl:requestBodyProcessor=URLENCODED"`,
+				`Secrule REQUEST_HEADERS:Content-Type "@rx ^multipart/form-data" "id:101,phase:1,pass,nolog,noauditlog,ctl:requestBodyProcessor=MULTIPART"`,
+				`Secrule REQUEST_HEADERS:Content-Type "@rx ^application/x-www-form-urlencoded" "id:100,phase:1,pass,nolog,noauditlog,ctl:requestBodyProcessor=URLENCODED"`,
+			},
+			afterload_asserts: func(runner AppsecRunner) {
+				require.Len(t, runner.AppsecInbandEngine.GetRuleGroup().GetRules(), 2)
+			},
+		},
+		{
+			name:             "multi simple rule load",
+			expected_load_ok: true,
+			inband_rules: []appsec_rule.CustomRule{
+				{
+					Name:  "rule1",
+					Zones: []string{"ARGS"},
+					Match: appsec_rule.Match{Type: "equals", Value: "toto"},
+				},
+				{
+					Name:  "rule2",
+					Zones: []string{"ARGS"},
+					Match: appsec_rule.Match{Type: "equals", Value: "toto"},
+				},
+			},
+			afterload_asserts: func(runner AppsecRunner) {
+				require.Len(t, runner.AppsecInbandEngine.GetRuleGroup().GetRules(), 2)
+			},
+		},
+		{
+			name:             "multi simple rule load",
+			expected_load_ok: true,
+			inband_rules: []appsec_rule.CustomRule{
+				{
+					Name:  "rule1",
+					Zones: []string{"ARGS"},
+					Match: appsec_rule.Match{Type: "equals", Value: "toto"},
+				},
+				{
+					Name:  "rule2",
+					Zones: []string{"ARGS"},
+					Match: appsec_rule.Match{Type: "equals", Value: "toto"},
+				},
+			},
+			afterload_asserts: func(runner AppsecRunner) {
+				require.Len(t, runner.AppsecInbandEngine.GetRuleGroup().GetRules(), 2)
+			},
+		},
+		{
+			name:             "imbricated rule load",
+			expected_load_ok: true,
+			inband_rules: []appsec_rule.CustomRule{
+				{
+					Name: "rule1",
+
+					Or: []appsec_rule.CustomRule{
+						{
+							//Name:  "rule1",
+							Zones: []string{"ARGS"},
+							Match: appsec_rule.Match{Type: "equals", Value: "toto"},
+						},
+						{
+							//Name:  "rule1",
+							Zones: []string{"ARGS"},
+							Match: appsec_rule.Match{Type: "equals", Value: "tutu"},
+						},
+						{
+							//Name:  "rule1",
+							Zones: []string{"ARGS"},
+							Match: appsec_rule.Match{Type: "equals", Value: "tata"},
+						}, {
+							//Name:  "rule1",
+							Zones: []string{"ARGS"},
+							Match: appsec_rule.Match{Type: "equals", Value: "titi"},
+						},
+					},
+				},
+			},
+			afterload_asserts: func(runner AppsecRunner) {
+				require.Len(t, runner.AppsecInbandEngine.GetRuleGroup().GetRules(), 4)
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			loadAppSecEngine(test, t)
+		})
+	}
+}

pkg/acquisition/modules/appsec/appsec_test.go

@@ -18,7 +18,8 @@ type appsecRuleTest struct {
 	expected_load_ok       bool
 	inband_rules           []appsec_rule.CustomRule
 	outofband_rules        []appsec_rule.CustomRule
-	seclang_rules          []string
+	inband_native_rules    []string
+	outofband_native_rules []string
 	on_load                []appsec.Hook
 	pre_eval               []appsec.Hook
 	post_eval              []appsec.Hook
@@ -29,6 +30,7 @@ type appsecRuleTest struct {
 	DefaultRemediation     string
 	DefaultPassAction      string
 	input_request          appsec.ParsedRequest
+	afterload_asserts      func(runner AppsecRunner)
 	output_asserts         func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int)
 }
 
@@ -54,6 +56,8 @@ func loadAppSecEngine(test appsecRuleTest, t *testing.T) {
 		inbandRules = append(inbandRules, strRule)
 
 	}
+	inbandRules = append(inbandRules, test.inband_native_rules...)
+	outofbandRules = append(outofbandRules, test.outofband_native_rules...)
 	for ridx, rule := range test.outofband_rules {
 		strRule, _, err := rule.Convert(appsec_rule.ModsecurityRuleType, rule.Name)
 		if err != nil {
@@ -62,8 +66,6 @@ func loadAppSecEngine(test appsecRuleTest, t *testing.T) {
 		outofbandRules = append(outofbandRules, strRule)
 	}
 
-	inbandRules = append(inbandRules, test.seclang_rules...)
-
 	appsecCfg := appsec.AppsecConfig{Logger: logger,
 		OnLoad:                 test.on_load,
 		PreEval:                test.pre_eval,
@@ -97,6 +99,13 @@ func loadAppSecEngine(test appsecRuleTest, t *testing.T) {
 		t.Fatalf("unable to initialize runner : %s", err)
 	}
 
+	if test.afterload_asserts != nil {
+		//afterload asserts are just to evaluate the state of the runner after the rules have been loaded
+		//if it's present, don't try to process requests
+		test.afterload_asserts(runner)
+		return
+	}
+
 	input := test.input_request
 	input.ResponseChannel = make(chan appsec.AppsecTempResponse)
 	OutputEvents := make([]types.Event, 0)