Add (out-of-spec) support for JSONPath in fieldmappings (#1)

* Add basic support for (non-standard) JSONPath field mappings

* Add support for unmarshalling nested JSON

evaluator/evaluate_search.go

@@ -2,10 +2,13 @@ package evaluator
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"path"
+	"regexp"
 	"strings"
 
+	"github.com/PaesslerAG/jsonpath"
 	"github.com/bradleyjkemp/sigma-go"
 )
 
@@ -95,73 +98,117 @@ func (rule RuleEvaluator) evaluateSearch(search sigma.Search, event map[string]i
 		}
 
 		// field matchers can specify modifiers (FieldName|modifier1|modifier2) which change the matching behaviour
-		valueMatcher := baseMatcher
+		comparator := baseComparator
 		for _, name := range fieldModifiers {
 			if modifiers[name] == nil {
 				panic(fmt.Errorf("unsupported modifier %s", name))
 			}
-			valueMatcher = modifiers[name](valueMatcher)
+			comparator = modifiers[name](comparator)
 		}
 
-		fieldMatched := allValuesMustMatch
-		for _, value := range matcher.Values {
-			// There are multiple possible event fields that each value needs to be compared against
-			var valueMatches bool
-			if len(rule.fieldmappings[matcher.Field]) == 0 {
-				// No FieldMapping exists so use the name directly from the rule
-				valueMatches = valueMatcher(event[matcher.Field], value)
+		if !rule.matcherMatchesValues(matcher, comparator, allValuesMustMatch, event) {
+			// this field didn't match so the overall matcher doesn't match
+			return false
+		}
+	}
+
+	// all fields matched
+	return true
+}
+
+func (rule *RuleEvaluator) matcherMatchesValues(matcher sigma.FieldMatcher, comparator valueComparator, allValuesMustMatch bool, event map[string]interface{}) bool {
+	// First collect this list of event values we're matching against
+	var actualValues []interface{}
+	if len(rule.fieldmappings[matcher.Field]) == 0 {
+		// No FieldMapping exists so use the name directly from the rule
+		actualValues = []interface{}{event[matcher.Field]}
+	} else {
+		// FieldMapping does exist so check each of the possible mapped names instead of the name from the rule
+		for _, mapping := range rule.fieldmappings[matcher.Field] {
+			if strings.HasPrefix(mapping, "$.") || strings.HasPrefix(mapping, "$[") {
+				// This is a jsonpath expression
+				actualValues = append(actualValues, evaluateJSONPath(mapping, event))
 			} else {
-				// FieldMapping does exist so check each of the possible mapped names instead of the name from the rule
-				for _, field := range rule.fieldmappings[matcher.Field] {
-					valueMatches = valueMatcher(event[field], value)
-					if valueMatches {
-						break
-					}
-				}
+				// This is just a field name
+				actualValues = append(actualValues, event[mapping])
 			}
+		}
+	}
 
-			if allValuesMustMatch {
-				fieldMatched = fieldMatched && valueMatches
-			} else {
-				fieldMatched = fieldMatched || valueMatches
+	matched := allValuesMustMatch
+	for _, expectedValue := range matcher.Values {
+		valueMatchedEvent := false
+		// There are multiple possible event fields that each expected value needs to be compared against
+		for _, actualValue := range actualValues {
+			if comparator(actualValue, expectedValue) {
+				valueMatchedEvent = true
+				break
 			}
 		}
 
-		if !fieldMatched {
-			// this field didn't match so the overall matcher doesn't match
-			return false
+		if allValuesMustMatch {
+			matched = matched && valueMatchedEvent
+		} else {
+			matched = matched || valueMatchedEvent
 		}
 	}
+	return matched
+}
 
-	// all fields matched
-	return true
+// This is a hack because none of the JSONPath libraries expose the parsed AST :(
+// Matches JSONPaths with either a $.fieldname or $["fieldname"] prefix and extracts 'fieldname'
+var firstJSONPathField = regexp.MustCompile(`^\$(?:[.]|\[")([a-zA-Z0-9_\-]+)(?:"])?`)
+
+func evaluateJSONPath(expr string, event map[string]interface{}) interface{} {
+	jsonPathField := firstJSONPathField.FindStringSubmatch(expr)
+	if jsonPathField == nil {
+		panic("couldn't parse JSONPath expression")
+	}
+	fmt.Println("evaluating JSONPath", expr, event, jsonPathField)
+
+	var subValue interface{}
+	switch sub := event[jsonPathField[1]].(type) {
+	case string:
+		json.Unmarshal([]byte(sub), &subValue)
+	case []byte:
+		json.Unmarshal(sub, &subValue)
+	default:
+		// Oh well, don't try to unmarshal the nested field
+		value, _ := jsonpath.Get(expr, event)
+		return value
+	}
+
+	value, _ := jsonpath.Get(expr, map[string]interface{}{
+		jsonPathField[1]: subValue,
+	})
+	return value
 }
 
-type valueMatcher func(actual interface{}, expected string) bool
+type valueComparator func(actual interface{}, expected string) bool
 
-func baseMatcher(actual interface{}, expected string) bool {
+func baseComparator(actual interface{}, expected string) bool {
 	return fmt.Sprintf("%v", actual) == expected
 }
 
-type valueModifier func(next valueMatcher) valueMatcher
+type valueModifier func(next valueComparator) valueComparator
 
 var modifiers = map[string]valueModifier{
-	"contains": func(next valueMatcher) valueMatcher {
+	"contains": func(next valueComparator) valueComparator {
 		return func(actual interface{}, expected string) bool {
 			return strings.Contains(fmt.Sprintf("%v", actual), expected)
 		}
 	},
-	"endswith": func(next valueMatcher) valueMatcher {
+	"endswith": func(next valueComparator) valueComparator {
 		return func(actual interface{}, expected string) bool {
 			return strings.HasSuffix(fmt.Sprintf("%v", actual), expected)
 		}
 	},
-	"startswith": func(next valueMatcher) valueMatcher {
+	"startswith": func(next valueComparator) valueComparator {
 		return func(actual interface{}, expected string) bool {
 			return strings.HasPrefix(fmt.Sprintf("%v", actual), expected)
 		}
 	},
-	"base64": func(next valueMatcher) valueMatcher {
+	"base64": func(next valueComparator) valueComparator {
 		return func(actual interface{}, expected string) bool {
 			return next(actual, base64.StdEncoding.EncodeToString([]byte(expected)))
 		}

evaluator/fieldmappings_test.go

@@ -44,3 +44,75 @@ func TestRuleEvaluator_HandlesBasicFieldMappings(t *testing.T) {
 		t.Error("If a field is mapped, the mapped name should work")
 	}
 }
+
+func TestRuleEvaluator_HandlesJSONPathFieldMappings(t *testing.T) {
+	rule := ForRule(sigma.Rule{
+		Logsource: sigma.Logsource{
+			Category: "category",
+			Product:  "product",
+			Service:  "service",
+		},
+		Detection: sigma.Detection{
+			Searches: map[string]sigma.Search{
+				"test": {
+					FieldMatchers: []sigma.FieldMatcher{{
+						Field:  "name",
+						Values: []string{"value"},
+					}},
+				},
+			},
+			Conditions: []sigma.Condition{
+				{Search: sigma.SearchIdentifier{Name: "test"}}},
+		},
+	}, WithConfig(sigma.Config{
+		FieldMappings: map[string]sigma.FieldMapping{
+			"name": {TargetNames: []string{"$.mapped.name"}},
+		},
+	}))
+
+	if rule.Matches(context.Background(), map[string]interface{}{
+		"name": "value",
+	}) {
+		t.Error("If a field is mapped, the old name shouldn't be used")
+	}
+
+	if !rule.Matches(context.Background(), map[string]interface{}{
+		"mapped": map[string]interface{}{
+			"name": "value",
+		},
+	}) {
+		t.Error("If a fieldmapping is a JSONPath expression, the nested field should be matched")
+	}
+}
+
+func TestRuleEvaluator_HandlesJSONPathByteSlice(t *testing.T) {
+	rule := ForRule(sigma.Rule{
+		Logsource: sigma.Logsource{
+			Category: "category",
+			Product:  "product",
+			Service:  "service",
+		},
+		Detection: sigma.Detection{
+			Searches: map[string]sigma.Search{
+				"test": {
+					FieldMatchers: []sigma.FieldMatcher{{
+						Field:  "name",
+						Values: []string{"value"},
+					}},
+				},
+			},
+			Conditions: []sigma.Condition{
+				{Search: sigma.SearchIdentifier{Name: "test"}}},
+		},
+	}, WithConfig(sigma.Config{
+		FieldMappings: map[string]sigma.FieldMapping{
+			"name": {TargetNames: []string{"$.mapped.name"}},
+		},
+	}))
+
+	if !rule.Matches(context.Background(), map[string]interface{}{
+		"mapped": `{"name": "value"}`,
+	}) {
+		t.Error("If a JSONPath expression encounters a string, the string should be parsed and then matched")
+	}
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/bradleyjkemp/sigma-go
 go 1.15
 
 require (
+	github.com/PaesslerAG/jsonpath v0.1.1
 	github.com/alecthomas/participle v0.6.0
 	github.com/bradleyjkemp/cupaloy/v2 v2.6.0
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776

go.sum

@@ -1,3 +1,8 @@
+github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
+github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
+github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
+github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
 github.com/alecthomas/participle v0.6.0 h1:Pvo8XUCQKgIywVjz/+Ci3IsjGg+g/TdKkMcfgghKCEw=
 github.com/alecthomas/participle v0.6.0/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY=
 github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=