Add apm_config.obfuscation.cache.max_size to set the maximum size of obfuscation query cache

comp/trace/config/config_test.go

@@ -1776,6 +1776,22 @@ func TestLoadEnv(t *testing.T) {
 		assert.False(t, cfg.Obfuscation.Cache.Enabled)
 	})
 
+	env = "DD_APM_OBFUSCATION_CACHE_MAX_SIZE"
+	t.Run(env, func(t *testing.T) {
+		t.Setenv(env, "1234567")
+
+		c := buildConfigComponent(t, true, fx.Replace(corecomp.MockParams{
+			Params: corecomp.Params{ConfFilePath: "./testdata/full.yaml"},
+		}))
+		cfg := c.Object()
+
+		assert.NotNil(t, cfg)
+		actualConfig := pkgconfigsetup.Datadog().GetString("apm_config.obfuscation.cache.max_size")
+		actualParsed := cfg.Obfuscation.Cache.MaxSize
+		assert.Equal(t, "1234567", actualConfig)
+		assert.Equal(t, int64(1234567), actualParsed)
+	})
+
 	env = "DD_APM_PROFILING_ADDITIONAL_ENDPOINTS"
 	t.Run(env, func(t *testing.T) {
 		t.Setenv(env, `{"url1": ["key1", "key2"], "url2": ["key3"]}`)

comp/trace/config/setup.go

@@ -436,6 +436,7 @@ func applyDatadogConfig(c *config.AgentConfig, core corecompcfg.Component) error
 	c.Obfuscation.CreditCards.Luhn = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.credit_cards.luhn")
 	c.Obfuscation.CreditCards.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.credit_cards.keep_values")
 	c.Obfuscation.Cache.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.cache.enabled")
+	c.Obfuscation.Cache.MaxSize = pkgconfigsetup.Datadog().GetInt64("apm_config.obfuscation.cache.max_size")
 
 	if core.IsSet("apm_config.filter_tags.require") {
 		tags := core.GetStringSlice("apm_config.filter_tags.require")

comp/trace/config/testdata/full.yaml

@@ -91,3 +91,4 @@ apm_config:
       luhn: true
     cache:
       enabled: true
+      max_size: 5555555

pkg/collector/python/test_datadog_agent.go

@@ -166,6 +166,7 @@ func testObfuscaterConfig(t *testing.T) {
 		},
 		Cache: obfuscate.CacheConfig{
 			Enabled: true,
+			MaxSize: 5000000,
 		},
 	}
 	assert.Equal(t, expected, obfuscaterConfig)

pkg/config/config_template.yaml

@@ -1265,6 +1265,10 @@ api_key:
   ##        Enables caching obfuscated statements. Currently supported for SQL and MongoDB queries.
   ##        Enabled by default.
   #         enabled: true
+  ##        @param DD_APM_OBFUSCATION_CACHE_MAX_SIZE - integer - optional - default: 5000000
+  ##        The maximum size of the cache in bytes. The maximum allowed resource length is 5000.
+  ##        We can store a minimum of 1000 queries (= 5000000 / 5000) by default.
+  #         max_size: 5000000
 
   ## @param filter_tags - object - optional
   ## @env DD_APM_FILTER_TAGS_REQUIRE - object - optional

pkg/config/setup/apm.go

@@ -44,6 +44,7 @@ func setupAPM(config pkgconfigmodel.Setup) {
 	config.BindEnvAndSetDefault("apm_config.obfuscation.memcached.enabled", true, "DD_APM_OBFUSCATION_MEMCACHED_ENABLED")
 	config.BindEnvAndSetDefault("apm_config.obfuscation.memcached.keep_command", false, "DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND")
 	config.BindEnvAndSetDefault("apm_config.obfuscation.cache.enabled", true, "DD_APM_OBFUSCATION_CACHE_ENABLED")
+	config.BindEnvAndSetDefault("apm_config.obfuscation.cache.max_size", 5000000, "DD_APM_OBFUSCATION_CACHE_MAX_SIZE")
 	config.SetKnown("apm_config.filter_tags.require")
 	config.SetKnown("apm_config.filter_tags.reject")
 	config.SetKnown("apm_config.filter_tags_regex.require")

pkg/config/setup/config_test.go

@@ -1501,6 +1501,7 @@ func TestAPMObfuscationDefaultValue(t *testing.T) {
 	assert.False(t, conf.GetBool("apm_config.obfuscation.credit_cards.luhn"))
 	assert.Len(t, conf.GetStringSlice("apm_config.obfuscation.credit_cards.keep_values"), 0)
 	assert.True(t, conf.GetBool("apm_config.obfuscation.cache.enabled"))
+	assert.Equal(t, 5000000, conf.GetInt64("apm_config.obfuscation.cache.max_size"))
 }
 
 func TestAgentConfigInit(t *testing.T) {

pkg/flare/envvars.go

@@ -108,6 +108,7 @@ var allowedEnvvarNames = []string{
 	"DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_KEEP_VALUES",
 	"DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_OBFUSCATE_SQL_VALUES",
 	"DD_APM_OBFUSCATION_CACHE_ENABLED",
+	"DD_APM_OBFUSCATION_CACHE_MAX_SIZE",
 	"DD_APM_DEBUG_PORT",
 	"DD_APM_INSTRUMENTATION_ENABLED",
 	"DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES",

pkg/obfuscate/cache.go

@@ -51,8 +51,9 @@ func (c *measuredCache) statsLoop() {
 }
 
 type cacheOptions struct {
-	On     bool
-	Statsd StatsClient
+	On      bool
+	Statsd  StatsClient
+	MaxSize int64
 }
 
 // newMeasuredCache returns a new measuredCache.
@@ -62,19 +63,10 @@ func newMeasuredCache(opts cacheOptions) *measuredCache {
 		return &measuredCache{}
 	}
 	cfg := &ristretto.Config{
-		// We know that the maximum allowed resource length is 5K. This means that
-		// in 5MB we can store a minimum of 1000 queries.
-		MaxCost: 5000000,
-
-		// An appromixated worst-case scenario when the cache is filled with small
-		// queries averaged as being of length 11 ("LOCK TABLES"), we would be able
-		// to fit 476K of them into 5MB of cost.
-		//
-		// We average it to 500K and multiply 10x as the documentation recommends.
-		NumCounters: 500000 * 10,
-
-		BufferItems: 64,   // default recommended value
-		Metrics:     true, // enable hit/miss counters
+		MaxCost:     opts.MaxSize,
+		NumCounters: opts.MaxSize * 10, // Multiplied by 10 as per ristretto recommendation
+		BufferItems: 64,                // default recommended value
+		Metrics:     true,              // enable hit/miss counters
 	}
 	cache, err := ristretto.NewCache(cfg)
 	if err != nil {

pkg/obfuscate/obfuscate.go

@@ -277,6 +277,9 @@ type CreditCardsConfig struct {
 type CacheConfig struct {
 	// Enabled specifies whether caching should be enabled.
 	Enabled bool `mapstructure:"enabled"`
+
+	// MaxSize is the maximum size of the cache in bytes.
+	MaxSize int64 `mapstructure:"max_size"`
 }
 
 // NewObfuscator creates a new obfuscator
@@ -286,7 +289,7 @@ func NewObfuscator(cfg Config) *Obfuscator {
 	}
 	o := Obfuscator{
 		opts:              &cfg,
-		queryCache:        newMeasuredCache(cacheOptions{On: cfg.Cache.Enabled, Statsd: cfg.Statsd}),
+		queryCache:        newMeasuredCache(cacheOptions{On: cfg.Cache.Enabled, Statsd: cfg.Statsd, MaxSize: cfg.Cache.MaxSize}),
 		sqlLiteralEscapes: atomic.NewBool(false),
 		log:               cfg.Logger,
 	}

releasenotes/notes/add-obfuscation-cache-max-size-411301253b92e191.yaml

@@ -0,0 +1,12 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Add ``apm_config.obfuscation.cache.max_size`` to set the maximum size of the
+    cache in bytes.