From d652e16b27b513c5e1e085ea23a3180fd700315a Mon Sep 17 00:00:00 2001 From: keisku Date: Tue, 24 Dec 2024 13:46:49 +0000 Subject: [PATCH 1/4] new obfuscation config to set the maximum size of the cache in bytes Signed-off-by: keisku --- comp/trace/config/config_test.go | 16 ++++++++++++++ comp/trace/config/setup.go | 1 + comp/trace/config/testdata/full.yaml | 1 + pkg/collector/python/test_datadog_agent.go | 1 + pkg/config/config_template.yaml | 4 ++++ pkg/config/setup/apm.go | 1 + pkg/config/setup/config_test.go | 1 + pkg/flare/envvars.go | 1 + pkg/obfuscate/cache.go | 22 ++++++------------- pkg/obfuscate/obfuscate.go | 5 ++++- ...ation-cache-max-size-411301253b92e191.yaml | 12 ++++++++++ 11 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 releasenotes/notes/add-obfuscation-cache-max-size-411301253b92e191.yaml diff --git a/comp/trace/config/config_test.go b/comp/trace/config/config_test.go index 9706e2c784f4c..fa6ae5c31759b 100644 --- a/comp/trace/config/config_test.go +++ b/comp/trace/config/config_test.go @@ -1776,6 +1776,22 @@ func TestLoadEnv(t *testing.T) { assert.False(t, cfg.Obfuscation.Cache.Enabled) }) + env = "DD_APM_OBFUSCATION_CACHE_MAX_SIZE" + t.Run(env, func(t *testing.T) { + t.Setenv(env, "1234567") + + c := buildConfigComponent(t, true, fx.Replace(corecomp.MockParams{ + Params: corecomp.Params{ConfFilePath: "./testdata/full.yaml"}, + })) + cfg := c.Object() + + assert.NotNil(t, cfg) + actualConfig := pkgconfigsetup.Datadog().GetString("apm_config.obfuscation.cache.max_size") + actualParsed := cfg.Obfuscation.Cache.MaxSize + assert.Equal(t, "1234567", actualConfig) + assert.Equal(t, int64(1234567), actualParsed) + }) + env = "DD_APM_PROFILING_ADDITIONAL_ENDPOINTS" t.Run(env, func(t *testing.T) { t.Setenv(env, `{"url1": ["key1", "key2"], "url2": ["key3"]}`) diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index 0eba6a37e8953..5f3ee056256a9 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -436,6 +436,7 @@ func applyDatadogConfig(c *config.AgentConfig, core corecompcfg.Component) error c.Obfuscation.CreditCards.Luhn = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.credit_cards.luhn") c.Obfuscation.CreditCards.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.credit_cards.keep_values") c.Obfuscation.Cache.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.cache.enabled") + c.Obfuscation.Cache.MaxSize = pkgconfigsetup.Datadog().GetInt64("apm_config.obfuscation.cache.max_size") if core.IsSet("apm_config.filter_tags.require") { tags := core.GetStringSlice("apm_config.filter_tags.require") diff --git a/comp/trace/config/testdata/full.yaml b/comp/trace/config/testdata/full.yaml index 0c6535550e615..cd166c07cbd3c 100644 --- a/comp/trace/config/testdata/full.yaml +++ b/comp/trace/config/testdata/full.yaml @@ -91,3 +91,4 @@ apm_config: luhn: true cache: enabled: true + max_size: 5555555 diff --git a/pkg/collector/python/test_datadog_agent.go b/pkg/collector/python/test_datadog_agent.go index 19bf1e24f04d4..09c750a3ab061 100644 --- a/pkg/collector/python/test_datadog_agent.go +++ b/pkg/collector/python/test_datadog_agent.go @@ -166,6 +166,7 @@ func testObfuscaterConfig(t *testing.T) { }, Cache: obfuscate.CacheConfig{ Enabled: true, + MaxSize: 5000000, }, } assert.Equal(t, expected, obfuscaterConfig) diff --git a/pkg/config/config_template.yaml b/pkg/config/config_template.yaml index 95efa801eb565..333cdcc0f4d75 100644 --- a/pkg/config/config_template.yaml +++ b/pkg/config/config_template.yaml @@ -1265,6 +1265,10 @@ api_key: ## Enables caching obfuscated statements. Currently supported for SQL and MongoDB queries. ## Enabled by default. # enabled: true + ## @param DD_APM_OBFUSCATION_CACHE_MAX_SIZE - integer - optional - default: 5000000 + ## The maximum size of the cache in bytes. The maximum allowed resource length is 5000. + ## We can store a minimum of 1000 queries (= 5000000 / 5000) by default. + # max_size: 5000000 ## @param filter_tags - object - optional ## @env DD_APM_FILTER_TAGS_REQUIRE - object - optional diff --git a/pkg/config/setup/apm.go b/pkg/config/setup/apm.go index 8472267c255ec..c127b3d52a8e0 100644 --- a/pkg/config/setup/apm.go +++ b/pkg/config/setup/apm.go @@ -44,6 +44,7 @@ func setupAPM(config pkgconfigmodel.Setup) { config.BindEnvAndSetDefault("apm_config.obfuscation.memcached.enabled", true, "DD_APM_OBFUSCATION_MEMCACHED_ENABLED") config.BindEnvAndSetDefault("apm_config.obfuscation.memcached.keep_command", false, "DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND") config.BindEnvAndSetDefault("apm_config.obfuscation.cache.enabled", true, "DD_APM_OBFUSCATION_CACHE_ENABLED") + config.BindEnvAndSetDefault("apm_config.obfuscation.cache.max_size", 5000000, "DD_APM_OBFUSCATION_CACHE_MAX_SIZE") config.SetKnown("apm_config.filter_tags.require") config.SetKnown("apm_config.filter_tags.reject") config.SetKnown("apm_config.filter_tags_regex.require") diff --git a/pkg/config/setup/config_test.go b/pkg/config/setup/config_test.go index a9c7a787689c4..480a009abf4aa 100644 --- a/pkg/config/setup/config_test.go +++ b/pkg/config/setup/config_test.go @@ -1501,6 +1501,7 @@ func TestAPMObfuscationDefaultValue(t *testing.T) { assert.False(t, conf.GetBool("apm_config.obfuscation.credit_cards.luhn")) assert.Len(t, conf.GetStringSlice("apm_config.obfuscation.credit_cards.keep_values"), 0) assert.True(t, conf.GetBool("apm_config.obfuscation.cache.enabled")) + assert.Equal(t, 5000000, conf.GetInt64("apm_config.obfuscation.cache.max_size")) } func TestAgentConfigInit(t *testing.T) { diff --git a/pkg/flare/envvars.go b/pkg/flare/envvars.go index 40b98a3b74d2d..43e4376c465a3 100644 --- a/pkg/flare/envvars.go +++ b/pkg/flare/envvars.go @@ -108,6 +108,7 @@ var allowedEnvvarNames = []string{ "DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_KEEP_VALUES", "DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_OBFUSCATE_SQL_VALUES", "DD_APM_OBFUSCATION_CACHE_ENABLED", + "DD_APM_OBFUSCATION_CACHE_MAX_SIZE", "DD_APM_DEBUG_PORT", "DD_APM_INSTRUMENTATION_ENABLED", "DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES", diff --git a/pkg/obfuscate/cache.go b/pkg/obfuscate/cache.go index 837b2b15b1914..70a4d44f60eb8 100644 --- a/pkg/obfuscate/cache.go +++ b/pkg/obfuscate/cache.go @@ -51,8 +51,9 @@ func (c *measuredCache) statsLoop() { } type cacheOptions struct { - On bool - Statsd StatsClient + On bool + Statsd StatsClient + MaxSize int64 } // newMeasuredCache returns a new measuredCache. @@ -62,19 +63,10 @@ func newMeasuredCache(opts cacheOptions) *measuredCache { return &measuredCache{} } cfg := &ristretto.Config{ - // We know that the maximum allowed resource length is 5K. This means that - // in 5MB we can store a minimum of 1000 queries. - MaxCost: 5000000, - - // An appromixated worst-case scenario when the cache is filled with small - // queries averaged as being of length 11 ("LOCK TABLES"), we would be able - // to fit 476K of them into 5MB of cost. - // - // We average it to 500K and multiply 10x as the documentation recommends. - NumCounters: 500000 * 10, - - BufferItems: 64, // default recommended value - Metrics: true, // enable hit/miss counters + MaxCost: opts.MaxSize, + NumCounters: opts.MaxSize * 10, // Multiplied by 10 as per ristretto recommendation + BufferItems: 64, // default recommended value + Metrics: true, // enable hit/miss counters } cache, err := ristretto.NewCache(cfg) if err != nil { diff --git a/pkg/obfuscate/obfuscate.go b/pkg/obfuscate/obfuscate.go index 84a0f38b2ec78..52ddb384e13bb 100644 --- a/pkg/obfuscate/obfuscate.go +++ b/pkg/obfuscate/obfuscate.go @@ -277,6 +277,9 @@ type CreditCardsConfig struct { type CacheConfig struct { // Enabled specifies whether caching should be enabled. Enabled bool `mapstructure:"enabled"` + + // MaxSize is the maximum size of the cache in bytes. + MaxSize int64 `mapstructure:"max_size"` } // NewObfuscator creates a new obfuscator @@ -286,7 +289,7 @@ func NewObfuscator(cfg Config) *Obfuscator { } o := Obfuscator{ opts: &cfg, - queryCache: newMeasuredCache(cacheOptions{On: cfg.Cache.Enabled, Statsd: cfg.Statsd}), + queryCache: newMeasuredCache(cacheOptions{On: cfg.Cache.Enabled, Statsd: cfg.Statsd, MaxSize: cfg.Cache.MaxSize}), sqlLiteralEscapes: atomic.NewBool(false), log: cfg.Logger, } diff --git a/releasenotes/notes/add-obfuscation-cache-max-size-411301253b92e191.yaml b/releasenotes/notes/add-obfuscation-cache-max-size-411301253b92e191.yaml new file mode 100644 index 0000000000000..01d13db1ac15d --- /dev/null +++ b/releasenotes/notes/add-obfuscation-cache-max-size-411301253b92e191.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Add ``apm_config.obfuscation.cache.max_size`` to set the maximum size of the + cache in bytes. From b14e08a5d9255dd6dd3e8344a652f02f6e9069bf Mon Sep 17 00:00:00 2001 From: keisku Date: Wed, 25 Dec 2024 00:42:01 +0000 Subject: [PATCH 2/4] fix type mismatch in unit test --- pkg/config/setup/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/setup/config_test.go b/pkg/config/setup/config_test.go index 480a009abf4aa..8eccca662eb9c 100644 --- a/pkg/config/setup/config_test.go +++ b/pkg/config/setup/config_test.go @@ -1501,7 +1501,7 @@ func TestAPMObfuscationDefaultValue(t *testing.T) { assert.False(t, conf.GetBool("apm_config.obfuscation.credit_cards.luhn")) assert.Len(t, conf.GetStringSlice("apm_config.obfuscation.credit_cards.keep_values"), 0) assert.True(t, conf.GetBool("apm_config.obfuscation.cache.enabled")) - assert.Equal(t, 5000000, conf.GetInt64("apm_config.obfuscation.cache.max_size")) + assert.Equal(t, int64(5000000), conf.GetInt64("apm_config.obfuscation.cache.max_size")) } func TestAgentConfigInit(t *testing.T) { From 531810ea6ddab6e4c8ec5e88b202001040f29e33 Mon Sep 17 00:00:00 2001 From: keisku Date: Wed, 25 Dec 2024 00:43:15 +0000 Subject: [PATCH 3/4] update doc --- pkg/config/config_template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config_template.yaml b/pkg/config/config_template.yaml index 333cdcc0f4d75..4c66aaca48384 100644 --- a/pkg/config/config_template.yaml +++ b/pkg/config/config_template.yaml @@ -1267,7 +1267,7 @@ api_key: # enabled: true ## @param DD_APM_OBFUSCATION_CACHE_MAX_SIZE - integer - optional - default: 5000000 ## The maximum size of the cache in bytes. The maximum allowed resource length is 5000. - ## We can store a minimum of 1000 queries (= 5000000 / 5000) by default. + ## Datadog stores a minimum of 1000 queries (5000000 / 5000) by default. # max_size: 5000000 ## @param filter_tags - object - optional From 523a21d48f32b3e6da050f22859be337071dd175 Mon Sep 17 00:00:00 2001 From: keisku Date: Wed, 25 Dec 2024 00:46:43 +0000 Subject: [PATCH 4/4] add test case for parsing yaml config --- comp/trace/config/config_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/comp/trace/config/config_test.go b/comp/trace/config/config_test.go index fa6ae5c31759b..41678904d2d67 100644 --- a/comp/trace/config/config_test.go +++ b/comp/trace/config/config_test.go @@ -629,6 +629,7 @@ func TestFullYamlConfig(t *testing.T) { assert.True(t, o.CreditCards.Enabled) assert.True(t, o.CreditCards.Luhn) assert.True(t, o.Cache.Enabled) + assert.Equal(t, int64(5555555), o.Cache.MaxSize) assert.True(t, cfg.InstallSignature.Found) assert.Equal(t, traceconfig.InstallSignatureConfig{