diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java index 96ca29bb89..903851a9b0 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java @@ -86,6 +86,24 @@ public interface JsonApiMetricsConfig { @WithDefault("index.usage.count") String indexUsageCounterMetrics(); + @NotBlank + @WithDefault("command.processor.latency") + String commandProcessorLatencyMetrics(); + + /** + * The minimum value that the meter is expected to observe for the command processor latency. The + * unit is milliseconds. + */ + @WithDefault("50") + int MinExpectedCommandProcessorLatency(); + + /** + * The maximum value that the meter is expected to observe for the command processor latency. The + * unit is milliseconds. + */ + @WithDefault("15000") + int MaxExpectedCommandProcessorLatency(); + /** List of values that can be used as value for metrics sort_type. */ enum SortType { // Uses vertor search sorting for document resolution diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index fa173dc444..307b9199da 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -106,9 +107,17 @@ public Uni processCom .onItem() .invoke( result -> { - Tags tags = getCustomTags(commandContext, command, result); - // add metrics - sample.stop(meterRegistry.timer(jsonApiMetricsConfig.metricsName(), tags)); + Tags complexTags = getCustomTags(commandContext, command, result); + Tags simpleTags = getSimpleTags(command); + // add command metrics with complex tags + long durationNs = + sample.stop(meterRegistry.timer(jsonApiMetricsConfig.metricsName(), complexTags)); + // add command metrics with simple tags for histogram (reassigned the timer to ensure + // two metrics are identical) + Timer.builder(jsonApiMetricsConfig.commandProcessorLatencyMetrics()) + .tags(simpleTags) + .register(meterRegistry) + .record(durationNs, TimeUnit.NANOSECONDS); if (isCommandLevelLoggingEnabled(result, false)) { logger.info(buildCommandLog(commandContext, command, result)); @@ -258,6 +267,13 @@ private Tags getCustomTags( return tags; } + private Tags getSimpleTags(Command command) { + Tag commandTag = Tag.of(jsonApiMetricsConfig.command(), command.getClass().getSimpleName()); + String tenant = dataApiRequestInfo.getTenantId().orElse(UNKNOWN_VALUE); + Tag tenantTag = Tag.of(tenantConfig.tenantTag(), tenant); + return Tags.of(commandTag, tenantTag); + } + private JsonApiMetricsConfig.SortType getVectorTypeTag(Command command) { int filterCount = 0; if (command instanceof Filterable fc && fc.filterClause() != null) { @@ -304,6 +320,22 @@ public DistributionStatisticConfig configure( .build() .merge(config); } + + // reduce the number of buckets by setting the min and max expected values to avoid the high + // cardinality problem in Grafana + if (id.getName().startsWith(jsonApiMetricsConfig.commandProcessorLatencyMetrics())) { + return DistributionStatisticConfig.builder() + .percentiles(0.5, 0.90, 0.95, 0.99) + .percentilesHistogram(true) + .minimumExpectedValue( + TimeUnit.MILLISECONDS.toNanos( + jsonApiMetricsConfig.MinExpectedCommandProcessorLatency())) // 0.05 seconds + .maximumExpectedValue( + TimeUnit.MILLISECONDS.toNanos( + jsonApiMetricsConfig.MaxExpectedCommandProcessorLatency())) // 15 seconds + .build() + .merge(config); + } return config; } }; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java index fd57ada062..5748d4e1d9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessorTest.java @@ -88,6 +88,43 @@ public void metrics() throws Exception { assertThat(line).contains("module=\"sgv2-jsonapi\""); }); }); + + List commandLatencyMetrics = + metrics + .lines() + .filter( + line -> + line.startsWith("command_processor_latency") + && line.contains("CountDocumentsCommand") + && !line.startsWith("command_processor_latency_seconds_bucket") + && !line.contains("quantile")) + .toList(); + assertThat(commandLatencyMetrics) + .satisfies( + lines -> { + assertThat(lines.size()).isEqualTo(3); + lines.forEach( + line -> { + assertThat(line).contains("command=\"CountDocumentsCommand\""); + assertThat(line).contains("tenant=\"test-tenant\""); + assertThat(line).contains("module=\"sgv2-jsonapi\""); + assertThat(line).doesNotContain("sort_type"); + assertThat(line).doesNotContain("error"); + assertThat(line).doesNotContain("error_code"); + assertThat(line).doesNotContain("error_class"); + assertThat(line).doesNotContain("vector_enabled"); + }); + }); + List commandLatencyHistogram = + metrics + .lines() + .filter(line -> line.startsWith("command_processor_latency_seconds_bucket")) + .toList(); + assertThat(commandLatencyHistogram) + .satisfies( + lines -> { + assertThat(lines.size()).isNotZero(); + }); } @Test