Add collector time period to rca.conf (#401)

* Add collector time period to rca.conf

* Resolve format changes

* Resolve format changes

* Split config per rca and ability to add predicate for rcaConf

* Address PR comments

* Add test for rca validation

pa_config/rca.conf

@@ -58,8 +58,12 @@
       "io-total-throughput-in-bytes" : 250000.0,
       "io-total-syscallrate-per-second" : 0.1
     },
-    "cache-config": {
-      "field-data-cache-size-threshold" : 0.8,
+    // field data cache rca
+    "field-data-cache-rca": {
+      "field-data-cache-size-threshold" : 0.8
+    },
+    // shard request cache rca
+    "shard-request-cache-rca": {
       "shard-request-cache-threshold" : 0.9
     }
   },

pa_config/rca_idle_master.conf

@@ -69,8 +69,12 @@
       "io-total-throughput-cluster-percentage" : 0.3,
       "io-total-syscallrate-cluster-percentage" : 0.3
     },
-    "cache-config": {
-      "field-data-cache-size-threshold" : 0.8,
+    // field data cache rca
+    "field-data-cache-rca": {
+      "field-data-cache-size-threshold" : 0.8
+    },
+    // shard request cache rca
+    "shard-request-cache-rca": {
       "shard-request-cache-threshold" : 0.9
     }
   },

pa_config/rca_master.conf

@@ -69,8 +69,12 @@
       "io-total-throughput-cluster-percentage" : 0.3,
       "io-total-syscallrate-cluster-percentage" : 0.3
     },
-    "cache-config": {
-      "field-data-cache-size-threshold" : 0.8,
+    // field data cache rca
+    "field-data-cache-rca": {
+      "field-data-cache-size-threshold" : 0.8
+    },
+    // shard request cache rca
+    "shard-request-cache-rca": {
       "shard-request-cache-threshold" : 0.9
     }
   },

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/FieldDataCacheRcaConfig.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
+
+/**
+ * config object to store rca config settings for FieldDataCacheRca
+ */
+public class FieldDataCacheRcaConfig {
+    public static final String CONFIG_NAME = "field-data-cache-rca-config";
+
+    private Double fieldDataCacheSizeThreshold;
+    private Integer fieldDataCollectorTimePeriodInSec;
+
+    // Field data cache size threshold is 80%
+    public static final double DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD = 0.8;
+    // Metrics like eviction, hits are collected every 300 sec in field data cache rca
+    public static final int DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC = 300;
+
+    public FieldDataCacheRcaConfig(final RcaConf rcaConf) {
+        fieldDataCacheSizeThreshold =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.FIELD_DATA_CACHE_SIZE_THRESHOLD,
+                        DEFAULT_FIELD_DATA_CACHE_SIZE_THRESHOLD,
+                        (s) -> (s > 0),
+                        Double.class);
+        fieldDataCollectorTimePeriodInSec =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        DEFAULT_FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        (s) -> (s > 0),
+                        Integer.class);
+    }
+
+    public double getFieldDataCacheSizeThreshold() {
+        return fieldDataCacheSizeThreshold;
+    }
+
+    public int getFieldDataCollectorTimePeriodInSec() {
+        return fieldDataCollectorTimePeriodInSec;
+    }
+
+    public static class RCA_CONF_KEY_CONSTANTS {
+        public static final String FIELD_DATA_CACHE_SIZE_THRESHOLD = "field-data-cache-size-threshold";
+        public static final String FIELD_DATA_COLLECTOR_TIME_PERIOD_IN_SEC = "field-data-collector-time-period-in-sec";
+    }
+}

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageOldGenRcaConfig.java

@@ -30,10 +30,7 @@ public class HighHeapUsageOldGenRcaConfig {
   public static final String CONFIG_NAME = "high-heap-usage-old-gen-rca";
 
   public HighHeapUsageOldGenRcaConfig(final RcaConf rcaConf) {
-    topK = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.TOP_K, Integer.class);
-    if (topK == null) {
-      topK = DEFAULT_TOP_K;
-    }
+    topK = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.TOP_K, DEFAULT_TOP_K, (s) -> (s > 0), Integer.class);
   }
 
   public int getTopK() {

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HighHeapUsageYoungGenRcaConfig.java

@@ -30,14 +30,11 @@ public class HighHeapUsageYoungGenRcaConfig {
   public static final int DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC = 400;
 
   public HighHeapUsageYoungGenRcaConfig(final RcaConf rcaConf) {
-    promotionRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, Integer.class);
-    youngGenGcTimeThreshold = rcaConf.readRcaConfig(CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.YOUNG_GEN_GC_TIME_THRES, Integer.class);
-    if (promotionRateThreshold == null) {
-      promotionRateThreshold = DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC;
-    }
-    if (youngGenGcTimeThreshold == null) {
-      youngGenGcTimeThreshold = DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC;
-    }
+    promotionRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
+            RCA_CONF_KEY_CONSTANTS.PROMOTION_RATE_THRES, DEFAULT_PROMOTION_RATE_THRESHOLD_IN_MB_PER_SEC, (s) -> (s > 0), Integer.class);
+    youngGenGcTimeThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
+            RCA_CONF_KEY_CONSTANTS.YOUNG_GEN_GC_TIME_THRES, DEFAULT_YOUNG_GEN_GC_TIME_THRESHOLD_IN_MS_PER_SEC,
+            (s) -> (s > 0), Integer.class);
   }
 
   public int getPromotionRateThreshold() {

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotNodeClusterRcaConfig.java

@@ -32,15 +32,9 @@ public class HotNodeClusterRcaConfig {
 
   public HotNodeClusterRcaConfig(final RcaConf rcaConf) {
     unbalancedResourceThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-        RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, Double.class);
+        RCA_CONF_KEY_CONSTANTS.UNBALANCED_RESOURCE_THRES, DEFAULT_UNBALANCED_RESOURCE_THRES, (s) -> (s > 0), Double.class);
     resourceUsageLowerBoundThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-        RCA_CONF_KEY_CONSTANTS.RESOURCE_USAGE_LOWER_BOUND_THRES, Double.class);
-    if (unbalancedResourceThreshold == null) {
-      unbalancedResourceThreshold = DEFAULT_UNBALANCED_RESOURCE_THRES;
-    }
-    if (resourceUsageLowerBoundThreshold == null) {
-      resourceUsageLowerBoundThreshold = DEFAULT_RESOURCE_USAGE_LOWER_BOUND_THRES;
-    }
+        RCA_CONF_KEY_CONSTANTS.RESOURCE_USAGE_LOWER_BOUND_THRES, DEFAULT_RESOURCE_USAGE_LOWER_BOUND_THRES, (s) -> (s > 0), Double.class);
   }
 
   public double getUnbalancedResourceThreshold() {

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardClusterRcaConfig.java

@@ -15,20 +15,14 @@ public class HotShardClusterRcaConfig {
 
     public HotShardClusterRcaConfig(final RcaConf rcaConf) {
         cpuUtilizationClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_CLUSTER_THRESHOLD, Double.class);
+                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_CLUSTER_THRESHOLD,
+                DEFAULT_CPU_UTILIZATION_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class);
         ioTotThroughputClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_THROUGHPUT_CLUSTER_THRESHOLD, Double.class);
+                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_THROUGHPUT_CLUSTER_THRESHOLD,
+                DEFAULT_IO_TOTAL_THROUGHPUT_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class);
         ioTotSysCallRateClusterThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_SYSCALLRATE_CLUSTER_THRESHOLD, Double.class);
-        if (cpuUtilizationClusterThreshold == null) {
-            cpuUtilizationClusterThreshold = DEFAULT_CPU_UTILIZATION_CLUSTER_THRESHOLD;
-        }
-        if (ioTotThroughputClusterThreshold == null) {
-            ioTotThroughputClusterThreshold = DEFAULT_IO_TOTAL_THROUGHPUT_CLUSTER_THRESHOLD;
-        }
-        if (ioTotSysCallRateClusterThreshold == null) {
-            ioTotSysCallRateClusterThreshold = DEFAULT_IO_TOTAL_SYSCALL_RATE_CLUSTER_THRESHOLD;
-        }
+                HotShardClusterRcaConfig.RCA_CONF_KEY_CONSTANTS.CLUSTER_IO_SYSCALLRATE_CLUSTER_THRESHOLD,
+                DEFAULT_IO_TOTAL_SYSCALL_RATE_CLUSTER_THRESHOLD, (s) -> (s > 0), Double.class);
     }
 
     public double getCpuUtilizationClusterThreshold() {

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/HotShardRcaConfig.java

@@ -15,20 +15,14 @@ public class HotShardRcaConfig {
 
     public HotShardRcaConfig(final RcaConf rcaConf) {
         cpuUtilizationThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_THRESHOLD, Double.class);
+                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.CPU_UTILIZATION_THRESHOLD,
+                DEFAULT_CPU_UTILIZATION_THRESHOLD, (s) -> (s > 0), Double.class);
         ioTotThroughputThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_THROUGHPUT_THRESHOLD_IN_BYTES, Double.class);
+                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_THROUGHPUT_THRESHOLD_IN_BYTES,
+                DEFAULT_IO_TOTAL_THROUGHPUT_THRESHOLD_IN_BYTE_PER_SEC, (s) -> (s > 0), Double.class);
         ioTotSysCallRateThreshold = rcaConf.readRcaConfig(CONFIG_NAME,
-                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_SYSCALL_RATE_THRESHOLD_PER_SECOND, Double.class);
-        if (cpuUtilizationThreshold == null) {
-            cpuUtilizationThreshold = DEFAULT_CPU_UTILIZATION_THRESHOLD;
-        }
-        if (ioTotThroughputThreshold == null) {
-            ioTotThroughputThreshold = DEFAULT_IO_TOTAL_THROUGHPUT_THRESHOLD_IN_BYTE_PER_SEC;
-        }
-        if (ioTotSysCallRateThreshold == null) {
-            ioTotSysCallRateThreshold = DEFAULT_IO_TOTAL_SYSCALL_RATE_THRESHOLD_PER_SEC;
-        }
+                HotShardRcaConfig.RCA_CONF_KEY_CONSTANTS.IO_TOT_SYSCALL_RATE_THRESHOLD_PER_SECOND,
+                DEFAULT_IO_TOTAL_SYSCALL_RATE_THRESHOLD_PER_SEC, (s) -> (s > 0), Double.class);
     }
 
     public double getCpuUtilizationThreshold() {

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java

@@ -24,10 +24,8 @@ public class QueueRejectionRcaConfig {
 
   public QueueRejectionRcaConfig(final RcaConf rcaConf) {
     rejectionTimePeriodInSeconds = rcaConf.readRcaConfig(
-        CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.REJECTION_TIME_PERIOD_IN_SECONDS, Integer.class);
-    if (rejectionTimePeriodInSeconds == null) {
-      rejectionTimePeriodInSeconds = DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS;
-    }
+        CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.REJECTION_TIME_PERIOD_IN_SECONDS,
+        DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS, (s) -> (s > 0), Integer.class);
   }
 
   public int getRejectionTimePeriodInSeconds() {

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/ShardRequestCacheRcaConfig.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs;
+
+import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf;
+
+/**
+ * config object to store rca config settings for ShardRequestCacheRca
+ */
+public class ShardRequestCacheRcaConfig {
+    public static final String CONFIG_NAME = "shard-request-cache-rca-config";
+
+    private Double shardRequestCacheSizeThreshold;
+    private Integer shardRequestCollectorTimePeriodInSec;
+
+    // Shard request cache size threshold is 90%
+    public static final double DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD = 0.9;
+    // Metrics like eviction, hits are collected every 300 sec in shard request cache rca
+    public static final int DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC = 300;
+
+    public ShardRequestCacheRcaConfig(final RcaConf rcaConf) {
+        shardRequestCacheSizeThreshold =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_CACHE_SIZE_THRESHOLD,
+                        DEFAULT_SHARD_REQUEST_CACHE_SIZE_THRESHOLD,
+                        (s) -> (s > 0),
+                        Double.class);
+        shardRequestCollectorTimePeriodInSec =
+                rcaConf.readRcaConfig(
+                        CONFIG_NAME,
+                        RCA_CONF_KEY_CONSTANTS.SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        DEFAULT_SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC,
+                        (s) -> (s > 0),
+                        Integer.class);
+    }
+
+    public double getShardRequestCacheSizeThreshold() {
+        return shardRequestCacheSizeThreshold;
+    }
+
+    public int getShardRequestCollectorTimePeriodInSec() {
+        return shardRequestCollectorTimePeriodInSec;
+    }
+
+    public static class RCA_CONF_KEY_CONSTANTS {
+        public static final String SHARD_REQUEST_CACHE_SIZE_THRESHOLD = "shard-request-cache-threshold";
+        public static final String SHARD_REQUEST_COLLECTOR_TIME_PERIOD_IN_SEC = "shard-request-collector-time-period-in-sec";
+    }
+}