Add test_execution changes to benchmark implementation

Signed-off-by: Divya Madala <[email protected]>
Divyaasm · May 29, 2024 · 07313e4 · 07313e4
1 parent 41ae02a
commit 07313e4
Show file tree

Hide file tree

Showing 6 changed files with 236 additions and 2 deletions.
diff --git a/benchmark.ini b/benchmark.ini
@@ -0,0 +1,32 @@
+[meta]
+config.version = 17
+
+[system]
+env.name = local
+
+[node]
+root.dir =
+src.root.dir =
+
+[source]
+remote.repo.url = https://github.com/opensearch-project/OpenSearch.git
+opensearch.src.subdir = opensearch
+
+[benchmarks]
+local.dataset.cache =
+
+[results_publishing]
+datastore.type = in-memory
+
+
+[workloads]
+default.url = https://github.com/rishabh6788/opensearch-benchmark-workloads
+
+[provision_configs]
+default.dir = default-provision-config
+
+[defaults]
+preserve_benchmark_candidate = false
+
+[distributions]
+release.cache = true
diff --git a/config.yml b/config.yml
@@ -0,0 +1,13 @@
+Description: Configuration file to store contants required to run rally for performance test
+Constants:
+  Repository: https://github.com/opensearch-project/opensearch-cluster-cdk
+  cidr: '172.31.0.0/16'
+  VpcId: 'vpc-0c413a9df7ca84d06'
+  AccountId: 724293578735
+  dataNodeStorage: 200
+  serverAccessType: prefixList
+  restrictServerAccessTo: pl-60b85b09
+  Region: us-east-1
+  Role: cfn-set-up
+  isInternal: 'false'
+  IamRoleArn: 'arn:aws:iam::724293578735:role/custom-benchmark-ec2-instance-role'
diff --git a/jenkins/opensearch/release-testing.jenkinsfile b/jenkins/opensearch/release-testing.jenkinsfile
@@ -0,0 +1,54 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    options {
+            timeout(time: 1, unit: 'HOURS')
+    }
+    agent {
+        docker {
+            label 'Jenkins-Agent-Windows2019-X64-M54xlarge-Docker-Host'
+            image 'opensearchstaging/ci-runner:ci-runner-windows2019-opensearch-build-v1'
+            registryUrl 'https://public.ecr.aws/'
+            alwaysPull true
+        }
+    }
+    parameters {
+        booleanParam(
+            name: 'TEST',
+            description: 'Allow validation even when the security plugin is absent.',
+            defaultValue: false
+        )
+
+
+    }
+
+    stages {
+        stage('Test Windows') {
+            steps {
+                script {
+                    if (TEST) {
+                        bat "dir"
+                        sh "ls -a"
+                        sh "pwd"
+                        echo "shell script executed"
+                    }
+
+                }
+            }
+        }
+    }
+    post() {
+        always {
+            script {
+                postCleanup()
+            }
+        }
+    }
+}
diff --git a/manifest.yml b/manifest.yml
@@ -0,0 +1,125 @@
+---
+schema-version: '1.1'
+build:
+  name: OpenSearch
+  version: 2.13.0
+  platform: linux
+  architecture: x64
+  distribution: tar
+  location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/dist/opensearch/opensearch-2.13.0-linux-x64.tar.gz
+  id: '9662'
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: 7ec678d1b7c87d6e779fdef94e33623e1f1e2647
+    commit_id: 7ec678d1b7c87d6e779fdef94e33623e1f1e2647
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/dist/opensearch-min-2.13.0-linux-x64.tar.gz
+  - name: common-utils
+    repository: https://github.com/opensearch-project/common-utils.git
+    ref: 937bc4ecdf5ff07a1087b259c330c9b361db5c32
+    commit_id: 937bc4ecdf5ff07a1087b259c330c9b361db5c32
+  - name: job-scheduler
+    repository: https://github.com/opensearch-project/job-scheduler.git
+    ref: d2bb21838c087c6331e53c7f696680931bfe8672
+    commit_id: d2bb21838c087c6331e53c7f696680931bfe8672
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-2.13.0.0.zip
+  - name: security
+    repository: https://github.com/opensearch-project/security.git
+    ref: 8f029ebc9753888d9009547e26cada1a71a81585
+    commit_id: 8f029ebc9753888d9009547e26cada1a71a81585
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-security-2.13.0.0.zip
+  - name: k-NN
+    repository: https://github.com/opensearch-project/k-NN.git
+    ref: 328f501a85591ccc8b228ef9d95966cdb782832d
+    commit_id: 328f501a85591ccc8b228ef9d95966cdb782832d
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-knn-2.13.0.0.zip
+  - name: geospatial
+    repository: https://github.com/opensearch-project/geospatial.git
+    ref: ae663a1f5dc0bec3cd1e1e43907336c5522100ae
+    commit_id: ae663a1f5dc0bec3cd1e1e43907336c5522100ae
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-geospatial-2.13.0.0.zip
+  - name: cross-cluster-replication
+    repository: https://github.com/opensearch-project/cross-cluster-replication.git
+    ref: b64bd49ff3d0fd0f3b6168add339b9000c6e91aa
+    commit_id: b64bd49ff3d0fd0f3b6168add339b9000c6e91aa
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-cross-cluster-replication-2.13.0.0.zip
+  - name: ml-commons
+    repository: https://github.com/opensearch-project/ml-commons.git
+    ref: 493e394628e0c7598f32fc103e8470636115c06e
+    commit_id: 493e394628e0c7598f32fc103e8470636115c06e
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-2.13.0.0.zip
+  - name: neural-search
+    repository: https://github.com/opensearch-project/neural-search.git
+    ref: 52795807658f6e719f4c22147a54e0baca5c43dd
+    commit_id: 52795807658f6e719f4c22147a54e0baca5c43dd
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-neural-search-2.13.0.0.zip
+  - name: notifications-core
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: d7f36a416b5c16000f6e367c6c4a78e2c92e1093
+    commit_id: d7f36a416b5c16000f6e367c6c4a78e2c92e1093
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-notifications-core-2.13.0.0.zip
+  - name: notifications
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: d7f36a416b5c16000f6e367c6c4a78e2c92e1093
+    commit_id: d7f36a416b5c16000f6e367c6c4a78e2c92e1093
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-notifications-2.13.0.0.zip
+  - name: opensearch-observability
+    repository: https://github.com/opensearch-project/observability.git
+    ref: a488b1fee983d416a858a9dbbe10c6773413ee07
+    commit_id: a488b1fee983d416a858a9dbbe10c6773413ee07
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-observability-2.13.0.0.zip
+  - name: opensearch-reports
+    repository: https://github.com/opensearch-project/reporting.git
+    ref: 6383f7a1460925f5aaf5763487f04943c30ce543
+    commit_id: 6383f7a1460925f5aaf5763487f04943c30ce543
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-reports-scheduler-2.13.0.0.zip
+  - name: asynchronous-search
+    repository: https://github.com/opensearch-project/asynchronous-search.git
+    ref: edb7cf2ff74fa4f04ae08a4969a657134fd2c891
+    commit_id: edb7cf2ff74fa4f04ae08a4969a657134fd2c891
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-asynchronous-search-2.13.0.0.zip
+  - name: anomaly-detection
+    repository: https://github.com/opensearch-project/anomaly-detection.git
+    ref: 68b350624db92ce494eb09416d5b3803fe4b9fe9
+    commit_id: 68b350624db92ce494eb09416d5b3803fe4b9fe9
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-anomaly-detection-2.13.0.0.zip
+  - name: alerting
+    repository: https://github.com/opensearch-project/alerting.git
+    ref: c0b737b20f61938ae2a2b1dfea6c3e2bbc16a636
+    commit_id: c0b737b20f61938ae2a2b1dfea6c3e2bbc16a636
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-alerting-2.13.0.0.zip
+  - name: security-analytics
+    repository: https://github.com/opensearch-project/security-analytics.git
+    ref: d3af2426bc2b3697c3a854fb3b24a76ba333640a
+    commit_id: d3af2426bc2b3697c3a854fb3b24a76ba333640a
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-security-analytics-2.13.0.0.zip
+  - name: index-management
+    repository: https://github.com/opensearch-project/index-management.git
+    ref: fa8d1b3cf484a08e7526ceb2b53a97d9c49b82c4
+    commit_id: fa8d1b3cf484a08e7526ceb2b53a97d9c49b82c4
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-index-management-2.13.0.0.zip
+  - name: custom-codecs
+    repository: https://github.com/opensearch-project/custom-codecs.git
+    ref: 5c4e4cb7afd2db8299ac7e66cc80e82a7c670028
+    commit_id: 5c4e4cb7afd2db8299ac7e66cc80e82a7c670028
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-custom-codecs-2.13.0.0.zip
+  - name: flow-framework
+    repository: https://github.com/opensearch-project/flow-framework.git
+    ref: 478bb7eae139064b41f5af15712022a2f5415dc7
+    commit_id: 478bb7eae139064b41f5af15712022a2f5415dc7
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-flow-framework-2.13.0.0.zip
+  - name: sql
+    repository: https://github.com/opensearch-project/sql.git
+    ref: 89647c14f624320a26e30e04af865c9471189008
+    commit_id: 89647c14f624320a26e30e04af865c9471189008
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-sql-2.13.0.0.zip
+  - name: skills
+    repository: https://github.com/opensearch-project/skills.git
+    ref: b61f8ee1233762c5bbc27207a8c66fc8ae622d75
+    commit_id: b61f8ee1233762c5bbc27207a8c66fc8ae622d75
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-skills-2.13.0.0.zip
+  - name: performance-analyzer
+    repository: https://github.com/opensearch-project/performance-analyzer.git
+    ref: 6863a65449a1f05bf2c45c76a3359fd1950b930f
+    commit_id: 6863a65449a1f05bf2c45c76a3359fd1950b930f
+    location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/9662/linux/x64/tar/builds/opensearch/plugins/opensearch-performance-analyzer-2.13.0.0.zip
diff --git a/src/test_workflow/benchmark_test/benchmark_create_cluster.py b/src/test_workflow/benchmark_test/benchmark_create_cluster.py
@@ -60,8 +60,7 @@ def __init__(
                 else:
                     params_list.append(f" -c {key}={value}")
         role_params = (
-            f" --require-approval=never --plugin cdk-assume-role-credential-plugin"
-            f" -c assume-role-credentials:writeIamRoleName={role} -c assume-role-credentials:readIamRoleName={role} "
+            f" --require-approval=never"
         )
         self.params = "".join(params_list) + role_params
         self.password = None if self.args.insecure else get_password(self.args.distribution_version)

diff --git a/test_execution_new-test.csv b/test_execution_new-test.csv
@@ -0,0 +1,11 @@
+task,operation,error_rate,duration,throughput.min,throughput.mean,throughput.median,throughput.max,throughput.unit,latency.50_0,latency.90_0,latency.99_0,latency.100_0,latency.mean,latency.unit,service_time.50_0,service_time.90_0,service_time.99_0,service_time.100_0,service_time.mean,service_time.unit,client_processing_time.50_0,client_processing_time.90_0,client_processing_time.99_0,client_processing_time.100_0,client_processing_time.mean,client_processing_time.unit,processing_time.50_0,processing_time.90_0,processing_time.99_0,processing_time.100_0,processing_time.mean,processing_time.unit
+index,index,0.0,65359.105945999996,30140.01086291147,35435.300316507084,36524.42794457866,37961.211625104035,docs/s,188.5493750000009,336.4812377999998,949.4505449999975,2669.8655009999898,256.80756365312493,ms,188.5493750000009,336.4812377999998,949.4505449999975,2669.8655009999898,256.80756365312493,ms,852.2072715000029,1218.0514750999928,1720.2804882500102,1899.768125999998,882.5101776156253,ms,1066.1703960000004,1610.1907419999995,2112.0126089800065,4060.6666689999997,1142.2319744750005,ms
+wait-until-merges-finish,index-stats,0.0,21.054042000002937,3.0080603576564076,3.0080603576564076,3.0080603576564076,3.0080603576564076,ops/s,,,,332.1505410000043,332.1505410000043,ms,,,,332.1505410000043,332.1505410000043,ms,,,,272.7151249999906,272.7151249999906,ms,,,,605.338875000001,605.338875000001,ms
+percolator_with_content_president_bush,percolator_with_content_president_bush,0.0,17946.470551000006,10.890759105067486,10.95616603473185,10.952750255274228,10.985230263493373,ops/s,10571.873860499993,13373.214291599988,13969.762413250017,14035.814842000007,10573.299484970003,ms,87.46966649999877,95.7589207999945,105.37684350999145,108.47579200000723,88.92379167999792,ms,0.3719995000039944,0.8171333000007058,8.34456417001379,12.533666999985371,0.6723791899997877,ms,88.37668750000205,97.3876166000025,105.90750158002551,109.25712499999918,89.71628794999958,ms
+percolator_with_content_saddam_hussein,percolator_with_content_saddam_hussein,0.0,16942.392342000006,11.260232735879683,11.463276022834238,11.47156491600956,11.625560806854654,ops/s,9944.694214500018,12411.073161500002,12970.568478089997,13033.83067500002,9929.192986350005,ms,81.5434999999951,84.40918779998015,92.90530116000042,94.13133300000709,82.12400997000003,ms,0.5552504999997154,0.8409754000069826,1.3766799100449423,6.971457999981112,0.6569088599994188,ms,82.44089599999427,85.75860389999548,93.68032575001735,94.84691700001235,82.97193626999871,ms
+percolator_with_content_hurricane_katrina,percolator_with_content_hurricane_katrina,0.0,17552.170632000012,10.915158360763959,11.061167569026717,11.069667149490211,11.211693843103783,ops/s,10428.988006000025,13001.733261500016,13584.75658183001,13648.053799000025,10414.61372663001,ms,83.84543750000262,88.89197500001559,101.0422357500059,105.07091700000615,85.54956953999863,ms,0.42960349999532355,0.8211088999871674,1.6172039200131791,9.293457999973498,0.5969954400015354,ms,84.61087499999564,90.2481083999902,101.3981383300015,105.80495799999312,86.3209429500003,ms
+percolator_with_content_google,percolator_with_content_google,0.0,25343.041303000005,7.697661344820763,7.76000988314016,7.767598862713707,7.795713239916954,ops/s,13638.368950962942,17190.69822329627,17992.835527466283,18081.297848629616,13642.734012362951,ms,125.23512500001743,128.69510059998106,136.25468907999706,139.37537499998598,125.9450417900004,ms,0.39039599998602625,0.5562710999868159,0.7102444100061693,0.7879999999715892,0.38375369000078763,ms,125.74197900001138,129.49393359999704,136.6981141699907,139.85266700001375,126.43986211999874,ms
+percolator_no_score_with_content_google,percolator_no_score_with_content_google,0.0,17227.265966999992,11.311930302612351,11.363515478481844,11.361465695477127,11.423406854863023,ops/s,11593.345797999988,14586.565528499983,15240.138256329992,15309.933090999977,11564.977978949992,ms,83.66564550000533,94.7975249999928,101.93001582999813,103.13195800000585,85.63766424000022,ms,0.5029379999967887,1.0907496999976731,6.411661999987087,8.886167000014211,0.7792499999993652,ms,84.752103999989,96.12220030000742,102.85456549999793,104.0532080000105,86.63573714000023,ms
+percolator_with_highlighting,percolator_with_highlighting,0.0,17224.637924000006,11.322046299871953,11.410853561637428,11.431226376444293,11.457602761920407,ops/s,10000.467151999997,12666.685603200003,13257.766263000005,13322.94588299999,10027.846928939995,ms,84.07287549999864,94.0007169000012,100.38816999998687,101.74150000000282,85.52249625000144,ms,0.4231454999938933,0.7044490999817299,1.320013330010287,2.0143330000053083,0.48438710999676005,ms,84.75131250000345,94.48906670001237,101.10354434001295,102.14337500002557,86.15047964000098,ms
+percolator_with_content_ignore_me,percolator_with_content_ignore_me,0.0,1308020.088719,0.08365061498655171,0.08416309224557855,0.08391279878465469,0.08661143770212658,ops/s,5626.463725000008,6556.371192300037,8003.344420700015,8015.668603999984,5717.688110130004,ms,5615.680648500018,6544.561740500024,7995.404442500005,8011.197170000003,5709.8822479699975,ms,0.9411664999561253,1.2224217999801115,4.606247160024858,6.144624000171461,0.9789717399968367,ms,5616.807294500006,6545.892598900048,7996.926097330027,8012.948587000039,5711.22799467,ms
+percolator_no_score_with_content_ignore_me,percolator_no_score_with_content_ignore_me,0.0,18004.422800000157,10.408965619838435,10.771493773901028,10.792625536563865,11.182012518329822,ops/s,4027.82869333339,4652.432186833471,4788.716558503338,4803.51384133337,3775.6394457934516,ms,81.59920850005165,91.12497139999505,363.6412039000848,641.4800410000225,92.16538543001434,ms,0.14027149995854415,0.49894169983417674,0.7527859199308341,1.626750000013999,0.23449292996929216,ms,81.77960400007578,91.57387529994594,364.3411466698754,641.9887920001202,92.46721293001428,ms