From 6d6f30e2fb23dd1b3e333dcacfd5773ff7e61b4e Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Thu, 14 Nov 2024 20:11:38 +0530 Subject: [PATCH 01/11] webhook implementation Signed-off-by: msvinaykumar --- design/BulkAPI.md | 96 ++++++++++++++----- .../minikube/kruize-crc-minikube.yaml | 3 +- .../openshift/kruize-crc-openshift.yaml | 1 + .../aks/kruize-crc-aks.yaml | 3 +- .../minikube/kruize-crc-minikube.yaml | 3 +- .../openshift/kruize-crc-openshift.yaml | 1 + .../serviceObjects/BulkJobStatus.java | 35 ++++++- .../analyzer/services/BulkService.java | 9 ++ .../analyzer/services/DummyWebhook.java | 65 +++++++++++++ .../analyzer/workerimpl/BulkJobManager.java | 69 ++++++++----- .../operator/KruizeDeploymentInfo.java | 1 + .../com/autotune/utils/KruizeConstants.java | 16 ++++ .../com/autotune/utils/ServerContext.java | 3 + 13 files changed, 256 insertions(+), 49 deletions(-) create mode 100644 src/main/java/com/autotune/analyzer/services/DummyWebhook.java diff --git a/design/BulkAPI.md b/design/BulkAPI.md index 0962c9d7d..4048034d1 100644 --- a/design/BulkAPI.md +++ b/design/BulkAPI.md @@ -119,7 +119,10 @@ GET /bulk?job_id=123e4567-e89b-12d3-a456-426614174000 "processed_experiments": 23, "job_id": "54905959-77d4-42ba-8e06-90bb97b823b9", "job_start_time": "2024-10-10T06:07:09.066Z", - "job_end_time": "2024-10-10T06:07:17.471Z" + "job_end_time": "2024-10-10T06:07:17.471Z", + "webhook": { + "status": "COMPLETED" + } } ``` @@ -189,7 +192,7 @@ example 1: } ``` -example 2: +example 2: Entire Job failed ```json { @@ -205,11 +208,35 @@ example 2: }, "job_id": "270fa4d9-2701-4ca0-b056-74229cc28498", "job_start_time": "2024-11-12T15:05:46.362Z", - "job_end_time": "2024-11-12T15:06:05.301Z" + "job_end_time": "2024-11-12T15:06:05.301Z", + "webhook": { + "status": "COMPLETED" + } } ``` +example 3: Only Webhook failed + +```json +{ + "status": "COMPLETED", + "total_experiments": 23, + "processed_experiments": 23, + "job_id": "54905959-77d4-42ba-8e06-90bb97b823b9", + "job_start_time": "2024-10-10T06:07:09.066Z", + "job_end_time": "2024-10-10T06:07:17.471Z", + "webhook": { + "status": "FAILED", + "notifications": { + "type": "ERROR", + "message": "HttpHostConnectException: Unable to connect to the webhook. Please try again later.", + "code": 503 + } + } +} +``` + ### Response Parameters ## API Description: Experiment and Recommendation Processing Status @@ -275,6 +302,20 @@ resource optimization in Kubernetes environments. Below is a breakdown of the JS - **Type**: `String (ISO 8601 format) or null` - **Description**: End timestamp of the job. If the job is still in progress, this will be `null`. +- **webhook**: + - **Type**: `Object` + - **Description**: An object that provides details about the webhook status and any errors encountered during the + webhook invocation. + + - The `webhook` parameter allows the system to notify an external service or consumer about the completion status of + an experiment-processing job. When a job is completed, this webhook will be triggered to send an HTTP request to a + predefined URL, configured either via an environment variable (`webhookURL`) or within a `kruize` configuration + file. + This notification mechanism is essential for systems that require real-time updates about the job's processing + status, enabling consumers to take immediate follow-up actions. For example, an external analytics dashboard, a + monitoring service, or a message queue like Kafka can listen for these webhook calls to further process or log the + job completion data. + **Note: Experiment Name:** - **Naming Pattern:** Experiment names are currently formed using the following pattern: @@ -300,27 +341,27 @@ resource optimization in Kubernetes environments. Below is a breakdown of the JS apiVersion: v1 kind: ConfigMap metadata: -name: kruizeconfig -namespace: openshift-tuning + name: kruizeconfig + namespace: openshift-tuning data: -kruizeconfigjson: | - { - "datasource": [ - { - "name": "prometheus-1", - "provider": "prometheus", - "serviceName": "prometheus-k8s", - "namespace": "openshift-monitoring", - "url": "", - "authentication": { - "type": "bearer", - "credentials": { - "tokenFilePath": "/var/run/secrets/kubernetes.io/serviceaccount/token" + kruizeconfigjson: | + { + "datasource": [ + { + "name": "prometheus-1", + "provider": "prometheus", + "serviceName": "prometheus-k8s", + "namespace": "openshift-monitoring", + "url": "", + "authentication": { + "type": "bearer", + "credentials": { + "tokenFilePath": "/var/run/secrets/kubernetes.io/serviceaccount/token" + } } } - } - ] - } + ] + } ``` ## Limits @@ -395,4 +436,15 @@ number of labels, or none at all. Here are some examples: - "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%" -> Default - "%label:org_id%|%label:source_id%|%label:cluster_id%|%namespace%|%workloadtype%|%workloadname%|%containername%" - "%label:org_id%|%namespace%|%workloadtype%|%workloadname%|%containername%" -- "%label:org_id%|%label:cluster_id%|%namespace%|%workloadtype%|%workloadname%" \ No newline at end of file +- "%label:org_id%|%label:cluster_id%|%namespace%|%workloadtype%|%workloadname%" + +## Webhook URL Configuration + +The webhook URL can be configured using an environment variable `webhookURL` or within the kruize configuration JSON +file: + +```json +{ + "webhookURL": "http://127.0.0.1:8080/webhook" +} +``` \ No newline at end of file diff --git a/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml b/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml index 2c5bc3d33..ab19d8300 100644 --- a/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml +++ b/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml @@ -36,7 +36,8 @@ data: "local": "true", "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", - "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", + "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", + "webhookURL":" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", "hibernate": { "dialect": "org.hibernate.dialect.PostgreSQLDialect", "driver": "org.postgresql.Driver", diff --git a/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml index f404b8a56..24dfed0d4 100644 --- a/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml +++ b/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml @@ -50,6 +50,7 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/createExperiment", + "webhookURL":" : "http://kruize.openshift-tuning.svc.cluster.local:8080/webhook", "hibernate": { "dialect": "org.hibernate.dialect.PostgreSQLDialect", "driver": "org.postgresql.Driver", diff --git a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml index d37708da1..caa30bd70 100644 --- a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml +++ b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml @@ -100,7 +100,8 @@ data: "local": "true", "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", - "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", + "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", + "webhookURL":" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { diff --git a/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml b/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml index 0fa7048fb..c0d255f26 100644 --- a/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml +++ b/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml @@ -114,7 +114,8 @@ data: "local": "true", "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", - "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", + "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", + "webhookURL":" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { diff --git a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml index 94b0d2c42..4a80b5171 100644 --- a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml +++ b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml @@ -109,6 +109,7 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/createExperiment", + "webhookURL":" : "http://kruize.openshift-tuning.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java b/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java index 16da934bb..0e760d885 100644 --- a/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java +++ b/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java @@ -48,6 +48,7 @@ public class BulkJobStatus { private String endTime; // Change to String to store formatted time private Map notifications; private Map experiments = Collections.synchronizedMap(new HashMap<>()); + private Webhook webhook; public BulkJobStatus(String jobID, String status, Instant startTime) { this.jobID = jobID; @@ -112,6 +113,14 @@ public void setExperiments(Map experiments) { this.experiments = experiments; } + public Webhook getWebhook() { + return webhook; + } + + public void setWebhook(Webhook webhook) { + this.webhook = webhook; + } + // Method to add a new experiment with "unprocessed" status and null notification public synchronized Experiment addExperiment(String experimentName) { Experiment experiment = new Experiment(experimentName); @@ -255,5 +264,29 @@ public void setCode(int code) { } } + public static class Webhook { + private KruizeConstants.KRUIZE_BULK_API.NotificationConstants.WebHookStatus status; + private Notification notifications; // Notifications can hold multiple entries + + public Webhook(KruizeConstants.KRUIZE_BULK_API.NotificationConstants.WebHookStatus status) { + this.status = status; + } + + public KruizeConstants.KRUIZE_BULK_API.NotificationConstants.WebHookStatus getStatus() { + return status; + } + + public void setStatus(KruizeConstants.KRUIZE_BULK_API.NotificationConstants.WebHookStatus status) { + this.status = status; + } + + public Notification getNotifications() { + return notifications; + } -} + public void setNotifications(Notification notifications) { + this.notifications = notifications; + } + } + + } diff --git a/src/main/java/com/autotune/analyzer/services/BulkService.java b/src/main/java/com/autotune/analyzer/services/BulkService.java index 6813251f5..4f507f51a 100644 --- a/src/main/java/com/autotune/analyzer/services/BulkService.java +++ b/src/main/java/com/autotune/analyzer/services/BulkService.java @@ -72,6 +72,15 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se boolean verbose = verboseParam != null && Boolean.parseBoolean(verboseParam); BulkJobStatus jobDetails; LOGGER.info("Job ID: " + jobID); + if (jobStatusMap.isEmpty()) { + sendErrorResponse( + resp, + null, + HttpServletResponse.SC_NOT_FOUND, + JOB_NOT_FOUND_MSG + ); + return; + } jobDetails = jobStatusMap.get(jobID); LOGGER.info("Job Status: " + jobDetails.getStatus()); resp.setContentType(JSON_CONTENT_TYPE); diff --git a/src/main/java/com/autotune/analyzer/services/DummyWebhook.java b/src/main/java/com/autotune/analyzer/services/DummyWebhook.java new file mode 100644 index 000000000..aff1c6546 --- /dev/null +++ b/src/main/java/com/autotune/analyzer/services/DummyWebhook.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (c) 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.autotune.analyzer.services; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; + +import static com.autotune.utils.KruizeConstants.KRUIZE_BULK_API.JOB_ID; + +@WebServlet(name = "DummyWebhookServlet", urlPatterns = "/webhook") +public class DummyWebhook extends HttpServlet { + private static final Logger LOGGER = LoggerFactory.getLogger(DummyWebhook.class); + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + // Set response type to JSON + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + + // Read the JSON payload from the request + StringBuilder jsonPayload = new StringBuilder(); + try (BufferedReader reader = request.getReader()) { + String line; + while ((line = reader.readLine()) != null) { + jsonPayload.append(line); + } + } + + // Log the received payload (for debugging purposes) + LOGGER.debug("Received Webhook Payload: " + jsonPayload.toString()); + + // Just sending a simple success response back + // Return the jobID to the user + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status", "success"); + jsonObject.put("message", "Webhook received successfully"); + response.getWriter().write(jsonObject.toString()); + + + } + +} diff --git a/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java b/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java index aa0c39005..d1fe844e7 100644 --- a/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java +++ b/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java @@ -118,23 +118,22 @@ private static Map parseLabelString(String labelString) { @Override public void run() { + DataSourceMetadataInfo metadataInfo = null; + DataSourceManager dataSourceManager = new DataSourceManager(); + DataSourceInfo datasource = null; try { String labelString = getLabels(this.bulkInput.getFilter()); if (null == this.bulkInput.getDatasource()) { this.bulkInput.setDatasource(CREATE_EXPERIMENT_CONFIG_BEAN.getDatasourceName()); } - DataSourceMetadataInfo metadataInfo = null; - DataSourceManager dataSourceManager = new DataSourceManager(); - DataSourceInfo datasource = null; try { datasource = CommonUtils.getDataSourceInfo(this.bulkInput.getDatasource()); } catch (Exception e) { LOGGER.error(e.getMessage()); e.printStackTrace(); - jobData.setStatus(FAILED); BulkJobStatus.Notification notification = DATASOURCE_NOT_REG_INFO; notification.setMessage(String.format(notification.getMessage(), e.getMessage())); - jobData.setNotification(String.valueOf(HttpURLConnection.HTTP_BAD_REQUEST), notification); + setFinalJobStatus(FAILED,String.valueOf(HttpURLConnection.HTTP_BAD_REQUEST),notification,datasource); } if (null != datasource) { JSONObject daterange = processDateRange(this.bulkInput.getTime_range()); @@ -144,16 +143,13 @@ public void run() { metadataInfo = dataSourceManager.importMetadataFromDataSource(datasource, labelString, 0, 0, 0); } if (null == metadataInfo) { - jobData.setStatus(COMPLETED); - jobData.setEndTime(Instant.now()); - jobData.setNotification(String.valueOf(HttpURLConnection.HTTP_OK), NOTHING_INFO); + setFinalJobStatus(COMPLETED,String.valueOf(HttpURLConnection.HTTP_OK),NOTHING_INFO,datasource); } else { Map createExperimentAPIObjectMap = getExperimentMap(labelString, jobData, metadataInfo, datasource); //Todo Store this map in buffer and use it if BulkAPI pods restarts and support experiment_type jobData.setTotal_experiments(createExperimentAPIObjectMap.size()); jobData.setProcessed_experiments(0); if (jobData.getTotal_experiments() > KruizeDeploymentInfo.BULK_API_LIMIT) { - jobData.setStatus(FAILED); - jobData.setNotification(String.valueOf(HttpURLConnection.HTTP_BAD_REQUEST), LIMIT_INFO); + setFinalJobStatus(FAILED,String.valueOf(HttpURLConnection.HTTP_BAD_REQUEST),LIMIT_INFO,datasource); } else { ExecutorService createExecutor = Executors.newFixedThreadPool(bulk_thread_pool_size); ExecutorService generateExecutor = Executors.newFixedThreadPool(bulk_thread_pool_size); @@ -185,8 +181,7 @@ public void run() { experiment.setNotification(new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR, e.getMessage(), HttpURLConnection.HTTP_BAD_REQUEST)); } finally { if (jobData.getTotal_experiments() == jobData.getProcessed_experiments()) { - jobData.setStatus(COMPLETED); - jobData.setEndTime(Instant.now()); + setFinalJobStatus(COMPLETED,null,null,finalDatasource); } } @@ -202,7 +197,6 @@ public void run() { recommendationResponseCode = recommendationApiClient.callKruizeAPI(null); LOGGER.debug("API Response code: {}", recommendationResponseCode); if (recommendationResponseCode.getStatusCode() == HttpURLConnection.HTTP_CREATED) { - jobData.setProcessed_experiments(jobData.getProcessed_experiments() + 1); experiment.getRecommendations().setStatus(NotificationConstants.Status.PROCESSED); } else { experiment.getRecommendations().setStatus(NotificationConstants.Status.FAILED); @@ -213,9 +207,9 @@ public void run() { experiment.getRecommendations().setStatus(NotificationConstants.Status.FAILED); experiment.getRecommendations().setNotifications(new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR, e.getMessage(), HttpURLConnection.HTTP_INTERNAL_ERROR)); } finally { + jobData.setProcessed_experiments(jobData.getProcessed_experiments() + 1); if (jobData.getTotal_experiments() == jobData.getProcessed_experiments()) { - jobData.setStatus(COMPLETED); - jobData.setEndTime(Instant.now()); + setFinalJobStatus(COMPLETED,null,null,finalDatasource); } } }); @@ -223,8 +217,10 @@ public void run() { } catch (Exception e) { e.printStackTrace(); experiment.setNotification(new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR, e.getMessage(), HttpURLConnection.HTTP_INTERNAL_ERROR)); - } finally { - + jobData.setProcessed_experiments(jobData.getProcessed_experiments() + 1); + if (jobData.getTotal_experiments() == jobData.getProcessed_experiments()) { + setFinalJobStatus(COMPLETED,null,null,finalDatasource); + } } }); } @@ -233,8 +229,6 @@ public void run() { } } catch (IOException e) { LOGGER.error(e.getMessage()); - jobData.setStatus(FAILED); - jobData.setEndTime(Instant.now()); BulkJobStatus.Notification notification; if (e instanceof SocketTimeoutException) { notification = DATASOURCE_GATEWAY_TIMEOUT_INFO; @@ -244,13 +238,42 @@ public void run() { notification = DATASOURCE_DOWN_INFO; } notification.setMessage(String.format(notification.getMessage(), e.getMessage())); - jobData.setNotification(String.valueOf(HttpURLConnection.HTTP_UNAVAILABLE), notification); + setFinalJobStatus(FAILED,String.valueOf(HttpURLConnection.HTTP_UNAVAILABLE),notification,datasource); } catch (Exception e) { LOGGER.error(e.getMessage()); e.printStackTrace(); - jobData.setStatus(FAILED); - jobData.setEndTime(Instant.now()); - jobData.setNotification(String.valueOf(HttpURLConnection.HTTP_INTERNAL_ERROR), new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR, e.getMessage(), HttpURLConnection.HTTP_INTERNAL_ERROR)); + setFinalJobStatus(FAILED,String.valueOf(HttpURLConnection.HTTP_INTERNAL_ERROR),new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR, e.getMessage(), HttpURLConnection.HTTP_INTERNAL_ERROR),datasource); + } + } + + public void setFinalJobStatus(String status,String notificationKey,BulkJobStatus.Notification notification,DataSourceInfo finalDatasource) { + jobData.setStatus(status); + jobData.setEndTime(Instant.now()); + if(null!=notification) + jobData.setNotification(notificationKey,notification); + GenericRestApiClient apiClient = new GenericRestApiClient(finalDatasource); + apiClient.setBaseURL(KruizeDeploymentInfo.webhook_url); + GenericRestApiClient.HttpResponseWrapper responseCode; + BulkJobStatus.Webhook webhook = new BulkJobStatus.Webhook(WebHookStatus.IN_PROGRESS); + jobData.setWebhook(webhook); + try { + responseCode = apiClient.callKruizeAPI("[" + new Gson().toJson(jobData) + "]"); + LOGGER.debug("API Response code: {}", responseCode); + if (responseCode.getStatusCode() == HttpURLConnection.HTTP_OK) { + webhook.setStatus(WebHookStatus.COMPLETED); + jobData.setWebhook(webhook); + } else { + BulkJobStatus.Notification webHookNotification = new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR,responseCode.getResponseBody().toString(),responseCode.getStatusCode()); + webhook.setNotifications(webHookNotification); + webhook.setStatus(WebHookStatus.FAILED); + jobData.setWebhook(webhook); + } + } catch (Exception e) { + e.printStackTrace(); + BulkJobStatus.Notification webHookNotification = new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR,e.toString(),HttpURLConnection.HTTP_INTERNAL_ERROR); + webhook.setNotifications(webHookNotification); + webhook.setStatus(WebHookStatus.FAILED); + jobData.setWebhook(webhook); } } diff --git a/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java b/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java index bec435227..e63260b86 100644 --- a/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java +++ b/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java @@ -81,6 +81,7 @@ public class KruizeDeploymentInfo { public static Boolean log_http_req_resp = false; public static String recommendations_url; public static String experiments_url; + public static String webhook_url; public static int BULK_API_LIMIT = 1000; public static int BULK_API_MAX_BATCH_SIZE = 100; public static Integer bulk_thread_pool_size = 3; diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index f863b38d4..0a0c5c374 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -697,6 +697,7 @@ public static final class KRUIZE_CONFIG_ENV_NAME { public static final String LOG_HTTP_REQ_RESP = "logAllHttpReqAndResp"; public static final String RECOMMENDATIONS_URL = "recommendationsURL"; public static final String EXPERIMENTS_URL = "experimentsURL"; + public static final String WEBHOOK_URL = "webhookURL"; public static final String BULK_API_LIMIT = "bulkapilimit"; public static final String BULK_API_CHUNK_SIZE = "bulkapichunksize"; public static final String BULK_THREAD_POOL_SIZE = "bulkThreadPoolSize"; @@ -882,6 +883,21 @@ public String getStatus() { } } + public enum WebHookStatus { + INITIATED, // The Webhook has initiated a request + IN_PROGRESS, // The request to the Webhook is actively being processed + QUEUED, // The request to the Webhook has been queued, waiting for resources + SENT, // The request has been sent to the Webhook, but no response yet + RECEIVED, // The Webhook has received a response, but further processing continues + SUCCESS, // The request to the Webhook was successful + FAILED, // The call to the Webhook failed due to an error + RETRYING, // The Webhook is retrying the call due to a transient error + TIMED_OUT, // The request to the Webhook exceeded the allowed response time + ERROR_LOGGED, // The error has been logged for debugging or monitoring + COMPLETED, // The entire process, including subsequent processing, is finished + CANCELLED // The request was cancelled, potentially by user action or system condition + } + } } diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index eac7f6079..fdc6fd14e 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -78,4 +78,7 @@ public class ServerContext { //Bulk Service public static final String BULK_SERVICE = ROOT_CONTEXT + "bulk"; + + //Web Hook + public static final String WEBHOOK_SERVICE = ROOT_CONTEXT + "webhook"; } From d547e1b841d025cf074bef6da48f2e884c23f27a Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Thu, 14 Nov 2024 20:15:01 +0530 Subject: [PATCH 02/11] webhook implementation Signed-off-by: msvinaykumar --- design/BulkAPI.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/BulkAPI.md b/design/BulkAPI.md index 4048034d1..2ffc97b22 100644 --- a/design/BulkAPI.md +++ b/design/BulkAPI.md @@ -192,7 +192,7 @@ example 1: } ``` -example 2: Entire Job failed +example 2: Job failed ```json { From 9f036d75d3d01566dd148c9878c5a737dbe0d32a Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Fri, 15 Nov 2024 11:00:48 +0530 Subject: [PATCH 03/11] fixed malformed yamls Signed-off-by: msvinaykumar --- .../default-db-included-installation/aks/kruize-crc-aks.yaml | 2 +- .../minikube/kruize-crc-minikube.yaml | 2 +- .../openshift/kruize-crc-openshift.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml index caa30bd70..50abc1a40 100644 --- a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml +++ b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml @@ -101,7 +101,7 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", - "webhookURL":" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", + "webhookURL" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { diff --git a/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml b/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml index c0d255f26..a53eaee92 100644 --- a/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml +++ b/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml @@ -115,7 +115,7 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", - "webhookURL":" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", + "webhookURL": "http://kruize.monitoring.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { diff --git a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml index 4a80b5171..7c4eafe50 100644 --- a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml +++ b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml @@ -109,7 +109,7 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/createExperiment", - "webhookURL":" : "http://kruize.openshift-tuning.svc.cluster.local:8080/webhook", + "webhookURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { From efffed288379556e45b492b6aeece0fc777e84e2 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Tue, 19 Nov 2024 16:36:53 +0530 Subject: [PATCH 04/11] incorporated review comments Signed-off-by: msvinaykumar --- .../analyzer/serviceObjects/BulkInput.java | 19 +++++++++++++++++++ .../serviceObjects/BulkJobStatus.java | 8 ++++++++ .../analyzer/workerimpl/BulkJobManager.java | 6 +++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/BulkInput.java b/src/main/java/com/autotune/analyzer/serviceObjects/BulkInput.java index e5e31d40d..dd5934945 100644 --- a/src/main/java/com/autotune/analyzer/serviceObjects/BulkInput.java +++ b/src/main/java/com/autotune/analyzer/serviceObjects/BulkInput.java @@ -25,6 +25,7 @@ public class BulkInput { private FilterWrapper filter; private TimeRange time_range; private String datasource; + private Webhook webhook; // Getters and Setters @@ -136,4 +137,22 @@ public void setEnd(String end) { this.end = end; } } + + public static class Webhook{ + private String url; + public String getUrl() { + return url; + } + public void setUrl(String url) { + this.url = url; + } + } + + public Webhook getWebhook() { + return webhook; + } + + public void setWebhook(Webhook webhook) { + this.webhook = webhook; + } } diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java b/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java index 0e760d885..7dae51bb6 100644 --- a/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java +++ b/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java @@ -197,6 +197,14 @@ public Recommendation getRecommendations() { public void setNotification(Notification notification) { this.notification = notification; } + + public Notification getNotification() { + return notification; + } + + public void setRecommendations(Recommendation recommendations) { + this.recommendations = recommendations; + } } public static class Recommendation { diff --git a/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java b/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java index d1fe844e7..8c3620b4c 100644 --- a/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java +++ b/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java @@ -252,7 +252,11 @@ public void setFinalJobStatus(String status,String notificationKey,BulkJobStatus if(null!=notification) jobData.setNotification(notificationKey,notification); GenericRestApiClient apiClient = new GenericRestApiClient(finalDatasource); - apiClient.setBaseURL(KruizeDeploymentInfo.webhook_url); + if(null != bulkInput.getWebhook() && null != bulkInput.getWebhook().getUrl()){ + apiClient.setBaseURL(bulkInput.getWebhook().getUrl()); + }else { + apiClient.setBaseURL(KruizeDeploymentInfo.webhook_url); + } GenericRestApiClient.HttpResponseWrapper responseCode; BulkJobStatus.Webhook webhook = new BulkJobStatus.Webhook(WebHookStatus.IN_PROGRESS); jobData.setWebhook(webhook); From 1f03587c6dd3fe1f36f289c3b1c0ba0de3251620 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Tue, 19 Nov 2024 17:07:50 +0530 Subject: [PATCH 05/11] incorporated review comments Signed-off-by: msvinaykumar --- design/BulkAPI.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/design/BulkAPI.md b/design/BulkAPI.md index 2ffc97b22..d4a118ed4 100644 --- a/design/BulkAPI.md +++ b/design/BulkAPI.md @@ -51,7 +51,10 @@ progress of the job. "experiment_types": [ "container", "namespace" - ] + ], + "webhooks": { + "url" : "http://127.0.0.1:8080/webhook" + } } ``` @@ -75,6 +78,11 @@ progress of the job. - **experiment_types:** Specifies the type(s) of experiments to run, e.g., `"container"` or `"namespace"`. +- **webhook:** The `webhook` parameter allows the system to notify an external service or consumer about the completion status of + an experiment-processing job. When a job is completed, this webhook will be triggered to send an HTTP request to a + predefined URL, configured either via an environment variable (`webhookURL`) or within a `kruize` configuration + file or via input payload. + ### Success Response - **Status:** 200 OK From c61752f6b8879e104b79e3082e735b16187479a2 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Wed, 20 Nov 2024 11:15:14 +0530 Subject: [PATCH 06/11] incorporated reveiw comments Signed-off-by: msvinaykumar --- .../aks/kruize-crc-aks.yaml | 1 - .../minikube/kruize-crc-minikube.yaml | 1 - .../openshift/kruize-crc-openshift.yaml | 41 ++++++------ .../analyzer/services/DummyWebhook.java | 65 ------------------- .../analyzer/workerimpl/BulkJobManager.java | 40 ++++++------ .../operator/KruizeDeploymentInfo.java | 1 - .../com/autotune/utils/KruizeConstants.java | 1 - 7 files changed, 39 insertions(+), 111 deletions(-) delete mode 100644 src/main/java/com/autotune/analyzer/services/DummyWebhook.java diff --git a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml index 50abc1a40..f94cf4c12 100644 --- a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml +++ b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml @@ -101,7 +101,6 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", - "webhookURL" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { diff --git a/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml b/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml index a53eaee92..b4a3c470f 100644 --- a/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml +++ b/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml @@ -115,7 +115,6 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", - "webhookURL": "http://kruize.monitoring.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { diff --git a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml index 7c4eafe50..8d9268591 100644 --- a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml +++ b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml @@ -109,7 +109,6 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/createExperiment", - "webhookURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/webhook", "experimentNameFormat" : "%datasource%|%clustername%|%namespace%|%workloadname%(%workloadtype%)|%containername%", "bulkapilimit" : 1000, "hibernate": { @@ -141,10 +140,10 @@ data: "namespace": "openshift-monitoring", "url": "", "authentication": { - "type": "bearer", - "credentials": { - "tokenFilePath": "/var/run/secrets/kubernetes.io/serviceaccount/token" - } + "type": "bearer", + "credentials": { + "tokenFilePath": "/var/run/secrets/kubernetes.io/serviceaccount/token" + } } } ] @@ -181,13 +180,13 @@ spec: value: kruizeDB - name: PGDATA value: /var/lib/pg_data - resources: - requests: - memory: "2Gi" - cpu: "2" - limits: - memory: "2Gi" - cpu: "2" +# resources: +# requests: +# memory: "2Gi" +# cpu: "2" +# limits: +# memory: "2Gi" +# cpu: "2" ports: - containerPort: 5432 volumeMounts: @@ -247,14 +246,14 @@ spec: done containers: - name: kruize - image: quay.io/kruize/autotune_operator:0.1.1 + image: quay.io/vinakuma/autotune_operator:shk imagePullPolicy: Always volumeMounts: - name: config-volume mountPath: /etc/config env: - name: LOGGING_LEVEL - value: "info" + value: "debug" - name: ROOT_LOGGING_LEVEL value: "error" - name: DB_CONFIG_FILE @@ -263,13 +262,13 @@ spec: value: "/etc/config/kruizeconfigjson" - name: JAVA_TOOL_OPTIONS value: "-XX:MaxRAMPercentage=80" - resources: - requests: - memory: "2Gi" - cpu: "2" - limits: - memory: "2Gi" - cpu: "2" +# resources: +# requests: +# memory: "2Gi" +# cpu: "2" +# limits: +# memory: "2Gi" +# cpu: "2" ports: - name: kruize-port containerPort: 8080 diff --git a/src/main/java/com/autotune/analyzer/services/DummyWebhook.java b/src/main/java/com/autotune/analyzer/services/DummyWebhook.java deleted file mode 100644 index aff1c6546..000000000 --- a/src/main/java/com/autotune/analyzer/services/DummyWebhook.java +++ /dev/null @@ -1,65 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2022 Red Hat, IBM Corporation and others. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -package com.autotune.analyzer.services; - -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.servlet.ServletException; -import javax.servlet.annotation.WebServlet; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.PrintWriter; - -import static com.autotune.utils.KruizeConstants.KRUIZE_BULK_API.JOB_ID; - -@WebServlet(name = "DummyWebhookServlet", urlPatterns = "/webhook") -public class DummyWebhook extends HttpServlet { - private static final Logger LOGGER = LoggerFactory.getLogger(DummyWebhook.class); - - @Override - protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - // Set response type to JSON - response.setContentType("application/json"); - response.setCharacterEncoding("UTF-8"); - - // Read the JSON payload from the request - StringBuilder jsonPayload = new StringBuilder(); - try (BufferedReader reader = request.getReader()) { - String line; - while ((line = reader.readLine()) != null) { - jsonPayload.append(line); - } - } - - // Log the received payload (for debugging purposes) - LOGGER.debug("Received Webhook Payload: " + jsonPayload.toString()); - - // Just sending a simple success response back - // Return the jobID to the user - JSONObject jsonObject = new JSONObject(); - jsonObject.put("status", "success"); - jsonObject.put("message", "Webhook received successfully"); - response.getWriter().write(jsonObject.toString()); - - - } - -} diff --git a/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java b/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java index 8c3620b4c..b50f66f51 100644 --- a/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java +++ b/src/main/java/com/autotune/analyzer/workerimpl/BulkJobManager.java @@ -252,32 +252,30 @@ public void setFinalJobStatus(String status,String notificationKey,BulkJobStatus if(null!=notification) jobData.setNotification(notificationKey,notification); GenericRestApiClient apiClient = new GenericRestApiClient(finalDatasource); - if(null != bulkInput.getWebhook() && null != bulkInput.getWebhook().getUrl()){ + if(null != bulkInput.getWebhook() && null != bulkInput.getWebhook().getUrl()) { apiClient.setBaseURL(bulkInput.getWebhook().getUrl()); - }else { - apiClient.setBaseURL(KruizeDeploymentInfo.webhook_url); - } - GenericRestApiClient.HttpResponseWrapper responseCode; - BulkJobStatus.Webhook webhook = new BulkJobStatus.Webhook(WebHookStatus.IN_PROGRESS); - jobData.setWebhook(webhook); - try { - responseCode = apiClient.callKruizeAPI("[" + new Gson().toJson(jobData) + "]"); - LOGGER.debug("API Response code: {}", responseCode); - if (responseCode.getStatusCode() == HttpURLConnection.HTTP_OK) { - webhook.setStatus(WebHookStatus.COMPLETED); - jobData.setWebhook(webhook); - } else { - BulkJobStatus.Notification webHookNotification = new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR,responseCode.getResponseBody().toString(),responseCode.getStatusCode()); + GenericRestApiClient.HttpResponseWrapper responseCode; + BulkJobStatus.Webhook webhook = new BulkJobStatus.Webhook(WebHookStatus.IN_PROGRESS); + jobData.setWebhook(webhook); + try { + responseCode = apiClient.callKruizeAPI("[" + new Gson().toJson(jobData) + "]"); + LOGGER.debug("API Response code: {}", responseCode); + if (responseCode.getStatusCode() == HttpURLConnection.HTTP_OK) { + webhook.setStatus(WebHookStatus.COMPLETED); + jobData.setWebhook(webhook); + } else { + BulkJobStatus.Notification webHookNotification = new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR, responseCode.getResponseBody().toString(), responseCode.getStatusCode()); + webhook.setNotifications(webHookNotification); + webhook.setStatus(WebHookStatus.FAILED); + jobData.setWebhook(webhook); + } + } catch (Exception e) { + e.printStackTrace(); + BulkJobStatus.Notification webHookNotification = new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR, e.toString(), HttpURLConnection.HTTP_INTERNAL_ERROR); webhook.setNotifications(webHookNotification); webhook.setStatus(WebHookStatus.FAILED); jobData.setWebhook(webhook); } - } catch (Exception e) { - e.printStackTrace(); - BulkJobStatus.Notification webHookNotification = new BulkJobStatus.Notification(BulkJobStatus.NotificationType.ERROR,e.toString(),HttpURLConnection.HTTP_INTERNAL_ERROR); - webhook.setNotifications(webHookNotification); - webhook.setStatus(WebHookStatus.FAILED); - jobData.setWebhook(webhook); } } diff --git a/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java b/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java index e63260b86..bec435227 100644 --- a/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java +++ b/src/main/java/com/autotune/operator/KruizeDeploymentInfo.java @@ -81,7 +81,6 @@ public class KruizeDeploymentInfo { public static Boolean log_http_req_resp = false; public static String recommendations_url; public static String experiments_url; - public static String webhook_url; public static int BULK_API_LIMIT = 1000; public static int BULK_API_MAX_BATCH_SIZE = 100; public static Integer bulk_thread_pool_size = 3; diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 0a0c5c374..51e64148c 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -697,7 +697,6 @@ public static final class KRUIZE_CONFIG_ENV_NAME { public static final String LOG_HTTP_REQ_RESP = "logAllHttpReqAndResp"; public static final String RECOMMENDATIONS_URL = "recommendationsURL"; public static final String EXPERIMENTS_URL = "experimentsURL"; - public static final String WEBHOOK_URL = "webhookURL"; public static final String BULK_API_LIMIT = "bulkapilimit"; public static final String BULK_API_CHUNK_SIZE = "bulkapichunksize"; public static final String BULK_THREAD_POOL_SIZE = "bulkThreadPoolSize"; From f342ab91b5b07cba124a77fa1e57a26c937092e8 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Wed, 20 Nov 2024 11:22:15 +0530 Subject: [PATCH 07/11] incorporated reveiw comments Signed-off-by: msvinaykumar --- .../minikube/kruize-crc-minikube.yaml | 1 - .../openshift/kruize-crc-openshift.yaml | 1 - .../openshift/kruize-crc-openshift.yaml | 30 +++++++++---------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml b/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml index ab19d8300..d1a9e9261 100644 --- a/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml +++ b/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml @@ -37,7 +37,6 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment", - "webhookURL":" : "http://kruize.monitoring.svc.cluster.local:8080/webhook", "hibernate": { "dialect": "org.hibernate.dialect.PostgreSQLDialect", "driver": "org.postgresql.Driver", diff --git a/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml index 24dfed0d4..f404b8a56 100644 --- a/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml +++ b/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml @@ -50,7 +50,6 @@ data: "logAllHttpReqAndResp": "true", "recommendationsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/generateRecommendations?experiment_name=%s", "experimentsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/createExperiment", - "webhookURL":" : "http://kruize.openshift-tuning.svc.cluster.local:8080/webhook", "hibernate": { "dialect": "org.hibernate.dialect.PostgreSQLDialect", "driver": "org.postgresql.Driver", diff --git a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml index 8d9268591..28ba2f296 100644 --- a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml +++ b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml @@ -180,13 +180,13 @@ spec: value: kruizeDB - name: PGDATA value: /var/lib/pg_data -# resources: -# requests: -# memory: "2Gi" -# cpu: "2" -# limits: -# memory: "2Gi" -# cpu: "2" + resources: + requests: + memory: "2Gi" + cpu: "2" + limits: + memory: "2Gi" + cpu: "2" ports: - containerPort: 5432 volumeMounts: @@ -246,7 +246,7 @@ spec: done containers: - name: kruize - image: quay.io/vinakuma/autotune_operator:shk + image: quay.io/kruize/autotune_operator:0.1.1 imagePullPolicy: Always volumeMounts: - name: config-volume @@ -262,13 +262,13 @@ spec: value: "/etc/config/kruizeconfigjson" - name: JAVA_TOOL_OPTIONS value: "-XX:MaxRAMPercentage=80" -# resources: -# requests: -# memory: "2Gi" -# cpu: "2" -# limits: -# memory: "2Gi" -# cpu: "2" + resources: + requests: + memory: "2Gi" + cpu: "2" + limits: + memory: "2Gi" + cpu: "2" ports: - name: kruize-port containerPort: 8080 From 8ab4bb3956772422185e1cf09090f437c0e66a40 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Wed, 20 Nov 2024 11:26:53 +0530 Subject: [PATCH 08/11] incorporated reveiw comments Signed-off-by: msvinaykumar --- src/main/java/com/autotune/utils/ServerContext.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index fdc6fd14e..eac7f6079 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -78,7 +78,4 @@ public class ServerContext { //Bulk Service public static final String BULK_SERVICE = ROOT_CONTEXT + "bulk"; - - //Web Hook - public static final String WEBHOOK_SERVICE = ROOT_CONTEXT + "webhook"; } From bf25e2b3425fce52e7721f289b1226bdf0c09a1f Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Wed, 20 Nov 2024 11:28:51 +0530 Subject: [PATCH 09/11] incorporated reveiw comments Signed-off-by: msvinaykumar --- .../openshift/kruize-crc-openshift.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml index 28ba2f296..754283328 100644 --- a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml +++ b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml @@ -253,7 +253,7 @@ spec: mountPath: /etc/config env: - name: LOGGING_LEVEL - value: "debug" + value: "info" - name: ROOT_LOGGING_LEVEL value: "error" - name: DB_CONFIG_FILE From 6d8aa019f9a4a26bcdc3bff5463fe2aba8532c10 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Thu, 21 Nov 2024 16:16:48 +0530 Subject: [PATCH 10/11] incorporated reveiw comments Signed-off-by: msvinaykumar --- design/BulkAPI.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/design/BulkAPI.md b/design/BulkAPI.md index d4a118ed4..06e90e1f4 100644 --- a/design/BulkAPI.md +++ b/design/BulkAPI.md @@ -52,7 +52,7 @@ progress of the job. "container", "namespace" ], - "webhooks": { + "webhook": { "url" : "http://127.0.0.1:8080/webhook" } } @@ -446,13 +446,3 @@ number of labels, or none at all. Here are some examples: - "%label:org_id%|%namespace%|%workloadtype%|%workloadname%|%containername%" - "%label:org_id%|%label:cluster_id%|%namespace%|%workloadtype%|%workloadname%" -## Webhook URL Configuration - -The webhook URL can be configured using an environment variable `webhookURL` or within the kruize configuration JSON -file: - -```json -{ - "webhookURL": "http://127.0.0.1:8080/webhook" -} -``` \ No newline at end of file From 642ef0d61730cd6f56ecf8c702f0c4428536b626 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Thu, 21 Nov 2024 19:45:17 +0530 Subject: [PATCH 11/11] incorporated reveiw comments Signed-off-by: msvinaykumar --- design/BulkAPI.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/design/BulkAPI.md b/design/BulkAPI.md index 06e90e1f4..2d73b8d0c 100644 --- a/design/BulkAPI.md +++ b/design/BulkAPI.md @@ -79,9 +79,7 @@ progress of the job. - **experiment_types:** Specifies the type(s) of experiments to run, e.g., `"container"` or `"namespace"`. - **webhook:** The `webhook` parameter allows the system to notify an external service or consumer about the completion status of - an experiment-processing job. When a job is completed, this webhook will be triggered to send an HTTP request to a - predefined URL, configured either via an environment variable (`webhookURL`) or within a `kruize` configuration - file or via input payload. + an experiment-processing job. Once a job is completed, this webhook will be triggered to send an HTTP request to the URL defined in the bulk request payload. ### Success Response @@ -316,9 +314,7 @@ resource optimization in Kubernetes environments. Below is a breakdown of the JS webhook invocation. - The `webhook` parameter allows the system to notify an external service or consumer about the completion status of - an experiment-processing job. When a job is completed, this webhook will be triggered to send an HTTP request to a - predefined URL, configured either via an environment variable (`webhookURL`) or within a `kruize` configuration - file. + an experiment-processing job. Once a job is completed, this webhook will be triggered to send an HTTP request to the URL defined in the bulk request payload. This notification mechanism is essential for systems that require real-time updates about the job's processing status, enabling consumers to take immediate follow-up actions. For example, an external analytics dashboard, a monitoring service, or a message queue like Kafka can listen for these webhook calls to further process or log the