Improve integration tests to fix intermittent test failures

modules/integration/tests-common/integration-test-utils/src/main/java/org/wso2/am/integration/test/utils/ServerPortsUtils.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved.
+ *
+ * WSO2 Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.wso2.am.integration.test.utils;
+
+import java.io.IOException;
+import java.net.Socket;
+
+public class ServerPortsUtils {
+
+    public static String LOCALHOST = "localhost";
+    public static final int httpPortLowerRange = 8080;
+    public static final int httpPortUpperRange = 8099;
+    public static final int httpsPortLowerRange = 9950;
+    public static final int httpsPortUpperRange = 9999;
+
+    /**
+     * Check whether give port is available
+     *
+     * @param port Port Number
+     * @return status
+     */
+    private static boolean isPortFree(int port, String host) {
+
+        Socket s = null;
+        try {
+            s = new Socket(host, port);
+            // Something is using the port and has responded.
+            return false;
+        } catch (IOException e) {
+            // Port available
+            return true;
+        } finally {
+            if (s != null) {
+                try {
+                    s.close();
+                } catch (IOException e) {
+                    throw new RuntimeException("Unable to close connection ", e);
+                }
+            }
+        }
+    }
+
+    public static int getAvailableHttpPort(String host) {
+        return getAvailablePort(httpPortLowerRange, httpPortUpperRange, host);
+    }
+
+    public static int getAvailableHttpsPort(String host) {
+        return getAvailablePort(httpsPortLowerRange, httpsPortUpperRange, host);
+    }
+
+    /**
+     * Find a free port to start backend WebSocket server in given port range
+     *
+     * @param lowerPortLimit from port number
+     * @param upperPortLimit to port number
+     * @return Available Port Number
+     */
+    private static int getAvailablePort(int lowerPortLimit, int upperPortLimit, String host) {
+        while (lowerPortLimit < upperPortLimit) {
+            if (ServerPortsUtils.isPortFree(lowerPortLimit, host)) {
+                return lowerPortLimit;
+            }
+            lowerPortLimit += 1;
+        }
+        return -1;
+    }
+
+}

modules/integration/tests-integration/tests-backend/src/test/java/org/wso2/am/integration/tests/api/lifecycle/APIEndpointCertificateTestCase.java

@@ -42,14 +42,18 @@
 import org.wso2.am.integration.test.utils.base.APIMIntegrationConstants;
 import org.wso2.am.integration.test.utils.bean.APIRequest;
 import org.wso2.am.integration.test.utils.http.HttpRequestUtil;
+import org.wso2.carbon.automation.engine.context.AutomationContext;
 import org.wso2.carbon.automation.engine.context.TestUserMode;
 import org.wso2.carbon.automation.test.utils.http.client.HttpResponse;
+import org.wso2.carbon.integration.common.admin.client.LogViewerClient;
+import org.wso2.carbon.logging.view.data.xsd.LogEvent;
 
 import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.Socket;
 import java.net.URL;
+import java.rmi.RemoteException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -78,6 +82,7 @@ public class APIEndpointCertificateTestCase extends APIManagerLifecycleBaseTest
     String apiId;
     WireMockServer wireMockServer;
     private String accessToken;
+    private LogViewerClient logViewerClient;
 
     @Factory(dataProvider = "userModeDataProvider")
     public APIEndpointCertificateTestCase(TestUserMode userMode) {
@@ -137,6 +142,11 @@ public void initialize() throws Exception {
         accessToken = applicationKeyDTO.getToken().getAccessToken();
         waitForAPIDeploymentSync(user.getUserName(), apiRequest.getName(), apiRequest.getVersion(),
                 APIMIntegrationConstants.IS_API_EXISTS);
+        AutomationContext autoContext = new AutomationContext();
+        logViewerClient = new LogViewerClient(autoContext.getContextUrls().getBackEndUrl(),
+                autoContext.getSuperTenant().getTenantAdmin().getUserName(),
+                autoContext.getSuperTenant().getTenantAdmin().getPassword());
+        logViewerClient.clearLogs();
     }
 
     @Test(groups = {"wso2.am"}, description = "Invoke API without inserting Endpoint Certificate")
@@ -263,7 +273,9 @@ public void testSearchEndpointCertificates() throws ApiException, ParseException
             "testSearchEndpointCertificates"})
     public void testInvokeAPI() throws ApiException, InterruptedException, XPathExpressionException, IOException {
 
-        Thread.sleep(60000); // Sleep to reload the transport
+        // Thread.sleep(60000); // Sleep to reload the transport
+        // Wait for SSLProfile with the uploaded certificate to be reloaded in Gateway
+        waitForSSLProfileReload();
         Map<String, String> requestHeaders = new HashMap<>();
         requestHeaders.put("accept", "application/json");
         requestHeaders.put("Authorization", "Bearer " + accessToken);
@@ -282,7 +294,8 @@ public void testInvokeAPIAfterRemovingCertificate() throws InterruptedException,
         Assert.assertEquals(response.getStatusCode(), 200);
         response = restAPIPublisher.deleteEndpointCertificate("endpoint-2");
         Assert.assertEquals(response.getStatusCode(), 200);
-        Thread.sleep(60500); // Sleep to reload the transport
+        // Thread.sleep(60500); // Sleep to reload the transport
+        waitForSSLProfileReload();
         Map<String, String> requestHeaders = new HashMap<>();
         requestHeaders.put("accept", "application/json");
         requestHeaders.put("Authorization", "Bearer " + accessToken);
@@ -320,6 +333,28 @@ private void startSecureEndpoint(int securedEndpointPort) {
         wireMockServer.start();
     }
 
+    private void waitForSSLProfileReload() throws RemoteException, InterruptedException {
+        LogEvent[] logEvents = new LogEvent[0];
+        Thread.sleep(60000);
+        logEvents = logViewerClient.getAllRemoteSystemLogs();
+        int retryAttempt = 0;
+        boolean isSSProfileReloaded = false;
+        while (retryAttempt < 5 && !isSSProfileReloaded) {
+            for (LogEvent logEvent : logEvents) {
+                if (logEvent.getMessage()
+                        .contains("PassThroughHttpSender reloading SSL Config")) {
+                    isSSProfileReloaded = true;
+                    log.info("SSLProfile has been reloaded successfully");
+                    logViewerClient.clearLogs();
+                    break;
+                }
+            }
+            retryAttempt++;
+            log.info("SSLProfile has not been reloaded. Retry attempt - " + retryAttempt);
+            Thread.sleep(12000);
+        }
+    }
+
     @AfterClass(alwaysRun = true)
     public void destroy() throws ApiException {
 

modules/integration/tests-integration/tests-backend/src/test/java/org/wso2/am/integration/tests/graphql/GraphqlSubscriptionTestCase.java

@@ -89,7 +89,6 @@
 import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
@@ -108,7 +107,6 @@ public class GraphqlSubscriptionTestCase extends APIMIntegrationBaseTest {
     private static final String GRAPHQL_API_NAME = "SnowtoothGraphQLSubAPI";
     private static final String GRAPHQL_API_CONTEXT = "snowtooth";
     private static final String GRAPHQL_API_VERSION = "1.0.0";
-    private final ExecutorService executorService = Executors.newSingleThreadExecutor();
     private int webSocketServerPort;
     private String webSocketServerHost;
     private String graphqlApiId;
@@ -118,6 +116,7 @@ public class GraphqlSubscriptionTestCase extends APIMIntegrationBaseTest {
     String throttleAppId;
     String complexAppId;
     String depthAppId;
+    Server server = null;
 
     private enum AUTH_IN {
         HEADER,
@@ -623,8 +622,6 @@ private void invokeGraphQLSubscriptionSuccess(WebSocketClient client, String acc
      */
     private void startGraphQLSubscriptionServer(final int serverPort) {
 
-        executorService.execute(() -> {
-
             WebSocketHandler wsHandler = new WebSocketHandler() {
                 @Override
                 public void configure(WebSocketServletFactory factory) {
@@ -633,7 +630,7 @@ public void configure(WebSocketServletFactory factory) {
                 }
             };
 
-            Server server = new Server(serverPort);
+            server = new Server(serverPort);
             server.setHandler(wsHandler);
             try {
                 server.start();
@@ -643,7 +640,6 @@ public void configure(WebSocketServletFactory factory) {
                 log.error("Error while starting graphql backend server at port: " + serverPort, e);
                 Assert.fail("Cannot start GraphQL WebSocket server");
             }
-        });
     }
 
     /**
@@ -1017,6 +1013,9 @@ private void testThrottling(String accessToken) throws Exception {
                     "Received response in not a Connection Ack response");
             socket.setResponseMessage(null);
             for (int count = 1; count <= limit; count++) {
+                if (count == limit) {
+                    Thread.sleep(3000);
+                }
                 if (count == 1) {
                     //Send initial graphQL subscription request message
                     textMessage = "{\"id\":\"2\",\"type\":\"start\",\"payload\":{\"variables\":{},\"extensions\":{},"
@@ -1065,14 +1064,16 @@ private void testThrottling(String accessToken) throws Exception {
 
     @AfterClass(alwaysRun = true)
     public void destroy() throws Exception {
+        if (server != null) {
+            server.stop();
+        }
         userManagementClient.deleteRole(GRAPHQL_ROLE);
         userManagementClient.deleteUser(GRAPHQL_TEST_USER);
         restAPIStore.deleteApplication(appJWTId);
         restAPIStore.deleteApplication(complexAppId);
         restAPIStore.deleteApplication(depthAppId);
         restAPIStore.deleteApplication(throttleAppId);
         undeployAndDeleteAPIRevisionsUsingRest(graphqlApiId, restAPIPublisher);
-        executorService.shutdownNow();
         super.cleanUp();
     }
 }

modules/integration/tests-integration/tests-backend/src/test/java/org/wso2/am/integration/tests/other/MandatoryPropertiesTestWithRestart.java

@@ -20,7 +20,7 @@
 import org.apache.http.HttpStatus;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Factory;
 import org.testng.annotations.Test;
@@ -67,7 +67,7 @@ public static Object[][] userModeDataProvider() {
         return new Object[][] { new Object[] { TestUserMode.SUPER_TENANT_ADMIN }};
     }
 
-    @BeforeClass(alwaysRun = true)
+    @BeforeTest(alwaysRun = true)
     public void setEnvironment() throws Exception {
         super.init(userMode);
         superTenantKeyManagerContext = new AutomationContext(APIMIntegrationConstants.AM_PRODUCT_GROUP_NAME,

modules/integration/tests-integration/tests-backend/src/test/java/org/wso2/am/integration/tests/other/NotificationTestCase.java

@@ -93,18 +93,24 @@ public void setEnvironment() throws Exception {
         super.init(userMode);
         storeURLHttp = "https://localhost:9943/";
 
-    }
-
-    @Test(groups = {"wso2.am"}, description = "Testing Notification Feature")
-    public void notificationTestCase() throws Exception {
-
         //Setting greenMail server
         ServerSetup setup = new ServerSetup(SMTP_TEST_PORT, "localhost", "smtp");
         greenMail = new GreenMail(setup);
         //Creating user in greenMail server
         greenMail.setUser(USER_EMAIL_ADDRESS, EMAIL_USERNAME, EMAIL_PASSWORD);
-        greenMail.start();
-        log.info("green mail server started ");
+        try {
+            greenMail.start();
+        } catch (IllegalStateException e) {
+            log.warn("There was a problem starting GreenMail server. Retrying in 10 seconds");
+            Thread.sleep(10000);
+            greenMail.start();
+        }
+        log.info("green mail server started");
+
+    }
+
+    @Test(groups = {"wso2.am"}, description = "Testing Notification Feature")
+    public void notificationTestCase() throws Exception {
 
         // Adding API
         String url = getGatewayURLNhttp() + "response";
@@ -212,6 +218,7 @@ public void destroy() throws Exception {
         undeployAndDeleteAPIRevisionsUsingRest(newApiId, restAPIPublisher);
         restAPIPublisher.deleteAPI(apiId);
         restAPIPublisher.deleteAPI(newApiId);
+        greenMail.stop();
     }
 
     @DataProvider