From abba3db5f78325f5292b6a25a1ea4c0d7f9e2512 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 20 Oct 2021 10:50:57 -0700 Subject: [PATCH 001/142] empty to test From d00deed5bb3c40b003b996cae5f2a297500df4c0 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 21 Oct 2021 16:45:32 -0700 Subject: [PATCH 002/142] phase 1 --- .../crt/http/Http2ClientConnection.java | 131 +++++ .../crt/http/Http2ConnectionSetting.java | 25 + .../amazon/awssdk/crt/http/Http2Stream.java | 23 + .../awssdk/crt/http/HttpClientConnection.java | 18 + .../crt/http/HttpClientConnectionManager.java | 8 +- .../HttpClientConnectionManagerOptions.java | 20 +- .../amazon/awssdk/crt/http/HttpStream.java | 2 +- src/native/http_connection_manager.c | 4 +- .../crt/test/Http2RequestResponseTest.java | 450 ++++++++++++++++++ 9 files changed, 676 insertions(+), 5 deletions(-) create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java create mode 100644 src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java new file mode 100644 index 000000000..c520e9ffe --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -0,0 +1,131 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package software.amazon.awssdk.crt.http; + +import java.util.concurrent.CompletableFuture; + +/** + * This class wraps aws-c-http to provide the basic HTTP/2 request/response + * functionality via the AWS Common Runtime. + * + * Http2ClientConnection represents a single connection to a HTTP/2 service + * endpoint. + * + * This class is not thread safe and should not be called from different + * threads. + */ +public class Http2ClientConnection extends HttpClientConnection { + + public enum Http2ErrorCode { + PROTOCOL_ERROR(1), + INTERNAL_ERROR(2), + FLOW_CONTROL_ERROR(3), + SETTINGS_TIMEOUT(4), + STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), + REFUSED_STREAM(7), + CANCEL(8), + COMPRESSION_ERROR(9), + CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), + INADEQUATE_SECURITY(12), + HTTP_1_1_REQUIRED(13); + + private int errorCode; + + Http2ErrorCode(int value) { + errorCode = value; + } + + public int getValue() { + return errorCode; + } + } + + private Http2ClientConnection(long connectionBinding) { + super(connectionBinding); + } + /** + * Send a SETTINGS frame. + * SETTINGS will be applied locally when SETTINGS ACK is received from peer. + * + * @param settings The array of settings to change. Note: each setting has its boundary. + * + * @return When this future completes without exception, the peer has acknowledged the settings and + * the change has been applied. + */ + public CompletableFuture changeSettings(final Http2ConnectionSetting settings[]) { + CompletableFuture completionFuture = new CompletableFuture<>(); + return completionFuture; + } + + /** + * Send a PING frame + * Round-trip-time is calculated when PING ACK is received from peer. + * + * @param pingData Optional 8 Bytes data with the PING frame + * + * @return When this future completes without exception, the peer has acknowledged the PING and + * future will be completed with the round trip time in nano seconds for the connection. + */ + public CompletableFuture sendPing(final byte[] pingData) { + CompletableFuture completionFuture = new CompletableFuture<>(); + return completionFuture; + } + + /** + * Send a custom GOAWAY frame. + * + * Note that the connection automatically attempts to send a GOAWAY during + * shutdown (unless a GOAWAY with a valid Last-Stream-ID has already been sent). + * + * This call can be used to gracefully warn the peer of an impending shutdown + * (http2_error=0, allow_more_streams=true), or to customize the final GOAWAY + * frame that is sent by this connection. + * + * The other end may not receive the goaway, if the connection already closed. + * + * @param Http2ErrorCode The HTTP/2 error code (RFC-7540 section 7) to send. + * `enum Http2ErrorCode` lists official codes. + * @param allowMoreStreams If true, new peer-initiated streams will continue + * to be acknowledged and the GOAWAY's Last-Stream-ID will be set to a max value. + * If false, new peer-initiated streams will be ignored and the GOAWAY's + * Last-Stream-ID will be set to the latest acknowledged stream. + * @param debugData Optional debug data to send. Size must not exceed 16KB. + */ + public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams, final byte[] debugData) { + } + + /** + * Increment the connection's flow-control window to keep data flowing. + * + * If the connection was created with `manualWindowManagement` set true, + * the flow-control window of the connection will shrink as body data is received for all the streams created on it. + * (headers, padding, and other metadata do not affect the window). + * The initial connection flow-control window is 65,535. + * Once the connection's flow-control window reaches to 0, all the streams on the connection stop receiving any further + * data. + * + * If `manualWindowManagement` is false, this call will have no effect. + * The connection maintains its flow-control windows such that + * no back-pressure is applied and data arrives as fast as possible. + * + * If you are not connected, this call will have no effect. + * + * Crashes when the connection is not http2 connection. + * The limit of the Maximum Size is 2**31 - 1. If the increment size cause the connection flow window exceeds the + * Maximum size, this call will result in the connection lost. + * + * @param increment_size The size to increment for the connection's flow control window + */ + public void updateConnectionWindow(long incrementSize){ + + } + + /** + * @TODO: bindings for getters of local/remote setting and goaway. + */ +} diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java new file mode 100644 index 000000000..a2b6ec683 --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -0,0 +1,25 @@ +package software.amazon.awssdk.crt.http; + +public class Http2ConnectionSetting { + public enum Http2ConnectionSettingID { + HEADER_TABLE_SIZE(1), + ENABLE_PUSH(2), + MAX_CONCURRENT_STREAMS(3), + INITIAL_WINDOW_SIZE(4), + MAX_FRAME_SIZE(5), + MAX_HEADER_LIST_SIZE(6); + + private int settingID; + + Http2ConnectionSettingID(int value) { + settingID = value; + } + + public int getValue() { + return settingID; + } + } + + public Http2ConnectionSettingID id; + public long value; +} diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java new file mode 100644 index 000000000..208c9cf8b --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java @@ -0,0 +1,23 @@ +package software.amazon.awssdk.crt.http; + +public class Http2Stream extends HttpStream { + + protected Http2Stream(long ptr) { + super(ptr); + } + + /** + * Reset the HTTP/2 stream + * Note that if the stream closes before this async call is fully processed, the RST_STREAM frame will not be sent. + * + * @param http2_stream HTTP/2 stream. + * @param http2_error aws_http2_error_code. Reason to reset the stream. + */ + public void resetStream(final Http2ClientConnection.Http2ErrorCode errorCode) { + + } + + /** + * @TODO getters for reset stream. Not sure anyone needs it. + */ +} diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index 21e3b7b80..614a718d7 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -23,6 +23,24 @@ * This class is not thread safe and should not be called from different threads. */ public class HttpClientConnection extends CrtResource { + /** + * HTTP protocol version. + */ + public enum AwsHTTPProtocolVersion { + HTTP_1_0(1), + HTTP_1_1(2), + HTTP_2(3); + + private int protocolVersion; + + AwsHTTPProtocolVersion(int value) { + protocolVersion = value; + } + + public int getValue() { + return protocolVersion; + } + } protected HttpClientConnection(long connectionBinding) { acquireNativeHandle(connectionBinding); diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java index 8ef3d9b09..f9b321b54 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java @@ -32,6 +32,7 @@ public class HttpClientConnectionManager extends CrtResource { private final int port; private final int maxConnections; private final CompletableFuture shutdownComplete = new CompletableFuture<>(); + private final HttpClientConnection.AwsHTTPProtocolVersion expectedProtocolVersion; public static HttpClientConnectionManager create(HttpClientConnectionManagerOptions options) { return new HttpClientConnectionManager(options); @@ -76,6 +77,7 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options) this.uri = uri; this.port = port; this.maxConnections = maxConnections; + this.expectedProtocolVersion = options.getExpectedProtocolVersion(); int proxyConnectionType = 0; String proxyHost = null; @@ -121,7 +123,8 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options) options.isManualWindowManagement(), options.getMaxConnectionIdleInMilliseconds(), monitoringThroughputThresholdInBytesPerSecond, - monitoringFailureIntervalInSeconds)); + monitoringFailureIntervalInSeconds, + expectedProtocolVersion.getValue())); /* we don't need to add a reference to socketOptions since it's copied during connection manager construction */ addReferenceTo(clientBootstrap); @@ -227,7 +230,8 @@ private static native long httpClientConnectionManagerNew(HttpClientConnectionMa boolean isManualWindowManagement, long maxConnectionIdleInMilliseconds, long monitoringThroughputThresholdInBytesPerSecond, - int monitoringFailureIntervalInSeconds) throws CrtRuntimeException; + int monitoringFailureIntervalInSeconds, + int expectedProtocol) throws CrtRuntimeException; private static native void httpClientConnectionManagerRelease(long conn_manager) throws CrtRuntimeException; diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java index a6bab7b91..2ce4f88b7 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java @@ -29,6 +29,7 @@ public class HttpClientConnectionManagerOptions { private boolean manualWindowManagement = false; private HttpMonitoringOptions monitoringOptions; private long maxConnectionIdleInMilliseconds = 0; + private HttpClientConnection.AwsHTTPProtocolVersion expectedProtocolVersion = HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1; public HttpClientConnectionManagerOptions() { } @@ -196,6 +197,24 @@ public HttpClientConnectionManagerOptions withManualWindowManagement(boolean man return this; } + /** + * If called, HTTP2 connection will be made by connection manager with ALPN (h2;http/1.1) when TLS is used. + * @TODO: When TLS is not used, HTTP2 connection will be made with prior knowledge + * + * @return this + */ + public HttpClientConnectionManagerOptions withExpectedProtocolVersion(HttpClientConnection.AwsHTTPProtocolVersion expectedProtocolVersion) { + this.expectedProtocolVersion = expectedProtocolVersion; + return this; + } + + /** + * @return Return the expected HTTP protocol version. + */ + public HttpClientConnection.AwsHTTPProtocolVersion getExpectedProtocolVersion() { + return expectedProtocolVersion; + } + /** * Sets maximum amount of time, in milliseconds, that the connection can be idle in the manager before * getting culled by the manager @@ -227,4 +246,3 @@ public HttpClientConnectionManagerOptions withMonitoringOptions(HttpMonitoringOp */ public HttpMonitoringOptions getMonitoringOptions() { return monitoringOptions; } } - diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java index cd9313a6a..bf44015d5 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java @@ -137,6 +137,6 @@ public int getResponseStatusCode() { private static native void httpStreamRelease(long http_stream); private static native void httpStreamIncrementWindow(long http_stream, int window_size); private static native void httpStreamActivate(long http_stream, HttpStream streamObj); - private static native int httpStreamGetResponseStatusCode(long http_stream); + private static native int httpStreamGetResponseStatusCode(long http_stream); private static native int httpStreamWriteChunk(long http_stream, byte[] chunkData, boolean isFinalChunk, HttpStreamWriteChunkCompletionCallback completionCallback); } diff --git a/src/native/http_connection_manager.c b/src/native/http_connection_manager.c index 010e29fa2..629542936 100644 --- a/src/native/http_connection_manager.c +++ b/src/native/http_connection_manager.c @@ -160,9 +160,11 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnectio jboolean jni_manual_window_management, jlong jni_max_connection_idle_in_milliseconds, jlong jni_monitoring_throughput_threshold_in_bytes_per_second, - jint jni_monitoring_failure_interval_in_seconds) { + jint jni_monitoring_failure_interval_in_seconds, + jint jni_expected_protocol_version) { (void)jni_class; + (void)jni_expected_protocol_version; struct aws_client_bootstrap *client_bootstrap = (struct aws_client_bootstrap *)jni_client_bootstrap; struct aws_socket_options *socket_options = (struct aws_socket_options *)jni_socket_options; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java new file mode 100644 index 000000000..a412bbca6 --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -0,0 +1,450 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package software.amazon.awssdk.crt.test; + +import static software.amazon.awssdk.crt.utils.ByteBufferUtils.transferData; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; +import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.io.TlsContextOptions; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +public class Http2RequestResponseTest extends HttpClientTestFixture { + private final static Charset UTF8 = StandardCharsets.UTF_8; + private final String EMPTY_BODY = ""; + private final static String TEST_DOC_LINE = "This is a sample to prove that http downloads and uploads work. It doesn't really matter what's in here, we mainly just need to verify the downloads and uploads work."; + private final static String TEST_DOC_SHA256 = "C7FDB5314B9742467B16BD5EA2F8012190B5E2C44A005F7984F89AAB58219534"; + + private class TestHttpResponse { + int statusCode = -1; + int blockType = -1; + List headers = new ArrayList<>(); + ByteBuffer bodyBuffer = ByteBuffer.wrap(new byte[16*1024*1024]); // Allow up to 16 MB Responses + int onCompleteErrorCode = -1; + + public String getBody() { + bodyBuffer.flip(); + return UTF8.decode(bodyBuffer).toString(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Status: " + statusCode); + int i = 0; + for (HttpHeader h: headers) { + builder.append("\nHeader[" + i + "]: " + h.toString()); + } + + builder.append("\nBody:\n"); + builder.append(getBody()); + + return builder.toString(); + } + } + + public static String byteArrayToHex(byte[] input) { + StringBuilder output = new StringBuilder(input.length * 2); + for (byte b: input) { + output.append(String.format("%02X", b)); + } + return output.toString(); + } + + private String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmException { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + digest.update(bodyBuffer); + return byteArrayToHex(digest.digest()); + } + + private HttpClientConnectionManager createHTTP2ConnectionPoolManager(URI uri) { + try(EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient().withAlpnList("h2;http/1.1"); + TlsContext tlsContext = createHttpClientTlsContext(tlsContextOptions)) { + + + HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() + .withClientBootstrap(bootstrap) + .withSocketOptions(sockOpts) + .withTlsContext(tlsContext) + .withUri(uri); + + return HttpClientConnectionManager.create(options); + } + } + + public TestHttpResponse getResponse(URI uri, HttpRequest request, byte[] chunkedData) throws Exception { + boolean actuallyConnected = false; + + final CompletableFuture reqCompleted = new CompletableFuture<>(); + + final TestHttpResponse response = new TestHttpResponse(); + + CompletableFuture shutdownComplete = null; + try (HttpClientConnectionManager connPool = createHTTP2ConnectionPoolManager(uri)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { + actuallyConnected = true; + HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { + response.statusCode = responseStatusCode; + Assert.assertEquals(responseStatusCode, stream.getResponseStatusCode()); + response.headers.addAll(Arrays.asList(nextHeaders)); + } + + @Override + public void onResponseHeadersDone(HttpStream stream, int blockType) { + response.blockType = blockType; + } + + @Override + public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + response.bodyBuffer.put(bodyBytesIn); + int amountRead = bodyBytesIn.length; + + // Slide the window open by the number of bytes just read + return amountRead; + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + response.onCompleteErrorCode = errorCode; + reqCompleted.complete(null); + stream.close(); + } + }; + + HttpStream stream = conn.makeRequest(request, streamHandler); + stream.activate(); + + if (chunkedData != null) { + stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); + } + // Give the request up to 60 seconds to complete, otherwise throw a TimeoutException + reqCompleted.get(60, TimeUnit.SECONDS); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(); + + CrtResource.waitForNoResources(); + + return response; + } + + private boolean shouldRetry(TestHttpResponse response) { + // Retry if we couldn't connect or if we got 503 response + if (response.onCompleteErrorCode != CRT.AWS_CRT_SUCCESS || response.statusCode == 503) { + return true; + } + return false; + } + + public TestHttpResponse testRequest(String method, String endpoint, String path, String requestBody, boolean useChunkedEncoding, int expectedStatus) throws Exception { + URI uri = new URI(endpoint); + + HttpHeader[] requestHeaders = null; + + if (!useChunkedEncoding) { + requestHeaders = + new HttpHeader[]{ + new HttpHeader("Host", uri.getHost()), + new HttpHeader("Content-Length", Integer.toString(requestBody.getBytes(UTF8).length)) + }; + } else { + requestHeaders = + new HttpHeader[]{ + new HttpHeader("Host", uri.getHost()), + new HttpHeader("Transfer-Encoding", "chunked") + }; + } + + HttpRequestBodyStream bodyStream = null; + + if (!useChunkedEncoding) { + final ByteBuffer bodyBytesIn = ByteBuffer.wrap(requestBody.getBytes(UTF8)); + + bodyStream = new HttpRequestBodyStream() { + @Override + public boolean sendRequestBody(ByteBuffer bodyBytesOut) { + transferData(bodyBytesIn, bodyBytesOut); + + return bodyBytesIn.remaining() == 0; + } + + @Override + public boolean resetPosition() { + bodyBytesIn.position(0); + + return true; + } + }; + } + + HttpRequest request = new HttpRequest(method, path, requestHeaders, bodyStream); + + TestHttpResponse response = null; + int numAttempts = 0; + do { + + if (request.getBodyStream() != null) { + request.getBodyStream().resetPosition(); + } + + numAttempts++; + response = null; + try { + if (useChunkedEncoding) { + response = getResponse(uri, request, requestBody.getBytes(UTF8)); + } else { + response = getResponse(uri, request, null); + } + } catch (Exception ex) { + //do nothing just let it retry + } + + } while ((response == null || shouldRetry(response)) && numAttempts < 3); + + Assert.assertNotEquals(-1, response.blockType); + + boolean hasContentLengthHeader = false; + + for (HttpHeader h: response.headers) { + if (h.getName().equals("Content-Length")) { + hasContentLengthHeader = true; + } + } + + Assert.assertTrue(hasContentLengthHeader); + if (response.statusCode < 500) { // if the server errored, not our fault + Assert.assertEquals("Expected and Actual Status Codes don't match", expectedStatus, response.statusCode); + } + + return response; + } + + + @Test + public void testHttpDelete() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + testRequest("DELETE", "https://httpbin.org", "/delete", EMPTY_BODY, false,200); + testRequest("DELETE", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); + testRequest("DELETE", "https://httpbin.org", "/post", EMPTY_BODY, false, 405); + testRequest("DELETE", "https://httpbin.org", "/put", EMPTY_BODY, false, 405); + } + + + @Test + public void testHttpGet() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + testRequest("GET", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); + testRequest("GET", "https://httpbin.org", "/get", EMPTY_BODY, false,200); + testRequest("GET", "https://httpbin.org", "/post", EMPTY_BODY, false,405); + testRequest("GET", "https://httpbin.org", "/put", EMPTY_BODY, false,405); + } + + @Test + public void testHttpPost() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + testRequest("POST", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); + testRequest("POST", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); + testRequest("POST", "https://httpbin.org", "/post", EMPTY_BODY, false,200); + testRequest("POST", "https://httpbin.org", "/put", EMPTY_BODY, false,405); + } + + @Test + public void testHttpPut() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + testRequest("PUT", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); + testRequest("PUT", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); + testRequest("PUT", "https://httpbin.org", "/post", EMPTY_BODY, false, 405); + testRequest("PUT", "https://httpbin.org", "/put", EMPTY_BODY, false,200); + } + + @Test + public void testHttpResponseStatusCodes() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + testRequest("GET", "https://httpbin.org", "/status/200", EMPTY_BODY, false,200); + testRequest("GET", "https://httpbin.org", "/status/300", EMPTY_BODY, false, 300); + testRequest("GET", "https://httpbin.org", "/status/400", EMPTY_BODY, false, 400); + testRequest("GET", "https://httpbin.org", "/status/500", EMPTY_BODY, false,500); + } + + + + @Test + public void testHttpDownload() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + TestHttpResponse response = testRequest("GET", "https://aws-crt-test-stuff.s3.amazonaws.com", "/http_test_doc.txt", EMPTY_BODY, false,200); + + ByteBuffer body = response.bodyBuffer; + body.flip(); //Flip from Write mode to Read mode + + Assert.assertEquals(TEST_DOC_SHA256, calculateBodyHash(body)); + } + + /** + * Removes trailing commas, and trims quote characters from a string. + * + * @param input + * @return + */ + private String extractValueFromJson(String input) { + return input.trim() // Remove spaces from front and back + .replaceAll(",$", "") // Remove comma if it's the last character + .replaceAll("^\"|\"$", ""); // Remove quotes from front and back + } + + private void testHttpUpload(boolean chunked) throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + String bodyToSend = TEST_DOC_LINE; + TestHttpResponse response = testRequest("PUT", "https://httpbin.org", "/anything", bodyToSend, chunked,200); + + // Get the Body bytes that were echoed back to us + String body = response.getBody(); + + /** + * Example Json Response Body from httpbin.org: + * + * { + * "args": {}, + * "data": "This is a sample to prove that http downloads and uploads work. It doesn't really matter what's in here, we mainly just need to verify the downloads and uploads work.", + * "files": {}, + * "form": {}, + * "headers": { + * "Content-Length": "166", + * "Host": "httpbin.org" + * }, + * "json": null, + * "method": "PUT", + * "origin": "1.2.3.4, 5.6.7.8", + * "url": "https://httpbin.org/anything" + * } + * + */ + + String echoedBody = null; + for (String line: body.split("\n")) { + String[] keyAndValue = line.split(":", 2); + + // Found JSON Key/Value Pair + if (keyAndValue.length == 2) { + String key = extractValueFromJson(keyAndValue[0]); + String val = extractValueFromJson(keyAndValue[1]); + + // Found Echoed Body + if (key.equals("data")) { + echoedBody = extractValueFromJson(val); + } + } + } + + Assert.assertNotNull("Response Body did not contain \"data\" JSON key:\n" + body, echoedBody); + Assert.assertEquals(bodyToSend, echoedBody); + } + + @Test + public void testHttpUpload() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + testHttpUpload(false); + } + + @Test + public void testHttpUploadChunked() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + testHttpUpload(true); + } + + @Test + public void testHttpRequestUnActivated() throws Exception { + Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); + + URI uri = new URI("https://httpbin.org"); + + HttpHeader[] requestHeaders = + new HttpHeader[]{ + new HttpHeader("Host", uri.getHost()) + }; + + HttpRequest request = new HttpRequest("GET", "/get", requestHeaders, null); + + CompletableFuture shutdownComplete = null; + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { + HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { + // do nothing + } + + @Override + public void onResponseHeadersDone(HttpStream stream, int blockType) { + // do nothing + } + + @Override + public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + //do nothing + return bodyBytesIn.length; + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + //do nothing. + } + }; + + HttpStream stream = conn.makeRequest(request, streamHandler); + stream.close(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (shutdownComplete != null) { + shutdownComplete.get(); + } + + CrtResource.waitForNoResources(); + } + +} From 6c3d841a43602013327471e67492e9ba44cf73c5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 22 Oct 2021 10:23:04 -0700 Subject: [PATCH 003/142] add comment --- .../software/amazon/awssdk/crt/http/Http2ClientConnection.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index c520e9ffe..b3e14f1e8 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -19,6 +19,7 @@ */ public class Http2ClientConnection extends HttpClientConnection { + /* Error codes that may be present in HTTP/2 RST_STREAM and GOAWAY frames (RFC-7540 7). */ public enum Http2ErrorCode { PROTOCOL_ERROR(1), INTERNAL_ERROR(2), From 0b37a886f35d6206db4ca55606226bb024ea37ad Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Oct 2021 09:45:27 -0700 Subject: [PATCH 004/142] test --- .../crt/test/Http2RequestResponseTest.java | 410 +++--------------- .../crt/test/HttpClientTestFixture.java | 1 - .../crt/test/HttpRequestResponseFixture.java | 202 +++++++++ .../crt/test/HttpRequestResponseTest.java | 250 ++--------- 4 files changed, 297 insertions(+), 566 deletions(-) create mode 100644 src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index a412bbca6..2be8ea7c6 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -10,214 +10,45 @@ import org.junit.Assert; import org.junit.Assume; import org.junit.Test; -import software.amazon.awssdk.crt.CRT; -import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.http.HttpClientConnection; -import software.amazon.awssdk.crt.http.HttpClientConnectionManager; -import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; import software.amazon.awssdk.crt.http.HttpRequestBodyStream; -import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; -import software.amazon.awssdk.crt.http.HttpStream; -import software.amazon.awssdk.crt.io.ClientBootstrap; -import software.amazon.awssdk.crt.io.EventLoopGroup; -import software.amazon.awssdk.crt.io.HostResolver; -import software.amazon.awssdk.crt.io.SocketOptions; -import software.amazon.awssdk.crt.io.TlsContext; -import software.amazon.awssdk.crt.io.TlsContextOptions; import java.net.URI; import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -public class Http2RequestResponseTest extends HttpClientTestFixture { - private final static Charset UTF8 = StandardCharsets.UTF_8; - private final String EMPTY_BODY = ""; - private final static String TEST_DOC_LINE = "This is a sample to prove that http downloads and uploads work. It doesn't really matter what's in here, we mainly just need to verify the downloads and uploads work."; - private final static String TEST_DOC_SHA256 = "C7FDB5314B9742467B16BD5EA2F8012190B5E2C44A005F7984F89AAB58219534"; - - private class TestHttpResponse { - int statusCode = -1; - int blockType = -1; - List headers = new ArrayList<>(); - ByteBuffer bodyBuffer = ByteBuffer.wrap(new byte[16*1024*1024]); // Allow up to 16 MB Responses - int onCompleteErrorCode = -1; - - public String getBody() { - bodyBuffer.flip(); - return UTF8.decode(bodyBuffer).toString(); - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Status: " + statusCode); - int i = 0; - for (HttpHeader h: headers) { - builder.append("\nHeader[" + i + "]: " + h.toString()); - } - - builder.append("\nBody:\n"); - builder.append(getBody()); - - return builder.toString(); - } - } - public static String byteArrayToHex(byte[] input) { - StringBuilder output = new StringBuilder(input.length * 2); - for (byte b: input) { - output.append(String.format("%02X", b)); - } - return output.toString(); - } +public class Http2RequestResponseTest extends HttpRequestResponseFixture { - private String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmException { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - digest.update(bodyBuffer); - return byteArrayToHex(digest.digest()); - } - - private HttpClientConnectionManager createHTTP2ConnectionPoolManager(URI uri) { - try(EventLoopGroup eventLoopGroup = new EventLoopGroup(1); - HostResolver resolver = new HostResolver(eventLoopGroup); - ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); - SocketOptions sockOpts = new SocketOptions(); - TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient().withAlpnList("h2;http/1.1"); - TlsContext tlsContext = createHttpClientTlsContext(tlsContextOptions)) { - - - HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() - .withClientBootstrap(bootstrap) - .withSocketOptions(sockOpts) - .withTlsContext(tlsContext) - .withUri(uri); - - return HttpClientConnectionManager.create(options); - } - } - - public TestHttpResponse getResponse(URI uri, HttpRequest request, byte[] chunkedData) throws Exception { - boolean actuallyConnected = false; - - final CompletableFuture reqCompleted = new CompletableFuture<>(); - - final TestHttpResponse response = new TestHttpResponse(); - - CompletableFuture shutdownComplete = null; - try (HttpClientConnectionManager connPool = createHTTP2ConnectionPoolManager(uri)) { - shutdownComplete = connPool.getShutdownCompleteFuture(); - try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { - actuallyConnected = true; - HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { - @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { - response.statusCode = responseStatusCode; - Assert.assertEquals(responseStatusCode, stream.getResponseStatusCode()); - response.headers.addAll(Arrays.asList(nextHeaders)); - } - - @Override - public void onResponseHeadersDone(HttpStream stream, int blockType) { - response.blockType = blockType; - } - - @Override - public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { - response.bodyBuffer.put(bodyBytesIn); - int amountRead = bodyBytesIn.length; - - // Slide the window open by the number of bytes just read - return amountRead; - } - - @Override - public void onResponseComplete(HttpStream stream, int errorCode) { - response.onCompleteErrorCode = errorCode; - reqCompleted.complete(null); - stream.close(); - } - }; - - HttpStream stream = conn.makeRequest(request, streamHandler); - stream.activate(); - - if (chunkedData != null) { - stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); - } - // Give the request up to 60 seconds to complete, otherwise throw a TimeoutException - reqCompleted.get(60, TimeUnit.SECONDS); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - Assert.assertTrue(actuallyConnected); - - shutdownComplete.get(); - - CrtResource.waitForNoResources(); - - return response; - } - - private boolean shouldRetry(TestHttpResponse response) { - // Retry if we couldn't connect or if we got 503 response - if (response.onCompleteErrorCode != CRT.AWS_CRT_SUCCESS || response.statusCode == 503) { - return true; - } - return false; - } - - public TestHttpResponse testRequest(String method, String endpoint, String path, String requestBody, boolean useChunkedEncoding, int expectedStatus) throws Exception { + public TestHttpResponse testHttp2Request(String method, String endpoint, String path, String requestBody, + int expectedStatus) throws Exception { URI uri = new URI(endpoint); HttpHeader[] requestHeaders = null; - if (!useChunkedEncoding) { - requestHeaders = - new HttpHeader[]{ - new HttpHeader("Host", uri.getHost()), - new HttpHeader("Content-Length", Integer.toString(requestBody.getBytes(UTF8).length)) - }; - } else { - requestHeaders = - new HttpHeader[]{ - new HttpHeader("Host", uri.getHost()), - new HttpHeader("Transfer-Encoding", "chunked") - }; - } + /* TODO: Http2 headers and request */ + requestHeaders = new HttpHeader[] { new HttpHeader("host", uri.getHost()), + new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)) }; HttpRequestBodyStream bodyStream = null; - if (!useChunkedEncoding) { - final ByteBuffer bodyBytesIn = ByteBuffer.wrap(requestBody.getBytes(UTF8)); + final ByteBuffer bodyBytesIn = ByteBuffer.wrap(requestBody.getBytes(UTF8)); - bodyStream = new HttpRequestBodyStream() { - @Override - public boolean sendRequestBody(ByteBuffer bodyBytesOut) { - transferData(bodyBytesIn, bodyBytesOut); + bodyStream = new HttpRequestBodyStream() { + @Override + public boolean sendRequestBody(ByteBuffer bodyBytesOut) { + transferData(bodyBytesIn, bodyBytesOut); - return bodyBytesIn.remaining() == 0; - } + return bodyBytesIn.remaining() == 0; + } - @Override - public boolean resetPosition() { - bodyBytesIn.position(0); + @Override + public boolean resetPosition() { + bodyBytesIn.position(0); - return true; - } - }; - } + return true; + } + }; HttpRequest request = new HttpRequest(method, path, requestHeaders, bodyStream); @@ -232,13 +63,10 @@ public boolean resetPosition() { numAttempts++; response = null; try { - if (useChunkedEncoding) { - response = getResponse(uri, request, requestBody.getBytes(UTF8)); - } else { - response = getResponse(uri, request, null); - } + response = getResponse(uri, request, null, HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2); + } catch (Exception ex) { - //do nothing just let it retry + // do nothing just let it retry } } while ((response == null || shouldRetry(response)) && numAttempts < 3); @@ -247,8 +75,10 @@ public boolean resetPosition() { boolean hasContentLengthHeader = false; - for (HttpHeader h: response.headers) { - if (h.getName().equals("Content-Length")) { + /* The first header of response has to be ":status" for HTTP/2 response */ + response.headers.get(0).getName().equals(":status"); + for (HttpHeader h : response.headers) { + if (h.getName().equals("content-length")) { hasContentLengthHeader = true; } } @@ -261,190 +91,52 @@ public boolean resetPosition() { return response; } - - @Test - public void testHttpDelete() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testRequest("DELETE", "https://httpbin.org", "/delete", EMPTY_BODY, false,200); - testRequest("DELETE", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); - testRequest("DELETE", "https://httpbin.org", "/post", EMPTY_BODY, false, 405); - testRequest("DELETE", "https://httpbin.org", "/put", EMPTY_BODY, false, 405); - } - - @Test public void testHttpGet() throws Exception { Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testRequest("GET", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); - testRequest("GET", "https://httpbin.org", "/get", EMPTY_BODY, false,200); - testRequest("GET", "https://httpbin.org", "/post", EMPTY_BODY, false,405); - testRequest("GET", "https://httpbin.org", "/put", EMPTY_BODY, false,405); + testHttp2Request("GET", "https://httpbin.org", "/delete", EMPTY_BODY, 405); + testHttp2Request("GET", "https://httpbin.org", "/get", EMPTY_BODY, 200); + testHttp2Request("GET", "https://httpbin.org", "/post", EMPTY_BODY, 405); + testHttp2Request("GET", "https://httpbin.org", "/put", EMPTY_BODY, 405); } @Test public void testHttpPost() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testRequest("POST", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); - testRequest("POST", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); - testRequest("POST", "https://httpbin.org", "/post", EMPTY_BODY, false,200); - testRequest("POST", "https://httpbin.org", "/put", EMPTY_BODY, false,405); + skipIfNetworkUnavailable(); + testHttp2Request("POST", "https://httpbin.org", "/delete", EMPTY_BODY, 405); + testHttp2Request("POST", "https://httpbin.org", "/get", EMPTY_BODY, 405); + testHttp2Request("POST", "https://httpbin.org", "/post", EMPTY_BODY, 200); + testHttp2Request("POST", "https://httpbin.org", "/put", EMPTY_BODY, 405); } @Test public void testHttpPut() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testRequest("PUT", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); - testRequest("PUT", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); - testRequest("PUT", "https://httpbin.org", "/post", EMPTY_BODY, false, 405); - testRequest("PUT", "https://httpbin.org", "/put", EMPTY_BODY, false,200); + skipIfNetworkUnavailable(); + testHttp2Request("PUT", "https://httpbin.org", "/delete", EMPTY_BODY, 405); + testHttp2Request("PUT", "https://httpbin.org", "/get", EMPTY_BODY, 405); + testHttp2Request("PUT", "https://httpbin.org", "/post", EMPTY_BODY, 405); + testHttp2Request("PUT", "https://httpbin.org", "/put", EMPTY_BODY, 200); } @Test public void testHttpResponseStatusCodes() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testRequest("GET", "https://httpbin.org", "/status/200", EMPTY_BODY, false,200); - testRequest("GET", "https://httpbin.org", "/status/300", EMPTY_BODY, false, 300); - testRequest("GET", "https://httpbin.org", "/status/400", EMPTY_BODY, false, 400); - testRequest("GET", "https://httpbin.org", "/status/500", EMPTY_BODY, false,500); + skipIfNetworkUnavailable(); + testHttp2Request("GET", "https://httpbin.org", "/status/200", EMPTY_BODY, 200); + testHttp2Request("GET", "https://httpbin.org", "/status/300", EMPTY_BODY, 300); + testHttp2Request("GET", "https://httpbin.org", "/status/400", EMPTY_BODY, 400); + testHttp2Request("GET", "https://httpbin.org", "/status/500", EMPTY_BODY, 500); } - - @Test public void testHttpDownload() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - TestHttpResponse response = testRequest("GET", "https://aws-crt-test-stuff.s3.amazonaws.com", "/http_test_doc.txt", EMPTY_BODY, false,200); + skipIfNetworkUnavailable(); + /* cloudfront uses HTTP/2 */ + TestHttpResponse response = testHttp2Request("GET", "https://d1cz66xoahf9cl.cloudfront.net/", + "/http_test_doc.txt", EMPTY_BODY, 200); ByteBuffer body = response.bodyBuffer; - body.flip(); //Flip from Write mode to Read mode + body.flip(); // Flip from Write mode to Read mode Assert.assertEquals(TEST_DOC_SHA256, calculateBodyHash(body)); } - - /** - * Removes trailing commas, and trims quote characters from a string. - * - * @param input - * @return - */ - private String extractValueFromJson(String input) { - return input.trim() // Remove spaces from front and back - .replaceAll(",$", "") // Remove comma if it's the last character - .replaceAll("^\"|\"$", ""); // Remove quotes from front and back - } - - private void testHttpUpload(boolean chunked) throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - String bodyToSend = TEST_DOC_LINE; - TestHttpResponse response = testRequest("PUT", "https://httpbin.org", "/anything", bodyToSend, chunked,200); - - // Get the Body bytes that were echoed back to us - String body = response.getBody(); - - /** - * Example Json Response Body from httpbin.org: - * - * { - * "args": {}, - * "data": "This is a sample to prove that http downloads and uploads work. It doesn't really matter what's in here, we mainly just need to verify the downloads and uploads work.", - * "files": {}, - * "form": {}, - * "headers": { - * "Content-Length": "166", - * "Host": "httpbin.org" - * }, - * "json": null, - * "method": "PUT", - * "origin": "1.2.3.4, 5.6.7.8", - * "url": "https://httpbin.org/anything" - * } - * - */ - - String echoedBody = null; - for (String line: body.split("\n")) { - String[] keyAndValue = line.split(":", 2); - - // Found JSON Key/Value Pair - if (keyAndValue.length == 2) { - String key = extractValueFromJson(keyAndValue[0]); - String val = extractValueFromJson(keyAndValue[1]); - - // Found Echoed Body - if (key.equals("data")) { - echoedBody = extractValueFromJson(val); - } - } - } - - Assert.assertNotNull("Response Body did not contain \"data\" JSON key:\n" + body, echoedBody); - Assert.assertEquals(bodyToSend, echoedBody); - } - - @Test - public void testHttpUpload() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testHttpUpload(false); - } - - @Test - public void testHttpUploadChunked() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testHttpUpload(true); - } - - @Test - public void testHttpRequestUnActivated() throws Exception { - Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - - URI uri = new URI("https://httpbin.org"); - - HttpHeader[] requestHeaders = - new HttpHeader[]{ - new HttpHeader("Host", uri.getHost()) - }; - - HttpRequest request = new HttpRequest("GET", "/get", requestHeaders, null); - - CompletableFuture shutdownComplete = null; - try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri)) { - shutdownComplete = connPool.getShutdownCompleteFuture(); - try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { - HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { - @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { - // do nothing - } - - @Override - public void onResponseHeadersDone(HttpStream stream, int blockType) { - // do nothing - } - - @Override - public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { - //do nothing - return bodyBytesIn.length; - } - - @Override - public void onResponseComplete(HttpStream stream, int errorCode) { - //do nothing. - } - }; - - HttpStream stream = conn.makeRequest(request, streamHandler); - stream.close(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - if (shutdownComplete != null) { - shutdownComplete.get(); - } - - CrtResource.waitForNoResources(); - } - -} +} \ No newline at end of file diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java index d23f6471d..6aa0d3063 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java @@ -17,4 +17,3 @@ public TlsContext createHttpClientTlsContext(TlsContextOptions tlsOpts) { return new TlsContext(configureTlsContextOptions(tlsOpts, getContext().trustStore)); } } - diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java new file mode 100644 index 000000000..00e8d3445 --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -0,0 +1,202 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package software.amazon.awssdk.crt.test; + +import org.junit.Assert; +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; +import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.io.TlsContextOptions; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +public class HttpRequestResponseFixture extends HttpClientTestFixture { + + protected final static Charset UTF8 = StandardCharsets.UTF_8; + protected final String EMPTY_BODY = ""; + protected final static String TEST_DOC_LINE = "This is a sample to prove that http downloads and uploads work. It doesn't really matter what's in here, we mainly just need to verify the downloads and uploads work."; + protected final static String TEST_DOC_SHA256 = "C7FDB5314B9742467B16BD5EA2F8012190B5E2C44A005F7984F89AAB58219534"; + + protected class TestHttpResponse { + int statusCode = -1; + int blockType = -1; + List headers = new ArrayList<>(); + ByteBuffer bodyBuffer = ByteBuffer.wrap(new byte[16 * 1024 * 1024]); // Allow up to 16 MB Responses + int onCompleteErrorCode = -1; + + public String getBody() { + bodyBuffer.flip(); + return UTF8.decode(bodyBuffer).toString(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Status: " + statusCode); + int i = 0; + for (HttpHeader h : headers) { + builder.append("\nHeader[" + i + "]: " + h.toString()); + } + + builder.append("\nBody:\n"); + builder.append(getBody()); + + return builder.toString(); + } + } + + protected boolean shouldRetry(TestHttpResponse response) { + // Retry if we couldn't connect or if we got 503 response + if (response.onCompleteErrorCode != CRT.AWS_CRT_SUCCESS || response.statusCode == 503) { + return true; + } + return false; + } + + public static String byteArrayToHex(byte[] input) { + StringBuilder output = new StringBuilder(input.length * 2); + for (byte b : input) { + output.append(String.format("%02X", b)); + } + return output.toString(); + } + + public String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmException { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + digest.update(bodyBuffer); + return byteArrayToHex(digest.digest()); + } + + protected HttpClientConnectionManager createHTTP2ConnectionPoolManager(URI uri) { + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient() + .withAlpnList("h2;http/1.1"); + TlsContext tlsContext = createHttpClientTlsContext(tlsContextOptions)) { + + HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() + .withClientBootstrap(bootstrap).withSocketOptions(sockOpts).withTlsContext(tlsContext).withUri(uri); + + return HttpClientConnectionManager.create(options); + } + } + + protected HttpClientConnectionManager createHTTP1ConnectionPoolManager(URI uri) { + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContext tlsContext = createHttpClientTlsContext()) { + + HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() + .withClientBootstrap(bootstrap).withSocketOptions(sockOpts).withTlsContext(tlsContext).withUri(uri); + + return HttpClientConnectionManager.create(options); + } + } + + public HttpClientConnectionManager createConnectionPoolManager(URI uri, + HttpClientConnection.AwsHTTPProtocolVersion expectedVersion) { + if (expectedVersion == HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2) { + return createHTTP2ConnectionPoolManager(uri); + } else { + return createHTTP1ConnectionPoolManager(uri); + } + + } + + public TestHttpResponse getResponse(URI uri, HttpRequest request, byte[] chunkedData, + HttpClientConnection.AwsHTTPProtocolVersion expectedVersion) throws Exception { + boolean actuallyConnected = false; + + final CompletableFuture reqCompleted = new CompletableFuture<>(); + + final TestHttpResponse response = new TestHttpResponse(); + + CompletableFuture shutdownComplete = null; + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, expectedVersion)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { + actuallyConnected = true; + HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + response.statusCode = responseStatusCode; + Assert.assertEquals(responseStatusCode, stream.getResponseStatusCode()); + response.headers.addAll(Arrays.asList(nextHeaders)); + } + + @Override + public void onResponseHeadersDone(HttpStream stream, int blockType) { + response.blockType = blockType; + } + + @Override + public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + response.bodyBuffer.put(bodyBytesIn); + int amountRead = bodyBytesIn.length; + + // Slide the window open by the number of bytes just read + return amountRead; + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + response.onCompleteErrorCode = errorCode; + reqCompleted.complete(null); + stream.close(); + } + }; + + HttpStream stream = conn.makeRequest(request, streamHandler); + stream.activate(); + + if (chunkedData != null) { + stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); + } + // Give the request up to 60 seconds to complete, otherwise throw a + // TimeoutException + reqCompleted.get(60, TimeUnit.SECONDS); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + + return response; + + } + +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java index 710425db6..9e9d52363 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java @@ -8,9 +8,7 @@ import static software.amazon.awssdk.crt.utils.ByteBufferUtils.transferData; import org.junit.Assert; -import org.junit.Assume; import org.junit.Test; -import software.amazon.awssdk.crt.CRT; import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; @@ -25,174 +23,26 @@ import software.amazon.awssdk.crt.io.HostResolver; import software.amazon.awssdk.crt.io.SocketOptions; import software.amazon.awssdk.crt.io.TlsContext; -import software.amazon.awssdk.crt.io.TlsContextOptions; import java.net.URI; import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -public class HttpRequestResponseTest extends HttpClientTestFixture { - private final static Charset UTF8 = StandardCharsets.UTF_8; - private final String EMPTY_BODY = ""; - private final static String TEST_DOC_LINE = "This is a sample to prove that http downloads and uploads work. It doesn't really matter what's in here, we mainly just need to verify the downloads and uploads work."; - private final static String TEST_DOC_SHA256 = "C7FDB5314B9742467B16BD5EA2F8012190B5E2C44A005F7984F89AAB58219534"; - - private class TestHttpResponse { - int statusCode = -1; - int blockType = -1; - List headers = new ArrayList<>(); - ByteBuffer bodyBuffer = ByteBuffer.wrap(new byte[16*1024*1024]); // Allow up to 16 MB Responses - int onCompleteErrorCode = -1; - - public String getBody() { - bodyBuffer.flip(); - return UTF8.decode(bodyBuffer).toString(); - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Status: " + statusCode); - int i = 0; - for (HttpHeader h: headers) { - builder.append("\nHeader[" + i + "]: " + h.toString()); - } - - builder.append("\nBody:\n"); - builder.append(getBody()); - - return builder.toString(); - } - } - - public static String byteArrayToHex(byte[] input) { - StringBuilder output = new StringBuilder(input.length * 2); - for (byte b: input) { - output.append(String.format("%02X", b)); - } - return output.toString(); - } - - private String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmException { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - digest.update(bodyBuffer); - return byteArrayToHex(digest.digest()); - } - - private HttpClientConnectionManager createConnectionPoolManager(URI uri) { - try(EventLoopGroup eventLoopGroup = new EventLoopGroup(1); - HostResolver resolver = new HostResolver(eventLoopGroup); - ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); - SocketOptions sockOpts = new SocketOptions(); - TlsContext tlsContext = createHttpClientTlsContext()) { - - HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() - .withClientBootstrap(bootstrap) - .withSocketOptions(sockOpts) - .withTlsContext(tlsContext) - .withUri(uri); - - return HttpClientConnectionManager.create(options); - } - } +public class HttpRequestResponseTest extends HttpRequestResponseFixture { - public TestHttpResponse getResponse(URI uri, HttpRequest request, byte[] chunkedData) throws Exception { - boolean actuallyConnected = false; - - final CompletableFuture reqCompleted = new CompletableFuture<>(); - - final TestHttpResponse response = new TestHttpResponse(); - - CompletableFuture shutdownComplete = null; - try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri)) { - shutdownComplete = connPool.getShutdownCompleteFuture(); - try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { - actuallyConnected = true; - HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { - @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { - response.statusCode = responseStatusCode; - Assert.assertEquals(responseStatusCode, stream.getResponseStatusCode()); - response.headers.addAll(Arrays.asList(nextHeaders)); - } - - @Override - public void onResponseHeadersDone(HttpStream stream, int blockType) { - response.blockType = blockType; - } - - @Override - public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { - response.bodyBuffer.put(bodyBytesIn); - int amountRead = bodyBytesIn.length; - - // Slide the window open by the number of bytes just read - return amountRead; - } - - @Override - public void onResponseComplete(HttpStream stream, int errorCode) { - response.onCompleteErrorCode = errorCode; - reqCompleted.complete(null); - stream.close(); - } - }; - - HttpStream stream = conn.makeRequest(request, streamHandler); - stream.activate(); - - if (chunkedData != null) { - stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); - } - // Give the request up to 60 seconds to complete, otherwise throw a TimeoutException - reqCompleted.get(60, TimeUnit.SECONDS); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - Assert.assertTrue(actuallyConnected); - - shutdownComplete.get(); - - CrtResource.waitForNoResources(); - - return response; - } - - private boolean shouldRetry(TestHttpResponse response) { - // Retry if we couldn't connect or if we got 503 response - if (response.onCompleteErrorCode != CRT.AWS_CRT_SUCCESS || response.statusCode == 503) { - return true; - } - return false; - } - - public TestHttpResponse testRequest(String method, String endpoint, String path, String requestBody, boolean useChunkedEncoding, int expectedStatus) throws Exception { + public TestHttpResponse testRequest(String method, String endpoint, String path, String requestBody, + boolean useChunkedEncoding, int expectedStatus) throws Exception { URI uri = new URI(endpoint); HttpHeader[] requestHeaders = null; if (!useChunkedEncoding) { - requestHeaders = - new HttpHeader[]{ - new HttpHeader("Host", uri.getHost()), - new HttpHeader("Content-Length", Integer.toString(requestBody.getBytes(UTF8).length)) - }; + requestHeaders = new HttpHeader[] { new HttpHeader("Host", uri.getHost()), + new HttpHeader("Content-Length", Integer.toString(requestBody.getBytes(UTF8).length)) }; } else { - requestHeaders = - new HttpHeader[]{ - new HttpHeader("Host", uri.getHost()), - new HttpHeader("Transfer-Encoding", "chunked") - }; + requestHeaders = new HttpHeader[] { new HttpHeader("Host", uri.getHost()), + new HttpHeader("Transfer-Encoding", "chunked") }; } HttpRequestBodyStream bodyStream = null; @@ -231,12 +81,13 @@ public boolean resetPosition() { response = null; try { if (useChunkedEncoding) { - response = getResponse(uri, request, requestBody.getBytes(UTF8)); + response = getResponse(uri, request, requestBody.getBytes(UTF8), + HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1); } else { - response = getResponse(uri, request, null); + response = getResponse(uri, request, null, HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1); } } catch (Exception ex) { - //do nothing just let it retry + // do nothing just let it retry } } while ((response == null || shouldRetry(response)) && numAttempts < 3); @@ -245,7 +96,7 @@ public boolean resetPosition() { boolean hasContentLengthHeader = false; - for (HttpHeader h: response.headers) { + for (HttpHeader h : response.headers) { if (h.getName().equals("Content-Length")) { hasContentLengthHeader = true; } @@ -259,62 +110,59 @@ public boolean resetPosition() { return response; } - @Test public void testHttpDelete() throws Exception { skipIfNetworkUnavailable(); - testRequest("DELETE", "https://httpbin.org", "/delete", EMPTY_BODY, false,200); + testRequest("DELETE", "https://httpbin.org", "/delete", EMPTY_BODY, false, 200); testRequest("DELETE", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); testRequest("DELETE", "https://httpbin.org", "/post", EMPTY_BODY, false, 405); testRequest("DELETE", "https://httpbin.org", "/put", EMPTY_BODY, false, 405); } - @Test public void testHttpGet() throws Exception { skipIfNetworkUnavailable(); - testRequest("GET", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); - testRequest("GET", "https://httpbin.org", "/get", EMPTY_BODY, false,200); - testRequest("GET", "https://httpbin.org", "/post", EMPTY_BODY, false,405); - testRequest("GET", "https://httpbin.org", "/put", EMPTY_BODY, false,405); + testRequest("GET", "https://httpbin.org", "/delete", EMPTY_BODY, false, 405); + testRequest("GET", "https://httpbin.org", "/get", EMPTY_BODY, false, 200); + testRequest("GET", "https://httpbin.org", "/post", EMPTY_BODY, false, 405); + testRequest("GET", "https://httpbin.org", "/put", EMPTY_BODY, false, 405); } @Test public void testHttpPost() throws Exception { skipIfNetworkUnavailable(); - testRequest("POST", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); + testRequest("POST", "https://httpbin.org", "/delete", EMPTY_BODY, false, 405); testRequest("POST", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); - testRequest("POST", "https://httpbin.org", "/post", EMPTY_BODY, false,200); - testRequest("POST", "https://httpbin.org", "/put", EMPTY_BODY, false,405); + testRequest("POST", "https://httpbin.org", "/post", EMPTY_BODY, false, 200); + testRequest("POST", "https://httpbin.org", "/put", EMPTY_BODY, false, 405); } @Test public void testHttpPut() throws Exception { skipIfNetworkUnavailable(); - testRequest("PUT", "https://httpbin.org", "/delete", EMPTY_BODY, false,405); + testRequest("PUT", "https://httpbin.org", "/delete", EMPTY_BODY, false, 405); testRequest("PUT", "https://httpbin.org", "/get", EMPTY_BODY, false, 405); testRequest("PUT", "https://httpbin.org", "/post", EMPTY_BODY, false, 405); - testRequest("PUT", "https://httpbin.org", "/put", EMPTY_BODY, false,200); + testRequest("PUT", "https://httpbin.org", "/put", EMPTY_BODY, false, 200); } @Test public void testHttpResponseStatusCodes() throws Exception { skipIfNetworkUnavailable(); - testRequest("GET", "https://httpbin.org", "/status/200", EMPTY_BODY, false,200); + testRequest("GET", "https://httpbin.org", "/status/200", EMPTY_BODY, false, 200); testRequest("GET", "https://httpbin.org", "/status/300", EMPTY_BODY, false, 300); testRequest("GET", "https://httpbin.org", "/status/400", EMPTY_BODY, false, 400); - testRequest("GET", "https://httpbin.org", "/status/500", EMPTY_BODY, false,500); + testRequest("GET", "https://httpbin.org", "/status/500", EMPTY_BODY, false, 500); } - - @Test public void testHttpDownload() throws Exception { skipIfNetworkUnavailable(); - TestHttpResponse response = testRequest("GET", "https://aws-crt-test-stuff.s3.amazonaws.com", "/http_test_doc.txt", EMPTY_BODY, false,200); + TestHttpResponse response = testRequest("GET", "https://aws-crt-test-stuff.s3.amazonaws.com", + "/http_test_doc.txt", EMPTY_BODY, false, 200); ByteBuffer body = response.bodyBuffer; - body.flip(); //Flip from Write mode to Read mode + body.flip(); // Flip from Write mode to Read mode Assert.assertEquals(TEST_DOC_SHA256, calculateBodyHash(body)); } @@ -326,15 +174,15 @@ public void testHttpDownload() throws Exception { * @return */ private String extractValueFromJson(String input) { - return input.trim() // Remove spaces from front and back - .replaceAll(",$", "") // Remove comma if it's the last character - .replaceAll("^\"|\"$", ""); // Remove quotes from front and back + return input.trim() // Remove spaces from front and back + .replaceAll(",$", "") // Remove comma if it's the last character + .replaceAll("^\"|\"$", ""); // Remove quotes from front and back } private void testHttpUpload(boolean chunked) throws Exception { skipIfNetworkUnavailable(); String bodyToSend = TEST_DOC_LINE; - TestHttpResponse response = testRequest("PUT", "https://httpbin.org", "/anything", bodyToSend, chunked,200); + TestHttpResponse response = testRequest("PUT", "https://httpbin.org", "/anything", bodyToSend, chunked, 200); // Get the Body bytes that were echoed back to us String body = response.getBody(); @@ -342,25 +190,16 @@ private void testHttpUpload(boolean chunked) throws Exception { /** * Example Json Response Body from httpbin.org: * - * { - * "args": {}, - * "data": "This is a sample to prove that http downloads and uploads work. It doesn't really matter what's in here, we mainly just need to verify the downloads and uploads work.", - * "files": {}, - * "form": {}, - * "headers": { - * "Content-Length": "166", - * "Host": "httpbin.org" - * }, - * "json": null, - * "method": "PUT", - * "origin": "1.2.3.4, 5.6.7.8", - * "url": "https://httpbin.org/anything" - * } + * { "args": {}, "data": "This is a sample to prove that http downloads and + * uploads work. It doesn't really matter what's in here, we mainly just need to + * verify the downloads and uploads work.", "files": {}, "form": {}, "headers": + * { "Content-Length": "166", "Host": "httpbin.org" }, "json": null, "method": + * "PUT", "origin": "1.2.3.4, 5.6.7.8", "url": "https://httpbin.org/anything" } * */ String echoedBody = null; - for (String line: body.split("\n")) { + for (String line : body.split("\n")) { String[] keyAndValue = line.split(":", 2); // Found JSON Key/Value Pair @@ -397,20 +236,19 @@ public void testHttpRequestUnActivated() throws Exception { URI uri = new URI("https://httpbin.org"); - HttpHeader[] requestHeaders = - new HttpHeader[]{ - new HttpHeader("Host", uri.getHost()) - }; + HttpHeader[] requestHeaders = new HttpHeader[] { new HttpHeader("Host", uri.getHost()) }; HttpRequest request = new HttpRequest("GET", "/get", requestHeaders, null); CompletableFuture shutdownComplete = null; - try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri)) { + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, + HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1)) { shutdownComplete = connPool.getShutdownCompleteFuture(); try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { // do nothing } @@ -421,13 +259,13 @@ public void onResponseHeadersDone(HttpStream stream, int blockType) { @Override public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { - //do nothing + // do nothing return bodyBytesIn.length; } @Override public void onResponseComplete(HttpStream stream, int errorCode) { - //do nothing. + // do nothing. } }; From 82e9810c7758936b364d6e7b1742aee802405c0a Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Oct 2021 10:01:34 -0700 Subject: [PATCH 005/142] comments --- .../awssdk/crt/test/CrtTestFixture.java | 1 - .../crt/test/Http2RequestResponseTest.java | 2 +- .../crt/test/HttpRequestResponseTest.java | 27 +++++++++++-------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java index e7c19aa4b..6a6691993 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java @@ -15,7 +15,6 @@ import software.amazon.awssdk.crt.io.HostResolver; import software.amazon.awssdk.crt.io.TlsContext; import software.amazon.awssdk.crt.io.TlsContextOptions; -import software.amazon.awssdk.crt.test.CrtTestContext; import software.amazon.awssdk.crt.auth.credentials.Credentials; import software.amazon.awssdk.crt.auth.credentials.DefaultChainCredentialsProvider; import java.io.File; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index 2be8ea7c6..6149a39cd 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -139,4 +139,4 @@ public void testHttpDownload() throws Exception { Assert.assertEquals(TEST_DOC_SHA256, calculateBodyHash(body)); } -} \ No newline at end of file +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java index 9e9d52363..b7fd3afd7 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java @@ -12,17 +12,11 @@ import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; -import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; import software.amazon.awssdk.crt.http.HttpRequestBodyStream; import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpStream; -import software.amazon.awssdk.crt.io.ClientBootstrap; -import software.amazon.awssdk.crt.io.EventLoopGroup; -import software.amazon.awssdk.crt.io.HostResolver; -import software.amazon.awssdk.crt.io.SocketOptions; -import software.amazon.awssdk.crt.io.TlsContext; import java.net.URI; import java.nio.ByteBuffer; @@ -190,11 +184,22 @@ private void testHttpUpload(boolean chunked) throws Exception { /** * Example Json Response Body from httpbin.org: * - * { "args": {}, "data": "This is a sample to prove that http downloads and - * uploads work. It doesn't really matter what's in here, we mainly just need to - * verify the downloads and uploads work.", "files": {}, "form": {}, "headers": - * { "Content-Length": "166", "Host": "httpbin.org" }, "json": null, "method": - * "PUT", "origin": "1.2.3.4, 5.6.7.8", "url": "https://httpbin.org/anything" } + * { + * "args": {}, + * "data": "This is a sample to prove that http downloads and + * uploads work. It doesn't really matter what's in here, we mainly just need to + * verify the downloads and uploads work.", + * "files": {}, + * "form": {}, + * "headers": { + * "Content-Length": "166", + * "Host": "httpbin.org" + * }, + * "json": null, + * "method": "PUT", + * "origin": "1.2.3.4, 5.6.7.8", + * "url": "https://httpbin.org/anything" + * } * */ From addd5315641c50e85a48ff3c9e1b1f6e89c1151e Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Oct 2021 16:18:23 -0700 Subject: [PATCH 006/142] WIP ping --- .../crt/http/Http2ConnectionSetting.java | 6 +- .../awssdk/crt/http/HttpClientConnection.java | 32 +++++++- .../crt/test/Http2ClientConnectionTest.java | 79 +++++++++++++++++++ .../crt/test/HttpClientTestFixture.java | 50 ++++++++++++ .../crt/test/HttpRequestResponseFixture.java | 47 ----------- 5 files changed, 161 insertions(+), 53 deletions(-) create mode 100644 src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index a2b6ec683..13fbdd7a3 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -2,11 +2,7 @@ public class Http2ConnectionSetting { public enum Http2ConnectionSettingID { - HEADER_TABLE_SIZE(1), - ENABLE_PUSH(2), - MAX_CONCURRENT_STREAMS(3), - INITIAL_WINDOW_SIZE(4), - MAX_FRAME_SIZE(5), + HEADER_TABLE_SIZE(1), ENABLE_PUSH(2), MAX_CONCURRENT_STREAMS(3), INITIAL_WINDOW_SIZE(4), MAX_FRAME_SIZE(5), MAX_HEADER_LIST_SIZE(6); private int settingID; diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index 614a718d7..d356ea83a 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -6,6 +6,8 @@ package software.amazon.awssdk.crt.http; import java.util.concurrent.CompletableFuture; +import java.util.Map; +import java.util.HashMap; import software.amazon.awssdk.crt.CRT; import software.amazon.awssdk.crt.CrtResource; @@ -32,11 +34,29 @@ public enum AwsHTTPProtocolVersion { HTTP_2(3); private int protocolVersion; + private static Map enumMapping = buildEnumMapping(); AwsHTTPProtocolVersion(int value) { protocolVersion = value; } + public static AwsHTTPProtocolVersion getEnumValueFromInteger(int value) { + AwsHTTPProtocolVersion enumValue = enumMapping.get(value); + if (enumValue != null) { + return enumValue; + } + + throw new RuntimeException("Illegal signature type value in signing configuration"); + } + private static Map buildEnumMapping() { + Map enumMapping = new HashMap(); + enumMapping.put(HTTP_1_0.getValue(), HTTP_1_0); + enumMapping.put(HTTP_1_1.getValue(), HTTP_1_1); + enumMapping.put(HTTP_2.getValue(), HTTP_2); + + return enumMapping; + } + public int getValue() { return protocolVersion; } @@ -96,6 +116,10 @@ public void shutdown() { httpClientConnectionShutdown(getNativeHandle()); } + public AwsHTTPProtocolVersion getVersion() { + short version = httpClientConnectionGetVersion(getNativeHandle()); + return AwsHTTPProtocolVersion.getEnumValueFromInteger((int)version); + }; /** Called from Native when a new connection is acquired **/ private static void onConnectionAcquired(CompletableFuture acquireFuture, long nativeConnectionBinding, int errorCode) { @@ -105,7 +129,12 @@ private static void onConnectionAcquired(CompletableFuture } HttpClientConnection conn = new HttpClientConnection(nativeConnectionBinding); - acquireFuture.complete(conn); + if(conn.getVersion() == AwsHTTPProtocolVersion.HTTP_2) { + HttpClientConnection h2Conn = new Http2ClientConnection(nativeConnectionBinding); + acquireFuture.complete(h2Conn); + } else{ + acquireFuture.complete(conn); + } } /******************************************************************************* @@ -119,4 +148,5 @@ private static native HttpStream httpClientConnectionMakeRequest(long connection private static native void httpClientConnectionShutdown(long connectionBinding) throws CrtRuntimeException; private static native void httpClientConnectionReleaseManaged(long connectionBinding) throws CrtRuntimeException; + private static native short httpClientConnectionGetVersion(long connectionBinding) throws CrtRuntimeException; } diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java new file mode 100644 index 000000000..21b96c469 --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -0,0 +1,79 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package software.amazon.awssdk.crt.test; + +import org.junit.Test; +import org.junit.Assert; + +import java.net.URI; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.Http2ClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.Log; + +public class Http2ClientConnectionTest extends HttpClientTestFixture { + protected final static String HOST = "https://httpbin.org"; + protected final static HttpClientConnection.AwsHTTPProtocolVersion EXPECTED_VERSION = HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2; + + @Test + public void testHttp2ConnectionGetVersion() throws Exception { + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean actuallyConnected = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { + actuallyConnected = true; + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } + + @Test + public void testHttp2ConnectionPing() throws Exception { + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean actuallyConnected = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, + TimeUnit.SECONDS);) { + Log.initLoggingToFile(Log.LogLevel.Trace, "tracelog.txt"); + actuallyConnected = true; + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + long time = conn.sendPing(null).get(5, TimeUnit.SECONDS); + Assert.assertNotNull(time); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } + +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java index 6aa0d3063..c260ea242 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java @@ -5,10 +5,50 @@ package software.amazon.awssdk.crt.test; +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; import software.amazon.awssdk.crt.io.TlsContext; import software.amazon.awssdk.crt.io.TlsContextOptions; +import java.net.URI; + public class HttpClientTestFixture extends CrtTestFixture { + + private HttpClientConnectionManager createHTTP2ConnectionPoolManager(URI uri) { + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient() + .withAlpnList("h2;http/1.1"); + TlsContext tlsContext = createHttpClientTlsContext(tlsContextOptions)) { + + HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() + .withClientBootstrap(bootstrap).withSocketOptions(sockOpts).withTlsContext(tlsContext).withUri(uri); + + return HttpClientConnectionManager.create(options); + } + } + + private HttpClientConnectionManager createHTTP1ConnectionPoolManager(URI uri) { + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContext tlsContext = createHttpClientTlsContext()) { + + HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() + .withClientBootstrap(bootstrap).withSocketOptions(sockOpts).withTlsContext(tlsContext).withUri(uri); + + return HttpClientConnectionManager.create(options); + } + } + public TlsContext createHttpClientTlsContext() { return createTlsContextOptions(getContext().trustStore); } @@ -16,4 +56,14 @@ public TlsContext createHttpClientTlsContext() { public TlsContext createHttpClientTlsContext(TlsContextOptions tlsOpts) { return new TlsContext(configureTlsContextOptions(tlsOpts, getContext().trustStore)); } + + public HttpClientConnectionManager createConnectionPoolManager(URI uri, + HttpClientConnection.AwsHTTPProtocolVersion expectedVersion) { + if (expectedVersion == HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2) { + return createHTTP2ConnectionPoolManager(uri); + } else { + return createHTTP1ConnectionPoolManager(uri); + } + + } } diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java index 00e8d3445..27857c6a6 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -9,17 +9,10 @@ import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; -import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpStream; -import software.amazon.awssdk.crt.io.ClientBootstrap; -import software.amazon.awssdk.crt.io.EventLoopGroup; -import software.amazon.awssdk.crt.io.HostResolver; -import software.amazon.awssdk.crt.io.SocketOptions; -import software.amazon.awssdk.crt.io.TlsContext; -import software.amazon.awssdk.crt.io.TlsContextOptions; import java.net.URI; import java.nio.ByteBuffer; @@ -90,46 +83,6 @@ public String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmExc return byteArrayToHex(digest.digest()); } - protected HttpClientConnectionManager createHTTP2ConnectionPoolManager(URI uri) { - try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); - HostResolver resolver = new HostResolver(eventLoopGroup); - ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); - SocketOptions sockOpts = new SocketOptions(); - TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient() - .withAlpnList("h2;http/1.1"); - TlsContext tlsContext = createHttpClientTlsContext(tlsContextOptions)) { - - HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() - .withClientBootstrap(bootstrap).withSocketOptions(sockOpts).withTlsContext(tlsContext).withUri(uri); - - return HttpClientConnectionManager.create(options); - } - } - - protected HttpClientConnectionManager createHTTP1ConnectionPoolManager(URI uri) { - try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); - HostResolver resolver = new HostResolver(eventLoopGroup); - ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); - SocketOptions sockOpts = new SocketOptions(); - TlsContext tlsContext = createHttpClientTlsContext()) { - - HttpClientConnectionManagerOptions options = new HttpClientConnectionManagerOptions() - .withClientBootstrap(bootstrap).withSocketOptions(sockOpts).withTlsContext(tlsContext).withUri(uri); - - return HttpClientConnectionManager.create(options); - } - } - - public HttpClientConnectionManager createConnectionPoolManager(URI uri, - HttpClientConnection.AwsHTTPProtocolVersion expectedVersion) { - if (expectedVersion == HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2) { - return createHTTP2ConnectionPoolManager(uri); - } else { - return createHTTP1ConnectionPoolManager(uri); - } - - } - public TestHttpResponse getResponse(URI uri, HttpRequest request, byte[] chunkedData, HttpClientConnection.AwsHTTPProtocolVersion expectedVersion) throws Exception { boolean actuallyConnected = false; From 289cb45ccdb8907ca11a7ee86585830eeae1e7c4 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Oct 2021 16:18:47 -0700 Subject: [PATCH 007/142] use future --- .../crt/http/Http2ClientConnection.java | 119 +++++++++------- src/native/http_request_response.c | 127 ++++++++++++++++++ 2 files changed, 200 insertions(+), 46 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index b3e14f1e8..14838d9d7 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -5,6 +5,9 @@ package software.amazon.awssdk.crt.http; +import software.amazon.awssdk.crt.AsyncCallback; +import software.amazon.awssdk.crt.CrtRuntimeException; + import java.util.concurrent.CompletableFuture; /** @@ -18,22 +21,16 @@ * threads. */ public class Http2ClientConnection extends HttpClientConnection { + private AsyncCallback pingAck; - /* Error codes that may be present in HTTP/2 RST_STREAM and GOAWAY frames (RFC-7540 7). */ + /* + * Error codes that may be present in HTTP/2 RST_STREAM and GOAWAY frames + * (RFC-7540 7). + */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), - INTERNAL_ERROR(2), - FLOW_CONTROL_ERROR(3), - SETTINGS_TIMEOUT(4), - STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), - REFUSED_STREAM(7), - CANCEL(8), - COMPRESSION_ERROR(9), - CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), - INADEQUATE_SECURITY(12), - HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); private int errorCode; @@ -46,17 +43,19 @@ public int getValue() { } } - private Http2ClientConnection(long connectionBinding) { + public Http2ClientConnection(long connectionBinding) { super(connectionBinding); } + /** - * Send a SETTINGS frame. - * SETTINGS will be applied locally when SETTINGS ACK is received from peer. + * Send a SETTINGS frame. SETTINGS will be applied locally when SETTINGS ACK is + * received from peer. * - * @param settings The array of settings to change. Note: each setting has its boundary. + * @param settings The array of settings to change. Note: each setting has its + * boundary. * - * @return When this future completes without exception, the peer has acknowledged the settings and - * the change has been applied. + * @return When this future completes without exception, the peer has + * acknowledged the settings and the change has been applied. */ public CompletableFuture changeSettings(final Http2ConnectionSetting settings[]) { CompletableFuture completionFuture = new CompletableFuture<>(); @@ -64,16 +63,19 @@ public CompletableFuture changeSettings(final Http2ConnectionSetting setti } /** - * Send a PING frame - * Round-trip-time is calculated when PING ACK is received from peer. + * Send a PING frame Round-trip-time is calculated when PING ACK is received + * from peer. * * @param pingData Optional 8 Bytes data with the PING frame * - * @return When this future completes without exception, the peer has acknowledged the PING and - * future will be completed with the round trip time in nano seconds for the connection. + * @return When this future completes without exception, the peer has + * acknowledged the PING and future will be completed with the round + * trip time in nano seconds for the connection. */ public CompletableFuture sendPing(final byte[] pingData) { CompletableFuture completionFuture = new CompletableFuture<>(); + pingAck = AsyncCallback.wrapFuture(completionFuture, null); + http2ClientConnectionSendPing(getNativeHandle(), completionFuture, pingData); return completionFuture; } @@ -89,44 +91,69 @@ public CompletableFuture sendPing(final byte[] pingData) { * * The other end may not receive the goaway, if the connection already closed. * - * @param Http2ErrorCode The HTTP/2 error code (RFC-7540 section 7) to send. - * `enum Http2ErrorCode` lists official codes. - * @param allowMoreStreams If true, new peer-initiated streams will continue - * to be acknowledged and the GOAWAY's Last-Stream-ID will be set to a max value. - * If false, new peer-initiated streams will be ignored and the GOAWAY's - * Last-Stream-ID will be set to the latest acknowledged stream. - * @param debugData Optional debug data to send. Size must not exceed 16KB. + * @param Http2ErrorCode The HTTP/2 error code (RFC-7540 section 7) to send. + * `enum Http2ErrorCode` lists official codes. + * @param allowMoreStreams If true, new peer-initiated streams will continue to + * be acknowledged and the GOAWAY's Last-Stream-ID will + * be set to a max value. If false, new peer-initiated + * streams will be ignored and the GOAWAY's + * Last-Stream-ID will be set to the latest acknowledged + * stream. + * @param debugData Optional debug data to send. Size must not exceed + * 16KB. */ - public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams, final byte[] debugData) { + public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams, + final byte[] debugData) { } /** * Increment the connection's flow-control window to keep data flowing. * - * If the connection was created with `manualWindowManagement` set true, - * the flow-control window of the connection will shrink as body data is received for all the streams created on it. - * (headers, padding, and other metadata do not affect the window). - * The initial connection flow-control window is 65,535. - * Once the connection's flow-control window reaches to 0, all the streams on the connection stop receiving any further - * data. + * If the connection was created with `manualWindowManagement` set true, the + * flow-control window of the connection will shrink as body data is received + * for all the streams created on it. (headers, padding, and other metadata do + * not affect the window). The initial connection flow-control window is 65,535. + * Once the connection's flow-control window reaches to 0, all the streams on + * the connection stop receiving any further data. * - * If `manualWindowManagement` is false, this call will have no effect. - * The connection maintains its flow-control windows such that - * no back-pressure is applied and data arrives as fast as possible. + * If `manualWindowManagement` is false, this call will have no effect. The + * connection maintains its flow-control windows such that no back-pressure is + * applied and data arrives as fast as possible. * * If you are not connected, this call will have no effect. * - * Crashes when the connection is not http2 connection. - * The limit of the Maximum Size is 2**31 - 1. If the increment size cause the connection flow window exceeds the - * Maximum size, this call will result in the connection lost. + * Crashes when the connection is not http2 connection. The limit of the Maximum + * Size is 2**31 - 1. If the increment size cause the connection flow window + * exceeds the Maximum size, this call will result in the connection lost. * - * @param increment_size The size to increment for the connection's flow control window + * @param increment_size The size to increment for the connection's flow control + * window */ - public void updateConnectionWindow(long incrementSize){ + public void updateConnectionWindow(long incrementSize) { } /** * @TODO: bindings for getters of local/remote setting and goaway. */ + /******************************************************************************* + * Native methods + ******************************************************************************/ + /* + * @TODO: Do we need the change settings? We should have the initial settings + * expose from connection manager... Any thing other than marshall the list to + * byte for the settings array. + */ + // private static native void http2ClientConnectionChangeSettings(long + // connectionBinding, + // CompletableFuture future /* settings */ ) throws CrtRuntimeException; + + private static native void http2ClientConnectionSendPing(long connectionBinding, CompletableFuture future, + byte[] pingData) throws CrtRuntimeException; + + private static native void http2ClientConnectionSendGoAway(long connectionBinding, long h2ErrorCode, + boolean allowMoreStreams, byte[] debugData) throws CrtRuntimeException; + + private static native void http2ClientConnectionUpdateConnectionWindow(long connectionBinding, long incrementSize) + throws CrtRuntimeException; } diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index f9e21ea66..861663c54 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -559,6 +559,133 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection aws_http_connection_close(native_conn); } +JNIEXPORT jshort JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection_httpClientConnectionGetVersion( + JNIEnv *env, + jclass jni_class, + jlong jni_connection) { + + (void)jni_class; + struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; + struct aws_http_connection *native_conn = connection_binding->connection; + + if (!native_conn) { + aws_jni_throw_runtime_exception(env, "HttpClientConnection.getVersion: Invalid aws_http_connection"); + return 0; + } + return (jshort)aws_http_connection_get_version(native_conn); +} + +struct s_aws_http2_ping_callback_data { + JavaVM *jvm; + jobject java_ping_result_future; +}; + +static void s_cleanup_ping_callback_data(struct s_aws_http2_ping_callback_data *callback_data) { + + JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); + + (*env)->DeleteGlobalRef(env, callback_data->java_ping_result_future); + + aws_mem_release(aws_jni_get_allocator(), callback_data); +} + +static void s_on_ping_completed( + struct aws_http_connection *http2_connection, + uint64_t round_trip_time_ns, + int error_code, + void *user_data) { + (void)http2_connection; + struct s_aws_http2_ping_callback_data *callback_data = user_data; + JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); + AWS_LOGF_DEBUG(AWS_LS_HTTP_CONNECTION, "Ping 2"); + if (error_code) { + jint jni_error_code = error_code; + struct aws_byte_cursor error_cursor = aws_byte_cursor_from_c_str(aws_error_name(error_code)); + jstring jni_error_string = aws_jni_string_from_cursor(env, &error_cursor); + AWS_FATAL_ASSERT(jni_error_string); + + jobject crt_exception = (*env)->NewObject( + env, + crt_runtime_exception_properties.crt_runtime_exception_class, + crt_runtime_exception_properties.constructor_method_id, + jni_error_code, + jni_error_string); + AWS_FATAL_ASSERT(crt_exception); + (*env)->CallBooleanMethod( + env, + callback_data->java_ping_result_future, + completable_future_properties.complete_exceptionally_method_id, + crt_exception); + aws_jni_check_and_clear_exception(env); + (*env)->DeleteLocalRef(env, jni_error_string); + (*env)->DeleteLocalRef(env, crt_exception); + goto done; + } + AWS_LOGF_DEBUG(AWS_LS_HTTP_CONNECTION, "Ping 3"); + (*env)->CallBooleanMethod( + env, + callback_data->java_ping_result_future, + completable_future_properties.complete_method_id, + (jlong)round_trip_time_ns); + AWS_LOGF_DEBUG(AWS_LS_HTTP_CONNECTION, "Ping 4"); + +done: + s_cleanup_ping_callback_data(callback_data); +} + +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionSendPing( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jobject java_ping_result_future, + jbyteArray ping_data) { + + (void)jni_class; + struct aws_allocator *allocator = aws_jni_get_allocator(); + struct s_aws_http2_ping_callback_data *callback_data = + aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_ping_callback_data)); + if (callback_data == NULL) { + aws_jni_throw_runtime_exception(env, "Failed to allocated ping callback data"); + return; + } + jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); + AWS_FATAL_ASSERT(jvmresult == 0); + + struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; + struct aws_http_connection *native_conn = connection_binding->connection; + + if (!native_conn) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid aws_http_connection"); + goto error; + } + + callback_data->java_ping_result_future = (*env)->NewGlobalRef(env, java_ping_result_future); + if (callback_data->java_ping_result_future == NULL) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionSendPing: failed to obtain ref to future"); + goto error; + } + if (ping_data) { + struct aws_byte_cursor ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); + if (aws_http2_connection_ping(native_conn, &ping_cur, s_on_ping_completed, callback_data)) { + aws_jni_throw_runtime_exception(env, "Failed to send ping"); + goto error; + } + + aws_jni_byte_cursor_from_jbyteArray_release(env, ping_data, ping_cur); + } else { + if (aws_http2_connection_ping(native_conn, NULL, s_on_ping_completed, callback_data)) { + aws_jni_throw_runtime_exception(env, "Failed to send ping"); + goto error; + } + } + return; +error: + s_cleanup_ping_callback_data(callback_data); + return; +} + #if UINTPTR_MAX == 0xffffffff # if defined(_MSC_VER) # pragma warning(pop) From 123249b3aac032833a47d5bef9b64ac99bd02997 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Oct 2021 16:34:04 -0700 Subject: [PATCH 008/142] fix the issue about object vs jlong --- .../crt/http/Http2ClientConnection.java | 19 +++++++++++++------ src/native/http_request_response.c | 11 +++++++---- .../crt/test/Http2ClientConnectionTest.java | 1 - 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index 14838d9d7..2d47c17cf 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -5,7 +5,6 @@ package software.amazon.awssdk.crt.http; -import software.amazon.awssdk.crt.AsyncCallback; import software.amazon.awssdk.crt.CrtRuntimeException; import java.util.concurrent.CompletableFuture; @@ -21,16 +20,25 @@ * threads. */ public class Http2ClientConnection extends HttpClientConnection { - private AsyncCallback pingAck; /* * Error codes that may be present in HTTP/2 RST_STREAM and GOAWAY frames * (RFC-7540 7). */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), + INTERNAL_ERROR(2), + FLOW_CONTROL_ERROR(3), + SETTINGS_TIMEOUT(4), + STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), + REFUSED_STREAM(7), + CANCEL(8), + COMPRESSION_ERROR(9), + CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), + INADEQUATE_SECURITY(12), + HTTP_1_1_REQUIRED(13); private int errorCode; @@ -74,7 +82,6 @@ public CompletableFuture changeSettings(final Http2ConnectionSetting setti */ public CompletableFuture sendPing(final byte[] pingData) { CompletableFuture completionFuture = new CompletableFuture<>(); - pingAck = AsyncCallback.wrapFuture(completionFuture, null); http2ClientConnectionSendPing(getNativeHandle(), completionFuture, pingData); return completionFuture; } diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 861663c54..08e339838 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -597,7 +597,6 @@ static void s_on_ping_completed( (void)http2_connection; struct s_aws_http2_ping_callback_data *callback_data = user_data; JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); - AWS_LOGF_DEBUG(AWS_LS_HTTP_CONNECTION, "Ping 2"); if (error_code) { jint jni_error_code = error_code; struct aws_byte_cursor error_cursor = aws_byte_cursor_from_c_str(aws_error_name(error_code)); @@ -621,13 +620,17 @@ static void s_on_ping_completed( (*env)->DeleteLocalRef(env, crt_exception); goto done; } - AWS_LOGF_DEBUG(AWS_LS_HTTP_CONNECTION, "Ping 3"); + /* Create a java.lang.long object to complete the future */ + jobject java_round_trip_time_ns = NULL; + jclass cls = (*env)->FindClass(env, "java/lang/Long"); + jmethodID longConstructor = (*env)->GetMethodID(env, cls, "", "(J)V"); + java_round_trip_time_ns = (*env)->NewObject(env, cls, longConstructor, (jlong)round_trip_time_ns); + (*env)->CallBooleanMethod( env, callback_data->java_ping_result_future, completable_future_properties.complete_method_id, - (jlong)round_trip_time_ns); - AWS_LOGF_DEBUG(AWS_LS_HTTP_CONNECTION, "Ping 4"); + java_round_trip_time_ns); done: s_cleanup_ping_callback_data(callback_data); diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 21b96c469..3857532f2 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -59,7 +59,6 @@ public void testHttp2ConnectionPing() throws Exception { shutdownComplete = connPool.getShutdownCompleteFuture(); try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, TimeUnit.SECONDS);) { - Log.initLoggingToFile(Log.LogLevel.Trace, "tracelog.txt"); actuallyConnected = true; Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); long time = conn.sendPing(null).get(5, TimeUnit.SECONDS); From cb35552696cc70a8cbb5dd5927a8175913a17cd4 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Oct 2021 16:40:12 -0700 Subject: [PATCH 009/142] mark the method unimplemented --- .../amazon/awssdk/crt/http/Http2ClientConnection.java | 6 +++--- .../software/amazon/awssdk/crt/http/Http2Stream.java | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index 2d47c17cf..ead07e42c 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -66,8 +66,7 @@ public Http2ClientConnection(long connectionBinding) { * acknowledged the settings and the change has been applied. */ public CompletableFuture changeSettings(final Http2ConnectionSetting settings[]) { - CompletableFuture completionFuture = new CompletableFuture<>(); - return completionFuture; + throw new CrtRuntimeException("Unimplemented"); } /** @@ -111,6 +110,7 @@ public CompletableFuture sendPing(final byte[] pingData) { */ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams, final byte[] debugData) { + throw new CrtRuntimeException("Unimplemented"); } /** @@ -137,7 +137,7 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM * window */ public void updateConnectionWindow(long incrementSize) { - + throw new CrtRuntimeException("Unimplemented"); } /** diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java index 208c9cf8b..ae7c6c7b6 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java @@ -1,5 +1,12 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + package software.amazon.awssdk.crt.http; +import software.amazon.awssdk.crt.CrtRuntimeException; + public class Http2Stream extends HttpStream { protected Http2Stream(long ptr) { @@ -14,7 +21,7 @@ protected Http2Stream(long ptr) { * @param http2_error aws_http2_error_code. Reason to reset the stream. */ public void resetStream(final Http2ClientConnection.Http2ErrorCode errorCode) { - + throw new CrtRuntimeException("Unimplemented"); } /** From 16a5111d101d8cc50d195189b86f988501c3bdb0 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Oct 2021 16:44:17 -0700 Subject: [PATCH 010/142] formatting --- .../amazon/awssdk/crt/http/Http2ConnectionSetting.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index 13fbdd7a3..a2b6ec683 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -2,7 +2,11 @@ public class Http2ConnectionSetting { public enum Http2ConnectionSettingID { - HEADER_TABLE_SIZE(1), ENABLE_PUSH(2), MAX_CONCURRENT_STREAMS(3), INITIAL_WINDOW_SIZE(4), MAX_FRAME_SIZE(5), + HEADER_TABLE_SIZE(1), + ENABLE_PUSH(2), + MAX_CONCURRENT_STREAMS(3), + INITIAL_WINDOW_SIZE(4), + MAX_FRAME_SIZE(5), MAX_HEADER_LIST_SIZE(6); private int settingID; From ece43cb92e223f7446563b7fcbb0820e5ee31550 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 09:24:00 -0700 Subject: [PATCH 011/142] a little better check --- .../amazon/awssdk/crt/http/HttpClientConnection.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index d356ea83a..710c6bf33 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -127,12 +127,11 @@ private static void onConnectionAcquired(CompletableFuture acquireFuture.completeExceptionally(new HttpException(errorCode)); return; } - - HttpClientConnection conn = new HttpClientConnection(nativeConnectionBinding); - if(conn.getVersion() == AwsHTTPProtocolVersion.HTTP_2) { + if(AwsHTTPProtocolVersion.getEnumValueFromInteger((int)httpClientConnectionGetVersion(nativeConnectionBinding)) == AwsHTTPProtocolVersion.HTTP_2) { HttpClientConnection h2Conn = new Http2ClientConnection(nativeConnectionBinding); acquireFuture.complete(h2Conn); - } else{ + } else { + HttpClientConnection conn = new HttpClientConnection(nativeConnectionBinding); acquireFuture.complete(conn); } } From 7c1004019a4f464dbe9cb4a935bbd57ec659aa31 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 13:21:33 -0700 Subject: [PATCH 012/142] optimize code --- src/native/http_request_response.c | 34 +++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 08e339838..319a75b42 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -644,7 +644,11 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio jbyteArray ping_data) { (void)jni_class; + bool success = false; struct aws_allocator *allocator = aws_jni_get_allocator(); + struct aws_byte_cursor *ping_cur_pointer = NULL; + struct aws_byte_cursor ping_cur; + AWS_ZERO_STRUCT(ping_cur); struct s_aws_http2_ping_callback_data *callback_data = aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_ping_callback_data)); if (callback_data == NULL) { @@ -660,31 +664,31 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio if (!native_conn) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid aws_http_connection"); - goto error; + goto done; } callback_data->java_ping_result_future = (*env)->NewGlobalRef(env, java_ping_result_future); if (callback_data->java_ping_result_future == NULL) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: failed to obtain ref to future"); - goto error; + goto done; } if (ping_data) { - struct aws_byte_cursor ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); - if (aws_http2_connection_ping(native_conn, &ping_cur, s_on_ping_completed, callback_data)) { - aws_jni_throw_runtime_exception(env, "Failed to send ping"); - goto error; - } - + ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); + ping_cur_pointer = &ping_cur; + } + if (aws_http2_connection_ping(native_conn, ping_cur_pointer, s_on_ping_completed, callback_data)) { + aws_jni_throw_runtime_exception(env, "Failed to send ping"); + goto done; + } + success = true; +done: + if (ping_cur_pointer) { aws_jni_byte_cursor_from_jbyteArray_release(env, ping_data, ping_cur); - } else { - if (aws_http2_connection_ping(native_conn, NULL, s_on_ping_completed, callback_data)) { - aws_jni_throw_runtime_exception(env, "Failed to send ping"); - goto error; - } } - return; -error: + if (success) { + return; + } s_cleanup_ping_callback_data(callback_data); return; } From 07223239041045b79c3f073abe4110fb21df8a65 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 14:49:30 -0700 Subject: [PATCH 013/142] goaway support --- .../crt/http/Http2ClientConnection.java | 3 +- src/native/http_request_response.c | 33 ++++++++++ .../crt/test/Http2ClientConnectionTest.java | 61 ++++++++++++++++++- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index ead07e42c..d657ffec2 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -110,7 +110,8 @@ public CompletableFuture sendPing(final byte[] pingData) { */ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams, final byte[] debugData) { - throw new CrtRuntimeException("Unimplemented"); + http2ClientConnectionSendGoAway(getNativeHandle(), (long) Http2ErrorCode.getValue(), allowMoreStreams, + debugData); } /** diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 08e339838..ee6df555b 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -689,6 +689,39 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio return; } +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionSendGoAway( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jlong h2_error_code, + jboolean allow_more_streams, + jbyteArray debug_data) { + + (void)jni_class; + struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; + struct aws_http_connection *native_conn = connection_binding->connection; + struct aws_byte_cursor *debug_cur_pointer = NULL; + struct aws_byte_cursor debug_cur; + AWS_ZERO_STRUCT(debug_cur); + + if (!native_conn) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionSendGoAway: Invalid aws_http_connection"); + return; + } + if (debug_data) { + debug_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, debug_data); + debug_cur_pointer = &debug_cur; + } + if (aws_http2_connection_send_goaway(native_conn, (uint32_t)h2_error_code, allow_more_streams, debug_cur_pointer)) { + aws_jni_throw_runtime_exception(env, "Failed to send goaway"); + } + if (debug_cur_pointer) { + aws_jni_byte_cursor_from_jbyteArray_release(env, debug_data, debug_cur); + } + return; +} + #if UINTPTR_MAX == 0xffffffff # if defined(_MSC_VER) # pragma warning(pop) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 3857532f2..255354dd5 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -12,9 +12,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.Http2ClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.Http2ClientConnection.Http2ErrorCode; import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.Log; @@ -61,8 +63,65 @@ public void testHttp2ConnectionPing() throws Exception { TimeUnit.SECONDS);) { actuallyConnected = true; Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); - long time = conn.sendPing(null).get(5, TimeUnit.SECONDS); + long time = conn.sendPing("12345678".getBytes()).get(5, TimeUnit.SECONDS); Assert.assertNotNull(time); + time = conn.sendPing(null).get(5, TimeUnit.SECONDS); + Assert.assertNotNull(time); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } + + @Test + public void testHttp2ConnectionPingExceptionPingDataLength() throws Exception { + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean exception = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, + TimeUnit.SECONDS);) { + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + long time = conn.sendPing("123".getBytes()).get(5, TimeUnit.SECONDS); + Assert.assertNotNull(time); + } + } catch (CrtRuntimeException e) { + exception = true; + Assert.assertTrue(e.getMessage().contains("AWS_ERROR_INVALID_ARGUMENT")); + } + + Assert.assertTrue(exception); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } + + @Test + public void testHttp2ConnectionSendGoAway() throws Exception { + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean actuallyConnected = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, + TimeUnit.SECONDS);) { + actuallyConnected = true; + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + conn.sendGoAway(Http2ErrorCode.INTERNAL_ERROR, false, null); } } catch (Exception e) { throw new RuntimeException(e); From ac051f89d14f103604c9045f806fe8bf194b2d6b Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 15:02:59 -0700 Subject: [PATCH 014/142] connection window update --- src/native/http_request_response.c | 20 +++++++++++ .../crt/test/Http2ClientConnectionTest.java | 35 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index ee6df555b..abcc86b61 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -722,6 +722,26 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio return; } +JNIEXPORT void JNICALL + Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionUpdateConnectionWindow( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jlong increment_size) { + + (void)jni_class; + struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; + struct aws_http_connection *native_conn = connection_binding->connection; + + if (!native_conn) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionUpdateConnectionWindow: Invalid aws_http_connection"); + return; + } + aws_http2_connection_update_window(native_conn, (uint32_t)increment_size); + return; +} + #if UINTPTR_MAX == 0xffffffff # if defined(_MSC_VER) # pragma warning(pop) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 255354dd5..bbe09a098 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -109,6 +109,10 @@ public void testHttp2ConnectionPingExceptionPingDataLength() throws Exception { @Test public void testHttp2ConnectionSendGoAway() throws Exception { + /* + * Test that the binding works not the actual functionality. C part has the test + * for functionality + */ skipIfNetworkUnavailable(); CompletableFuture shutdownComplete = null; @@ -134,4 +138,35 @@ public void testHttp2ConnectionSendGoAway() throws Exception { CrtResource.waitForNoResources(); } + @Test + public void testHttp2ConnectionUpdateConnectionWindow() throws Exception { + /* + * Test that the binding works not the actual functionality. C part has the test + * for functionality + */ + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean actuallyConnected = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, + TimeUnit.SECONDS);) { + actuallyConnected = true; + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + conn.updateConnectionWindow(100); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } + } From 22d49fbf633be91e7722429ef48455eedd4450b5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 16:39:04 -0700 Subject: [PATCH 015/142] WIP for settings --- .../crt/http/Http2ClientConnection.java | 19 ++-- .../crt/http/Http2ConnectionSetting.java | 52 ++++++++- src/native/http_request_response.c | 103 ++++++++++++++++-- src/native/java_class_ids.c | 15 +++ src/native/java_class_ids.h | 8 ++ 5 files changed, 170 insertions(+), 27 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index d657ffec2..b53eeac3b 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -8,6 +8,7 @@ import software.amazon.awssdk.crt.CrtRuntimeException; import java.util.concurrent.CompletableFuture; +import java.util.List; /** * This class wraps aws-c-http to provide the basic HTTP/2 request/response @@ -65,8 +66,11 @@ public Http2ClientConnection(long connectionBinding) { * @return When this future completes without exception, the peer has * acknowledged the settings and the change has been applied. */ - public CompletableFuture changeSettings(final Http2ConnectionSetting settings[]) { - throw new CrtRuntimeException("Unimplemented"); + public CompletableFuture changeSettings(final List settings) { + CompletableFuture completionFuture = new CompletableFuture<>(); + http2ClientConnectionChangeSettings(getNativeHandle(), completionFuture, + Http2ConnectionSetting.marshallSettingsForJNI(settings)); + return completionFuture; } /** @@ -147,14 +151,9 @@ public void updateConnectionWindow(long incrementSize) { /******************************************************************************* * Native methods ******************************************************************************/ - /* - * @TODO: Do we need the change settings? We should have the initial settings - * expose from connection manager... Any thing other than marshall the list to - * byte for the settings array. - */ - // private static native void http2ClientConnectionChangeSettings(long - // connectionBinding, - // CompletableFuture future /* settings */ ) throws CrtRuntimeException; + + private static native void http2ClientConnectionChangeSettings(long connectionBinding, + CompletableFuture future, long[] marshalledSettings) throws CrtRuntimeException; private static native void http2ClientConnectionSendPing(long connectionBinding, CompletableFuture future, byte[] pingData) throws CrtRuntimeException; diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index a2b6ec683..28f305e5c 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -1,12 +1,15 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + package software.amazon.awssdk.crt.http; +import java.util.List; + public class Http2ConnectionSetting { public enum Http2ConnectionSettingID { - HEADER_TABLE_SIZE(1), - ENABLE_PUSH(2), - MAX_CONCURRENT_STREAMS(3), - INITIAL_WINDOW_SIZE(4), - MAX_FRAME_SIZE(5), + HEADER_TABLE_SIZE(1), ENABLE_PUSH(2), MAX_CONCURRENT_STREAMS(3), INITIAL_WINDOW_SIZE(4), MAX_FRAME_SIZE(5), MAX_HEADER_LIST_SIZE(6); private int settingID; @@ -22,4 +25,43 @@ public int getValue() { public Http2ConnectionSettingID id; public long value; + + public Http2ConnectionSetting(Http2ConnectionSettingID id, long value) { + this.id = id; + this.value = value; + } + + /** + * Turn the setting toa list of two long, which makes it much easier for Jni to + * deal with. + * + * @return a long[] that with the [id, value] + */ + public long[] marshalForJni() { + long[] marshalled = new long[2]; + marshalled[0] = (long) id.getValue(); + marshalled[1] = value; + return marshalled; + } + + /** + * Marshals a list of settings into a list for Jni to deal with. + * + * @param settings list of headers to write to the headers block + * @return a long[] that with the [id, value, id, value, *] + */ + public static long[] marshallSettingsForJNI(List settings) { + /* Each setting is two long */ + int totalLength = settings.size(); + + long marshalledSettings[] = new long[totalLength * 2]; + + for (int i = 0; i < totalLength; i++) { + marshalledSettings[i * 2] = settings.get(i).id.getValue(); + marshalledSettings[i * 2 + 1] = settings.get(i).value; + } + + return marshalledSettings; + } + } diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index abcc86b61..a16a399b0 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -574,21 +574,100 @@ JNIEXPORT jshort JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnecti } return (jshort)aws_http_connection_get_version(native_conn); } - -struct s_aws_http2_ping_callback_data { +struct s_aws_http2_callback_data { JavaVM *jvm; - jobject java_ping_result_future; + jobject java_result_future; }; -static void s_cleanup_ping_callback_data(struct s_aws_http2_ping_callback_data *callback_data) { +static void s_cleanup_http2_callback_data(struct s_aws_http2_callback_data *callback_data) { JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); - (*env)->DeleteGlobalRef(env, callback_data->java_ping_result_future); + (*env)->DeleteGlobalRef(env, callback_data->java_result_future); aws_mem_release(aws_jni_get_allocator(), callback_data); } +// static void s_cleanup_http2_callback_data(struct s_aws_http2_ping_callback_data *callback_data) { + +// JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); + +// (*env)->DeleteGlobalRef(env, callback_data->java_result_future); + +// aws_mem_release(aws_jni_get_allocator(), callback_data); +// } + +// static void s_on_ping_completed( +// struct aws_http_connection *http2_connection, +// uint64_t round_trip_time_ns, +// int error_code, +// void *user_data) { +// (void)http2_connection; +// struct s_aws_http2_ping_callback_data *callback_data = user_data; +// JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); +// if (error_code) { +// jint jni_error_code = error_code; +// struct aws_byte_cursor error_cursor = aws_byte_cursor_from_c_str(aws_error_name(error_code)); +// jstring jni_error_string = aws_jni_string_from_cursor(env, &error_cursor); +// AWS_FATAL_ASSERT(jni_error_string); + +// jobject crt_exception = (*env)->NewObject( +// env, +// crt_runtime_exception_properties.crt_runtime_exception_class, +// crt_runtime_exception_properties.constructor_method_id, +// jni_error_code, +// jni_error_string); +// AWS_FATAL_ASSERT(crt_exception); +// (*env)->CallBooleanMethod( +// env, +// callback_data->java_result_future, +// completable_future_properties.complete_exceptionally_method_id, +// crt_exception); +// aws_jni_check_and_clear_exception(env); +// (*env)->DeleteLocalRef(env, jni_error_string); +// (*env)->DeleteLocalRef(env, crt_exception); +// goto done; +// } +// /* Create a java.lang.long object to complete the future */ +// jobject java_round_trip_time_ns = NULL; +// jclass cls = (*env)->FindClass(env, "java/lang/Long"); +// jmethodID longConstructor = (*env)->GetMethodID(env, cls, "", "(J)V"); +// java_round_trip_time_ns = (*env)->NewObject(env, cls, longConstructor, (jlong)round_trip_time_ns); + +// (*env)->CallBooleanMethod( +// env, +// callback_data->java_result_future, +// completable_future_properties.complete_method_id, +// java_round_trip_time_ns); + +// done: +// s_cleanup_http2_callback_data(callback_data); +// } + +// JNIEXPORT void JNICALL +// Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionChangeSettings( +// JNIEnv *env, +// jclass jni_class, +// jlong jni_connection, +// jobject java_change_settings_result_future, +// jlongArray java_marshalled_settings) { + +// (void)jni_class; +// struct aws_allocator *allocator = aws_jni_get_allocator(); +// size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); +// jlong *marshalled_settings = (*env)->GetLongArrayElements(env, java_marshalled_settings, NULL); +// for (int i = 0; i < len; i += 2) { +// jlong id = marshalled_settings[i]; +// jlong value = marshalled_settings[i + 1]; +// } + +// return; +// error: +// (*env)->ReleaseByteArrayElements(env, java_marshalled_settings, (jbyte *)marshalled_settings, JNI_ABORT); +// s_cleanup_http2_callback_data(callback_data); +// return; +// } + static void s_on_ping_completed( struct aws_http_connection *http2_connection, uint64_t round_trip_time_ns, @@ -612,7 +691,7 @@ static void s_on_ping_completed( AWS_FATAL_ASSERT(crt_exception); (*env)->CallBooleanMethod( env, - callback_data->java_ping_result_future, + callback_data->java_result_future, completable_future_properties.complete_exceptionally_method_id, crt_exception); aws_jni_check_and_clear_exception(env); @@ -628,19 +707,19 @@ static void s_on_ping_completed( (*env)->CallBooleanMethod( env, - callback_data->java_ping_result_future, + callback_data->java_result_future, completable_future_properties.complete_method_id, java_round_trip_time_ns); done: - s_cleanup_ping_callback_data(callback_data); + s_cleanup_http2_callback_data(callback_data); } JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionSendPing( JNIEnv *env, jclass jni_class, jlong jni_connection, - jobject java_ping_result_future, + jobject java_result_future, jbyteArray ping_data) { (void)jni_class; @@ -663,8 +742,8 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio goto error; } - callback_data->java_ping_result_future = (*env)->NewGlobalRef(env, java_ping_result_future); - if (callback_data->java_ping_result_future == NULL) { + callback_data->java_result_future = (*env)->NewGlobalRef(env, java_result_future); + if (callback_data->java_result_future == NULL) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: failed to obtain ref to future"); goto error; @@ -685,7 +764,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio } return; error: - s_cleanup_ping_callback_data(callback_data); + s_cleanup_http2_callback_data(callback_data); return; } diff --git a/src/native/java_class_ids.c b/src/native/java_class_ids.c index b408728a7..a86a9824c 100644 --- a/src/native/java_class_ids.c +++ b/src/native/java_class_ids.c @@ -328,6 +328,21 @@ static void s_cache_http_client_connection(JNIEnv *env) { AWS_FATAL_ASSERT(http_client_connection_properties.on_connection_acquired_method_id); } +struct java_http2_connection_setting_properties http2_connection_setting_properties; + +static void s_cache_http2_connection_setting(JNIEnv *env) { + jclass cls = (*env)->FindClass(env, "software/amazon/awssdk/crt/http/Http2ConnectionSetting"); + AWS_FATAL_ASSERT(cls); + http2_connection_setting_properties.http2_connection_setting_class = (*env)->NewGlobalRef(env, cls); + + http2_connection_setting_properties.id_field_id = (*env)->GetFieldID( + env, cls, "id", "Lsoftware/amazon/awssdk/crt/http/Http2ConnectionSetting/Http2ConnectionSettingID;"); + AWS_FATAL_ASSERT(aws_signing_result_properties.signed_request_field_id); + + http2_connection_setting_properties.value_field_id = (*env)->GetFieldID(env, cls, "value", "J"); + AWS_FATAL_ASSERT(aws_signing_result_properties.value_field_id); +} + struct java_http_stream_properties http_stream_properties; static void s_cache_http_stream(JNIEnv *env) { diff --git a/src/native/java_class_ids.h b/src/native/java_class_ids.h index 7f12fa25c..64cb101ed 100644 --- a/src/native/java_class_ids.h +++ b/src/native/java_class_ids.h @@ -150,6 +150,14 @@ struct java_http_client_connection_properties { }; extern struct java_http_client_connection_properties http_client_connection_properties; +/* Http2ConnectionSetting */ +struct java_http2_connection_setting_properties { + jclass http2_connection_setting_class; + jfieldID id_field_id; + jfieldID value_field_id; +}; +extern struct java_http2_connection_setting_properties http2_connection_setting_properties; + /* HttpStream */ struct java_http_stream_properties { jclass stream_class; From 9d2adbd5854f382a0ff980aea2879a8c3777ebf8 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 16:40:17 -0700 Subject: [PATCH 016/142] update the name --- src/native/http_request_response.c | 104 +++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 28 deletions(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 319a75b42..6e4e80598 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -574,17 +574,16 @@ JNIEXPORT jshort JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnecti } return (jshort)aws_http_connection_get_version(native_conn); } - -struct s_aws_http2_ping_callback_data { +struct s_aws_http2_callback_data { JavaVM *jvm; - jobject java_ping_result_future; + jobject java_result_future; }; -static void s_cleanup_ping_callback_data(struct s_aws_http2_ping_callback_data *callback_data) { +static void s_cleanup_http2_callback_data(struct s_aws_http2_callback_data *callback_data) { JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); - (*env)->DeleteGlobalRef(env, callback_data->java_ping_result_future); + (*env)->DeleteGlobalRef(env, callback_data->java_result_future); aws_mem_release(aws_jni_get_allocator(), callback_data); } @@ -612,7 +611,7 @@ static void s_on_ping_completed( AWS_FATAL_ASSERT(crt_exception); (*env)->CallBooleanMethod( env, - callback_data->java_ping_result_future, + callback_data->java_result_future, completable_future_properties.complete_exceptionally_method_id, crt_exception); aws_jni_check_and_clear_exception(env); @@ -628,27 +627,23 @@ static void s_on_ping_completed( (*env)->CallBooleanMethod( env, - callback_data->java_ping_result_future, + callback_data->java_result_future, completable_future_properties.complete_method_id, java_round_trip_time_ns); done: - s_cleanup_ping_callback_data(callback_data); + s_cleanup_http2_callback_data(callback_data); } JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionSendPing( JNIEnv *env, jclass jni_class, jlong jni_connection, - jobject java_ping_result_future, + jobject java_result_future, jbyteArray ping_data) { (void)jni_class; - bool success = false; struct aws_allocator *allocator = aws_jni_get_allocator(); - struct aws_byte_cursor *ping_cur_pointer = NULL; - struct aws_byte_cursor ping_cur; - AWS_ZERO_STRUCT(ping_cur); struct s_aws_http2_ping_callback_data *callback_data = aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_ping_callback_data)); if (callback_data == NULL) { @@ -664,32 +659,85 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio if (!native_conn) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid aws_http_connection"); - goto done; + goto error; } - callback_data->java_ping_result_future = (*env)->NewGlobalRef(env, java_ping_result_future); - if (callback_data->java_ping_result_future == NULL) { + callback_data->java_result_future = (*env)->NewGlobalRef(env, java_result_future); + if (callback_data->java_result_future == NULL) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: failed to obtain ref to future"); - goto done; + goto error; } if (ping_data) { - ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); - ping_cur_pointer = &ping_cur; + struct aws_byte_cursor ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); + if (aws_http2_connection_ping(native_conn, &ping_cur, s_on_ping_completed, callback_data)) { + aws_jni_throw_runtime_exception(env, "Failed to send ping"); + goto error; + } + + aws_jni_byte_cursor_from_jbyteArray_release(env, ping_data, ping_cur); + } else { + if (aws_http2_connection_ping(native_conn, NULL, s_on_ping_completed, callback_data)) { + aws_jni_throw_runtime_exception(env, "Failed to send ping"); + goto error; + } } - if (aws_http2_connection_ping(native_conn, ping_cur_pointer, s_on_ping_completed, callback_data)) { - aws_jni_throw_runtime_exception(env, "Failed to send ping"); - goto done; + return; +error: + s_cleanup_http2_callback_data(callback_data); + return; +} + +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionSendGoAway( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jlong h2_error_code, + jboolean allow_more_streams, + jbyteArray debug_data) { + + (void)jni_class; + struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; + struct aws_http_connection *native_conn = connection_binding->connection; + struct aws_byte_cursor *debug_cur_pointer = NULL; + struct aws_byte_cursor debug_cur; + AWS_ZERO_STRUCT(debug_cur); + + if (!native_conn) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionSendGoAway: Invalid aws_http_connection"); + return; } - success = true; -done: - if (ping_cur_pointer) { - aws_jni_byte_cursor_from_jbyteArray_release(env, ping_data, ping_cur); + if (debug_data) { + debug_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, debug_data); + debug_cur_pointer = &debug_cur; + } + if (aws_http2_connection_send_goaway(native_conn, (uint32_t)h2_error_code, allow_more_streams, debug_cur_pointer)) { + aws_jni_throw_runtime_exception(env, "Failed to send goaway"); } - if (success) { + if (debug_cur_pointer) { + aws_jni_byte_cursor_from_jbyteArray_release(env, debug_data, debug_cur); + } + return; +} + +JNIEXPORT void JNICALL + Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionUpdateConnectionWindow( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jlong increment_size) { + + (void)jni_class; + struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; + struct aws_http_connection *native_conn = connection_binding->connection; + + if (!native_conn) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionUpdateConnectionWindow: Invalid aws_http_connection"); return; } - s_cleanup_ping_callback_data(callback_data); + aws_http2_connection_update_window(native_conn, (uint32_t)increment_size); return; } From e8369d170045261bbd8ecec0e958a59c9fa5bea8 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 16:40:53 -0700 Subject: [PATCH 017/142] settings --- .../crt/http/Http2ConnectionSetting.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index a2b6ec683..4b218f845 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -1,5 +1,12 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + package software.amazon.awssdk.crt.http; +import java.util.List; + public class Http2ConnectionSetting { public enum Http2ConnectionSettingID { HEADER_TABLE_SIZE(1), @@ -22,4 +29,43 @@ public int getValue() { public Http2ConnectionSettingID id; public long value; + + public Http2ConnectionSetting(Http2ConnectionSettingID id, long value) { + this.id = id; + this.value = value; + } + + /** + * Turn the setting toa list of two long, which makes it much easier for Jni to + * deal with. + * + * @return a long[] that with the [id, value] + */ + public long[] marshalForJni() { + long[] marshalled = new long[2]; + marshalled[0] = (long) id.getValue(); + marshalled[1] = value; + return marshalled; + } + + /** + * Marshals a list of settings into a list for Jni to deal with. + * + * @param settings list of headers to write to the headers block + * @return a long[] that with the [id, value, id, value, *] + */ + public static long[] marshallSettingsForJNI(List settings) { + /* Each setting is two long */ + int totalLength = settings.size(); + + long marshalledSettings[] = new long[totalLength * 2]; + + for (int i = 0; i < totalLength; i++) { + marshalledSettings[i * 2] = settings.get(i).id.getValue(); + marshalledSettings[i * 2 + 1] = settings.get(i).value; + } + + return marshalledSettings; + } + } From e91328b29da28173a286f9a5160ee3a4cc3cc682 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 21:05:25 -0700 Subject: [PATCH 018/142] comments addressed --- .../crt/http/Http2ClientConnection.java | 28 ++++----- .../crt/http/Http2ConnectionSetting.java | 35 ++++++----- .../amazon/awssdk/crt/http/Http2Stream.java | 2 +- .../awssdk/crt/http/HttpClientConnection.java | 20 +++---- .../crt/http/HttpClientConnectionManager.java | 2 +- .../HttpClientConnectionManagerOptions.java | 6 +- src/native/aws_signing.c | 14 +---- src/native/crt.c | 12 ++++ src/native/crt.h | 7 +++ src/native/http_request_response.c | 60 ++++++++----------- src/native/java_class_ids.c | 3 +- .../crt/test/Http2ClientConnectionTest.java | 2 +- .../crt/test/Http2RequestResponseTest.java | 2 +- .../crt/test/HttpClientTestFixture.java | 4 +- .../crt/test/HttpRequestResponseFixture.java | 2 +- .../crt/test/HttpRequestResponseTest.java | 6 +- 16 files changed, 99 insertions(+), 106 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index ead07e42c..f9373fbdd 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -26,19 +26,9 @@ public class Http2ClientConnection extends HttpClientConnection { * (RFC-7540 7). */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), - INTERNAL_ERROR(2), - FLOW_CONTROL_ERROR(3), - SETTINGS_TIMEOUT(4), - STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), - REFUSED_STREAM(7), - CANCEL(8), - COMPRESSION_ERROR(9), - CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), - INADEQUATE_SECURITY(12), - HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); private int errorCode; @@ -70,7 +60,7 @@ public CompletableFuture changeSettings(final Http2ConnectionSetting setti } /** - * Send a PING frame Round-trip-time is calculated when PING ACK is received + * Send a PING frame. Round-trip-time is calculated when PING ACK is received * from peer. * * @param pingData Optional 8 Bytes data with the PING frame @@ -85,6 +75,12 @@ public CompletableFuture sendPing(final byte[] pingData) { return completionFuture; } + public CompletableFuture sendPing() { + CompletableFuture completionFuture = new CompletableFuture<>(); + http2ClientConnectionSendPing(getNativeHandle(), completionFuture, null); + return completionFuture; + } + /** * Send a custom GOAWAY frame. * @@ -113,6 +109,10 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM throw new CrtRuntimeException("Unimplemented"); } + public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams) { + throw new CrtRuntimeException("Unimplemented"); + } + /** * Increment the connection's flow-control window to keep data flowing. * diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index 4b218f845..30e550263 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -8,7 +8,10 @@ import java.util.List; public class Http2ConnectionSetting { - public enum Http2ConnectionSettingID { + /** + * Predefined settings identifiers (RFC-7540 6.5.2). + */ + public enum ID { HEADER_TABLE_SIZE(1), ENABLE_PUSH(2), MAX_CONCURRENT_STREAMS(3), @@ -18,7 +21,7 @@ public enum Http2ConnectionSettingID { private int settingID; - Http2ConnectionSettingID(int value) { + ID(int value) { settingID = value; } @@ -27,29 +30,25 @@ public int getValue() { } } - public Http2ConnectionSettingID id; - public long value; - - public Http2ConnectionSetting(Http2ConnectionSettingID id, long value) { - this.id = id; - this.value = value; - } + private ID id; + private long value; /** - * Turn the setting toa list of two long, which makes it much easier for Jni to - * deal with. + * HTTP/2 connection settings. * - * @return a long[] that with the [id, value] + * value is limited from 0 to UINT32_MAX (RFC-7540 6.5.1) */ - public long[] marshalForJni() { - long[] marshalled = new long[2]; - marshalled[0] = (long) id.getValue(); - marshalled[1] = value; - return marshalled; + public Http2ConnectionSetting(ID id, long value) { + if(value > 4294967296L || value < 0) { + throw new IllegalArgumentException(); + } + this.id = id; + this.value = value; } + /** - * Marshals a list of settings into a list for Jni to deal with. + * Marshals a list of settings into an array for Jni to deal with. * * @param settings list of headers to write to the headers block * @return a long[] that with the [id, value, id, value, *] diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java index ae7c6c7b6..ca3a592c1 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java @@ -14,7 +14,7 @@ protected Http2Stream(long ptr) { } /** - * Reset the HTTP/2 stream + * Reset the HTTP/2 stream. * Note that if the stream closes before this async call is fully processed, the RST_STREAM frame will not be sent. * * @param http2_stream HTTP/2 stream. diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index 710c6bf33..b9a35a7ac 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -28,28 +28,28 @@ public class HttpClientConnection extends CrtResource { /** * HTTP protocol version. */ - public enum AwsHTTPProtocolVersion { + public enum ProtocolVersion { HTTP_1_0(1), HTTP_1_1(2), HTTP_2(3); private int protocolVersion; - private static Map enumMapping = buildEnumMapping(); + private static Map enumMapping = buildEnumMapping(); - AwsHTTPProtocolVersion(int value) { + ProtocolVersion(int value) { protocolVersion = value; } - public static AwsHTTPProtocolVersion getEnumValueFromInteger(int value) { - AwsHTTPProtocolVersion enumValue = enumMapping.get(value); + public static ProtocolVersion getEnumValueFromInteger(int value) { + ProtocolVersion enumValue = enumMapping.get(value); if (enumValue != null) { return enumValue; } throw new RuntimeException("Illegal signature type value in signing configuration"); } - private static Map buildEnumMapping() { - Map enumMapping = new HashMap(); + private static Map buildEnumMapping() { + Map enumMapping = new HashMap(); enumMapping.put(HTTP_1_0.getValue(), HTTP_1_0); enumMapping.put(HTTP_1_1.getValue(), HTTP_1_1); enumMapping.put(HTTP_2.getValue(), HTTP_2); @@ -116,9 +116,9 @@ public void shutdown() { httpClientConnectionShutdown(getNativeHandle()); } - public AwsHTTPProtocolVersion getVersion() { + public ProtocolVersion getVersion() { short version = httpClientConnectionGetVersion(getNativeHandle()); - return AwsHTTPProtocolVersion.getEnumValueFromInteger((int)version); + return ProtocolVersion.getEnumValueFromInteger((int)version); }; /** Called from Native when a new connection is acquired **/ @@ -127,7 +127,7 @@ private static void onConnectionAcquired(CompletableFuture acquireFuture.completeExceptionally(new HttpException(errorCode)); return; } - if(AwsHTTPProtocolVersion.getEnumValueFromInteger((int)httpClientConnectionGetVersion(nativeConnectionBinding)) == AwsHTTPProtocolVersion.HTTP_2) { + if(ProtocolVersion.getEnumValueFromInteger((int)httpClientConnectionGetVersion(nativeConnectionBinding)) == ProtocolVersion.HTTP_2) { HttpClientConnection h2Conn = new Http2ClientConnection(nativeConnectionBinding); acquireFuture.complete(h2Conn); } else { diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java index f9b321b54..c9c55a21c 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java @@ -32,7 +32,7 @@ public class HttpClientConnectionManager extends CrtResource { private final int port; private final int maxConnections; private final CompletableFuture shutdownComplete = new CompletableFuture<>(); - private final HttpClientConnection.AwsHTTPProtocolVersion expectedProtocolVersion; + private final HttpClientConnection.ProtocolVersion expectedProtocolVersion; public static HttpClientConnectionManager create(HttpClientConnectionManagerOptions options) { return new HttpClientConnectionManager(options); diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java index 2ce4f88b7..142d2b9a2 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java @@ -29,7 +29,7 @@ public class HttpClientConnectionManagerOptions { private boolean manualWindowManagement = false; private HttpMonitoringOptions monitoringOptions; private long maxConnectionIdleInMilliseconds = 0; - private HttpClientConnection.AwsHTTPProtocolVersion expectedProtocolVersion = HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1; + private HttpClientConnection.ProtocolVersion expectedProtocolVersion = HttpClientConnection.ProtocolVersion.HTTP_1_1; public HttpClientConnectionManagerOptions() { } @@ -203,7 +203,7 @@ public HttpClientConnectionManagerOptions withManualWindowManagement(boolean man * * @return this */ - public HttpClientConnectionManagerOptions withExpectedProtocolVersion(HttpClientConnection.AwsHTTPProtocolVersion expectedProtocolVersion) { + public HttpClientConnectionManagerOptions withExpectedProtocolVersion(HttpClientConnection.ProtocolVersion expectedProtocolVersion) { this.expectedProtocolVersion = expectedProtocolVersion; return this; } @@ -211,7 +211,7 @@ public HttpClientConnectionManagerOptions withExpectedProtocolVersion(HttpClient /** * @return Return the expected HTTP protocol version. */ - public HttpClientConnection.AwsHTTPProtocolVersion getExpectedProtocolVersion() { + public HttpClientConnection.ProtocolVersion getExpectedProtocolVersion() { return expectedProtocolVersion; } diff --git a/src/native/aws_signing.c b/src/native/aws_signing.c index dd512562c..6f30f256e 100644 --- a/src/native/aws_signing.c +++ b/src/native/aws_signing.c @@ -129,18 +129,7 @@ static void s_complete_signing_exceptionally( error_code = AWS_ERROR_UNKNOWN; } - jint jni_error_code = error_code; - struct aws_byte_cursor error_cursor = aws_byte_cursor_from_c_str(aws_error_name(error_code)); - jstring jni_error_string = aws_jni_string_from_cursor(env, &error_cursor); - AWS_FATAL_ASSERT(jni_error_string); - - jobject crt_exception = (*env)->NewObject( - env, - crt_runtime_exception_properties.crt_runtime_exception_class, - crt_runtime_exception_properties.constructor_method_id, - jni_error_code, - jni_error_string); - AWS_FATAL_ASSERT(crt_exception); + jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); (*env)->CallBooleanMethod( env, @@ -149,7 +138,6 @@ static void s_complete_signing_exceptionally( crt_exception); aws_jni_check_and_clear_exception(env); - (*env)->DeleteLocalRef(env, jni_error_string); (*env)->DeleteLocalRef(env, crt_exception); } diff --git a/src/native/crt.c b/src/native/crt.c index 7b400df36..9d378448f 100644 --- a/src/native/crt.c +++ b/src/native/crt.c @@ -184,6 +184,18 @@ struct aws_string *aws_jni_new_string_from_jstring(JNIEnv *env, jstring str) { return result; } +jobject aws_jni_new_crt_exception_from_error_code(JNIEnv *env, int error_code) { + jint jni_error_code = error_code; + + jobject crt_exception = (*env)->NewObject( + env, + crt_runtime_exception_properties.crt_runtime_exception_class, + crt_runtime_exception_properties.constructor_method_id, + jni_error_code); + AWS_FATAL_ASSERT(crt_exception); + return crt_exception; +} + void s_detach_jvm_from_thread(void *user_data) { AWS_LOGF_DEBUG(AWS_LS_COMMON_GENERAL, "s_detach_jvm_from_thread invoked"); JavaVM *jvm = user_data; diff --git a/src/native/crt.h b/src/native/crt.h index e6c2065d1..d89a60234 100644 --- a/src/native/crt.h +++ b/src/native/crt.h @@ -141,6 +141,13 @@ struct aws_byte_cursor aws_jni_byte_cursor_from_direct_byte_buffer(JNIEnv *env, ******************************************************************************/ struct aws_string *aws_jni_new_string_from_jstring(JNIEnv *env, jstring str); +/******************************************************************************* + * aws_jni_new_crt_exception_from_error_code - Creates a new jobject from the aws + * error code, which is the type of software/amazon/awssdk/crt/CrtRuntimeException. + * Reference of the jobject needed to be cleaned up after use. + ******************************************************************************/ +jobject aws_jni_new_crt_exception_from_error_code(JNIEnv *env, int error_code); + /******************************************************************************* * aws_jni_get_thread_env - Gets the JNIEnv for the current thread from the VM, * attaching the env if necessary diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 6e4e80598..c75d7b737 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -594,28 +594,16 @@ static void s_on_ping_completed( int error_code, void *user_data) { (void)http2_connection; - struct s_aws_http2_ping_callback_data *callback_data = user_data; + struct s_aws_http2_callback_data *callback_data = user_data; JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); if (error_code) { - jint jni_error_code = error_code; - struct aws_byte_cursor error_cursor = aws_byte_cursor_from_c_str(aws_error_name(error_code)); - jstring jni_error_string = aws_jni_string_from_cursor(env, &error_cursor); - AWS_FATAL_ASSERT(jni_error_string); - - jobject crt_exception = (*env)->NewObject( - env, - crt_runtime_exception_properties.crt_runtime_exception_class, - crt_runtime_exception_properties.constructor_method_id, - jni_error_code, - jni_error_string); - AWS_FATAL_ASSERT(crt_exception); + jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); (*env)->CallBooleanMethod( env, callback_data->java_result_future, completable_future_properties.complete_exceptionally_method_id, crt_exception); aws_jni_check_and_clear_exception(env); - (*env)->DeleteLocalRef(env, jni_error_string); (*env)->DeleteLocalRef(env, crt_exception); goto done; } @@ -634,7 +622,6 @@ static void s_on_ping_completed( done: s_cleanup_http2_callback_data(callback_data); } - JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionSendPing( JNIEnv *env, jclass jni_class, @@ -643,13 +630,14 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio jbyteArray ping_data) { (void)jni_class; + bool success = false; struct aws_allocator *allocator = aws_jni_get_allocator(); - struct s_aws_http2_ping_callback_data *callback_data = - aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_ping_callback_data)); - if (callback_data == NULL) { - aws_jni_throw_runtime_exception(env, "Failed to allocated ping callback data"); - return; - } + struct aws_byte_cursor *ping_cur_pointer = NULL; + struct aws_byte_cursor ping_cur; + AWS_ZERO_STRUCT(ping_cur); + struct s_aws_http2_callback_data *callback_data = + aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_callback_data)); + jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); AWS_FATAL_ASSERT(jvmresult == 0); @@ -659,31 +647,31 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio if (!native_conn) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid aws_http_connection"); - goto error; + goto done; } callback_data->java_result_future = (*env)->NewGlobalRef(env, java_result_future); if (callback_data->java_result_future == NULL) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: failed to obtain ref to future"); - goto error; + goto done; } if (ping_data) { - struct aws_byte_cursor ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); - if (aws_http2_connection_ping(native_conn, &ping_cur, s_on_ping_completed, callback_data)) { - aws_jni_throw_runtime_exception(env, "Failed to send ping"); - goto error; - } - + ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); + ping_cur_pointer = &ping_cur; + } + if (aws_http2_connection_ping(native_conn, ping_cur_pointer, s_on_ping_completed, callback_data)) { + aws_jni_throw_runtime_exception(env, "Failed to send ping"); + goto done; + } + success = true; +done: + if (ping_cur_pointer) { aws_jni_byte_cursor_from_jbyteArray_release(env, ping_data, ping_cur); - } else { - if (aws_http2_connection_ping(native_conn, NULL, s_on_ping_completed, callback_data)) { - aws_jni_throw_runtime_exception(env, "Failed to send ping"); - goto error; - } } - return; -error: + if (success) { + return; + } s_cleanup_http2_callback_data(callback_data); return; } diff --git a/src/native/java_class_ids.c b/src/native/java_class_ids.c index b408728a7..9fe653f8c 100644 --- a/src/native/java_class_ids.c +++ b/src/native/java_class_ids.c @@ -568,8 +568,7 @@ static void s_cache_crt_runtime_exception(JNIEnv *env) { AWS_FATAL_ASSERT(cls); crt_runtime_exception_properties.crt_runtime_exception_class = (*env)->NewGlobalRef(env, cls); - crt_runtime_exception_properties.constructor_method_id = - (*env)->GetMethodID(env, cls, "", "(ILjava/lang/String;)V"); + crt_runtime_exception_properties.constructor_method_id = (*env)->GetMethodID(env, cls, "", "(I)V"); AWS_FATAL_ASSERT(crt_runtime_exception_properties.constructor_method_id); crt_runtime_exception_properties.error_code_field_id = (*env)->GetFieldID(env, cls, "errorCode", "I"); diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 3857532f2..c25adf47e 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -20,7 +20,7 @@ public class Http2ClientConnectionTest extends HttpClientTestFixture { protected final static String HOST = "https://httpbin.org"; - protected final static HttpClientConnection.AwsHTTPProtocolVersion EXPECTED_VERSION = HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2; + protected final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; @Test public void testHttp2ConnectionGetVersion() throws Exception { diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index 6149a39cd..f34ccf418 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -63,7 +63,7 @@ public boolean resetPosition() { numAttempts++; response = null; try { - response = getResponse(uri, request, null, HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2); + response = getResponse(uri, request, null, HttpClientConnection.ProtocolVersion.HTTP_2); } catch (Exception ex) { // do nothing just let it retry diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java index c260ea242..0a2cb93f3 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java @@ -58,8 +58,8 @@ public TlsContext createHttpClientTlsContext(TlsContextOptions tlsOpts) { } public HttpClientConnectionManager createConnectionPoolManager(URI uri, - HttpClientConnection.AwsHTTPProtocolVersion expectedVersion) { - if (expectedVersion == HttpClientConnection.AwsHTTPProtocolVersion.HTTP_2) { + HttpClientConnection.ProtocolVersion expectedVersion) { + if (expectedVersion == HttpClientConnection.ProtocolVersion.HTTP_2) { return createHTTP2ConnectionPoolManager(uri); } else { return createHTTP1ConnectionPoolManager(uri); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java index 27857c6a6..d55c5dd81 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -84,7 +84,7 @@ public String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmExc } public TestHttpResponse getResponse(URI uri, HttpRequest request, byte[] chunkedData, - HttpClientConnection.AwsHTTPProtocolVersion expectedVersion) throws Exception { + HttpClientConnection.ProtocolVersion expectedVersion) throws Exception { boolean actuallyConnected = false; final CompletableFuture reqCompleted = new CompletableFuture<>(); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java index b7fd3afd7..232ee53a5 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java @@ -76,9 +76,9 @@ public boolean resetPosition() { try { if (useChunkedEncoding) { response = getResponse(uri, request, requestBody.getBytes(UTF8), - HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1); + HttpClientConnection.ProtocolVersion.HTTP_1_1); } else { - response = getResponse(uri, request, null, HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1); + response = getResponse(uri, request, null, HttpClientConnection.ProtocolVersion.HTTP_1_1); } } catch (Exception ex) { // do nothing just let it retry @@ -247,7 +247,7 @@ public void testHttpRequestUnActivated() throws Exception { CompletableFuture shutdownComplete = null; try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, - HttpClientConnection.AwsHTTPProtocolVersion.HTTP_1_1)) { + HttpClientConnection.ProtocolVersion.HTTP_1_1)) { shutdownComplete = connPool.getShutdownCompleteFuture(); try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { From 6da168de3db4cd553edb7e8dfc7408d422e6a05f Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 26 Oct 2021 21:06:43 -0700 Subject: [PATCH 019/142] I don't like this format --- .../awssdk/crt/http/Http2ClientConnection.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index f9373fbdd..794ca51eb 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -26,9 +26,19 @@ public class Http2ClientConnection extends HttpClientConnection { * (RFC-7540 7). */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), + INTERNAL_ERROR(2), + FLOW_CONTROL_ERROR(3), + SETTINGS_TIMEOUT(4), + STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), + REFUSED_STREAM(7), + CANCEL(8), + COMPRESSION_ERROR(9), + CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), + INADEQUATE_SECURITY(12), + HTTP_1_1_REQUIRED(13); private int errorCode; From b1f64bd91ab0c1a4dc13471fe9b353cb995dbc03 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 27 Oct 2021 09:51:11 -0700 Subject: [PATCH 020/142] add getter --- .../awssdk/crt/http/Http2ConnectionSetting.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index 30e550263..7445f2d5d 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -33,20 +33,27 @@ public int getValue() { private ID id; private long value; + public long getValue() { + return value; + } + + public ID getId() { + return id; + } + /** * HTTP/2 connection settings. * * value is limited from 0 to UINT32_MAX (RFC-7540 6.5.1) */ public Http2ConnectionSetting(ID id, long value) { - if(value > 4294967296L || value < 0) { + if (value > 4294967296L || value < 0) { throw new IllegalArgumentException(); } this.id = id; this.value = value; } - /** * Marshals a list of settings into an array for Jni to deal with. * From d430a2ae2221309bc0ef3c2d6d80d5f29aef46cf Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 27 Oct 2021 10:00:12 -0700 Subject: [PATCH 021/142] exclude the marshall function from javadoc --- .../software/amazon/awssdk/crt/eventstream/Header.java | 5 +++-- .../amazon/awssdk/crt/http/Http2ConnectionSetting.java | 2 +- .../software/amazon/awssdk/crt/http/HttpRequest.java | 9 ++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/eventstream/Header.java b/src/main/java/software/amazon/awssdk/crt/eventstream/Header.java index 808db1ae7..15dcfc80f 100644 --- a/src/main/java/software/amazon/awssdk/crt/eventstream/Header.java +++ b/src/main/java/software/amazon/awssdk/crt/eventstream/Header.java @@ -445,8 +445,9 @@ public int getTotalByteLength() { } /** - * Marshals a list of headers into a usable headers block for an event-stream message. - * Used for sending headers across the JNI boundary more efficiently + * @exclude Marshals a list of headers into a usable headers block for an + * event-stream message. Used for sending headers across the JNI + * boundary more efficiently * @param headers list of headers to write to the headers block * @return a byte[] that matches the event-stream header-block format. */ diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index 7445f2d5d..7ced33d2e 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -55,7 +55,7 @@ public Http2ConnectionSetting(ID id, long value) { } /** - * Marshals a list of settings into an array for Jni to deal with. + * @exclude Marshals a list of settings into an array for Jni to deal with. * * @param settings list of headers to write to the headers block * @return a long[] that with the [id, value, id, value, *] diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java b/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java index fe3dc5509..e378eea4e 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java @@ -121,13 +121,12 @@ public HttpRequestBodyStream getBodyStream() { } /** - * Requests are marshalled as follows: + * @exclude Requests are marshalled as follows: * - * each string field is: - * [4-bytes BE] [variable length bytes specified by the previous field] + * each string field is: [4-bytes BE] [variable length bytes specified + * by the previous field] * - * Each request is then: - * [method][path][header name-value pairs] + * Each request is then: [method][path][header name-value pairs] * @return encoded blob of headers */ public byte[] marshalForJni() { From 0505971bc965176d383094273019d87f11a438b0 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 27 Oct 2021 10:17:41 -0700 Subject: [PATCH 022/142] clean up the jobject local ref --- src/native/http_request_response.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index c75d7b737..694172dc4 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -618,7 +618,7 @@ static void s_on_ping_completed( callback_data->java_result_future, completable_future_properties.complete_method_id, java_round_trip_time_ns); - + (*env)->DeleteLocalRef(env, java_round_trip_time_ns); done: s_cleanup_http2_callback_data(callback_data); } From ae4bbc7c6645efce243466a8afa34dd2d5994210 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 27 Oct 2021 13:41:03 -0700 Subject: [PATCH 023/142] update the doc about null is acceptable --- .../amazon/awssdk/crt/http/Http2ClientConnection.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index 794ca51eb..0e745f7ca 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -73,7 +73,7 @@ public CompletableFuture changeSettings(final Http2ConnectionSetting setti * Send a PING frame. Round-trip-time is calculated when PING ACK is received * from peer. * - * @param pingData Optional 8 Bytes data with the PING frame + * @param pingData 8 Bytes data with the PING frame or null for not include data in ping * * @return When this future completes without exception, the peer has * acknowledged the PING and future will be completed with the round @@ -112,7 +112,7 @@ public CompletableFuture sendPing() { * Last-Stream-ID will be set to the latest acknowledged * stream. * @param debugData Optional debug data to send. Size must not exceed - * 16KB. + * 16KB. null is acceptable to not include debug data. */ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams, final byte[] debugData) { From 3661a3b1bbe77a4e1ae406e0dbb4945999c0374e Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 28 Oct 2021 09:50:25 -0700 Subject: [PATCH 024/142] Merge branch 'h2-support' into h2-support-2 --- .../amazon/awssdk/crt/http/Http2ClientConnection.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index e1f232c3a..dc4537f1f 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -125,7 +125,8 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM } public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams) { - throw new CrtRuntimeException("Unimplemented"); + http2ClientConnectionSendGoAway(getNativeHandle(), (long) Http2ErrorCode.getValue(), allowMoreStreams, + null); } /** @@ -152,7 +153,7 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM * window */ public void updateConnectionWindow(long incrementSize) { - throw new CrtRuntimeException("Unimplemented"); + http2ClientConnectionUpdateConnectionWindow(getNativeHandle(), incrementSize); } /** From 41915251f4a6eeeaa30d0387704ae1033b8706d5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 28 Oct 2021 12:55:47 -0700 Subject: [PATCH 025/142] todo: testing --- src/native/http_request_response.c | 166 +++++++++++++++-------------- src/native/java_class_ids.c | 15 --- src/native/java_class_ids.h | 8 -- 3 files changed, 87 insertions(+), 102 deletions(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index ad258812d..7b5c77483 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -588,85 +588,93 @@ static void s_cleanup_http2_callback_data(struct s_aws_http2_callback_data *call aws_mem_release(aws_jni_get_allocator(), callback_data); } -// static void s_cleanup_http2_callback_data(struct s_aws_http2_ping_callback_data *callback_data) { - -// JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); - -// (*env)->DeleteGlobalRef(env, callback_data->java_result_future); - -// aws_mem_release(aws_jni_get_allocator(), callback_data); -// } - -// static void s_on_ping_completed( -// struct aws_http_connection *http2_connection, -// uint64_t round_trip_time_ns, -// int error_code, -// void *user_data) { -// (void)http2_connection; -// struct s_aws_http2_ping_callback_data *callback_data = user_data; -// JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); -// if (error_code) { -// jint jni_error_code = error_code; -// struct aws_byte_cursor error_cursor = aws_byte_cursor_from_c_str(aws_error_name(error_code)); -// jstring jni_error_string = aws_jni_string_from_cursor(env, &error_cursor); -// AWS_FATAL_ASSERT(jni_error_string); - -// jobject crt_exception = (*env)->NewObject( -// env, -// crt_runtime_exception_properties.crt_runtime_exception_class, -// crt_runtime_exception_properties.constructor_method_id, -// jni_error_code, -// jni_error_string); -// AWS_FATAL_ASSERT(crt_exception); -// (*env)->CallBooleanMethod( -// env, -// callback_data->java_result_future, -// completable_future_properties.complete_exceptionally_method_id, -// crt_exception); -// aws_jni_check_and_clear_exception(env); -// (*env)->DeleteLocalRef(env, jni_error_string); -// (*env)->DeleteLocalRef(env, crt_exception); -// goto done; -// } -// /* Create a java.lang.long object to complete the future */ -// jobject java_round_trip_time_ns = NULL; -// jclass cls = (*env)->FindClass(env, "java/lang/Long"); -// jmethodID longConstructor = (*env)->GetMethodID(env, cls, "", "(J)V"); -// java_round_trip_time_ns = (*env)->NewObject(env, cls, longConstructor, (jlong)round_trip_time_ns); - -// (*env)->CallBooleanMethod( -// env, -// callback_data->java_result_future, -// completable_future_properties.complete_method_id, -// java_round_trip_time_ns); - -// done: -// s_cleanup_http2_callback_data(callback_data); -// } - -// JNIEXPORT void JNICALL -// Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionChangeSettings( -// JNIEnv *env, -// jclass jni_class, -// jlong jni_connection, -// jobject java_change_settings_result_future, -// jlongArray java_marshalled_settings) { - -// (void)jni_class; -// struct aws_allocator *allocator = aws_jni_get_allocator(); -// size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); -// jlong *marshalled_settings = (*env)->GetLongArrayElements(env, java_marshalled_settings, NULL); -// for (int i = 0; i < len; i += 2) { -// jlong id = marshalled_settings[i]; -// jlong value = marshalled_settings[i + 1]; -// } - -// return; -// error: -// (*env)->ReleaseByteArrayElements(env, java_marshalled_settings, (jbyte *)marshalled_settings, JNI_ABORT); -// s_cleanup_http2_callback_data(callback_data); -// return; -// } +static void s_on_settings_completed(struct aws_http_connection *http2_connection, int error_code, void *user_data) { + (void)http2_connection; + struct s_aws_http2_callback_data *callback_data = user_data; + JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); + if (error_code) { + jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); + (*env)->CallBooleanMethod( + env, + callback_data->java_result_future, + completable_future_properties.complete_exceptionally_method_id, + crt_exception); + aws_jni_check_and_clear_exception(env); + (*env)->DeleteLocalRef(env, crt_exception); + goto done; + } + (*env)->CallBooleanMethod( + env, callback_data->java_result_future, completable_future_properties.complete_method_id, NULL); +done: + s_cleanup_http2_callback_data(callback_data); +} + +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionChangeSettings( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jobject java_result_future, + jlongArray java_marshalled_settings) { + + (void)jni_class; + struct aws_allocator *allocator = aws_jni_get_allocator(); + struct s_aws_http2_callback_data *callback_data = + aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_callback_data)); + + jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); + AWS_FATAL_ASSERT(jvmresult == 0); + + struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; + struct aws_http_connection *native_conn = connection_binding->connection; + + if (!native_conn) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionChangeSettings: Invalid aws_http_connection"); + s_cleanup_http2_callback_data(callback_data); + return; + } + + callback_data->java_result_future = (*env)->NewGlobalRef(env, java_result_future); + if (callback_data->java_result_future == NULL) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionChangeSettings: failed to obtain ref to future"); + s_cleanup_http2_callback_data(callback_data); + return; + } + + /* We marshalled each setting to two long integers, the long list will be number of settings times two */ + const size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); + if (len % 2) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionChangeSettings: failed to change settings"); + s_cleanup_http2_callback_data(callback_data); + return; + } + const size_t settings_len = len / 2; + struct aws_http2_setting settings[settings_len]; + + jlong *marshalled_settings = (*env)->GetLongArrayElements(env, java_marshalled_settings, NULL); + for (size_t i = 0; i < settings_len; i++) { + jlong id = marshalled_settings[i * 2]; + settings[i].id = id; + jlong value = marshalled_settings[i * 2 + 1]; + settings[i].value = (uint32_t)value; + } + + if (aws_http2_connection_change_settings( + native_conn, settings, settings_len, s_on_settings_completed, callback_data)) { + aws_jni_throw_runtime_exception( + env, "Http2ClientConnection.http2ClientConnectionChangeSettings: failed to change settings"); + goto error; + } + + (*env)->ReleaseLongArrayElements(env, java_marshalled_settings, (jlong *)marshalled_settings, JNI_ABORT); + return; +error: + (*env)->ReleaseLongArrayElements(env, java_marshalled_settings, (jlong *)marshalled_settings, JNI_ABORT); + s_cleanup_http2_callback_data(callback_data); + return; +} static void s_on_ping_completed( struct aws_http_connection *http2_connection, diff --git a/src/native/java_class_ids.c b/src/native/java_class_ids.c index 83ba781f0..9fe653f8c 100644 --- a/src/native/java_class_ids.c +++ b/src/native/java_class_ids.c @@ -328,21 +328,6 @@ static void s_cache_http_client_connection(JNIEnv *env) { AWS_FATAL_ASSERT(http_client_connection_properties.on_connection_acquired_method_id); } -struct java_http2_connection_setting_properties http2_connection_setting_properties; - -static void s_cache_http2_connection_setting(JNIEnv *env) { - jclass cls = (*env)->FindClass(env, "software/amazon/awssdk/crt/http/Http2ConnectionSetting"); - AWS_FATAL_ASSERT(cls); - http2_connection_setting_properties.http2_connection_setting_class = (*env)->NewGlobalRef(env, cls); - - http2_connection_setting_properties.id_field_id = (*env)->GetFieldID( - env, cls, "id", "Lsoftware/amazon/awssdk/crt/http/Http2ConnectionSetting/Http2ConnectionSettingID;"); - AWS_FATAL_ASSERT(aws_signing_result_properties.signed_request_field_id); - - http2_connection_setting_properties.value_field_id = (*env)->GetFieldID(env, cls, "value", "J"); - AWS_FATAL_ASSERT(aws_signing_result_properties.value_field_id); -} - struct java_http_stream_properties http_stream_properties; static void s_cache_http_stream(JNIEnv *env) { diff --git a/src/native/java_class_ids.h b/src/native/java_class_ids.h index 64cb101ed..7f12fa25c 100644 --- a/src/native/java_class_ids.h +++ b/src/native/java_class_ids.h @@ -150,14 +150,6 @@ struct java_http_client_connection_properties { }; extern struct java_http_client_connection_properties http_client_connection_properties; -/* Http2ConnectionSetting */ -struct java_http2_connection_setting_properties { - jclass http2_connection_setting_class; - jfieldID id_field_id; - jfieldID value_field_id; -}; -extern struct java_http2_connection_setting_properties http2_connection_setting_properties; - /* HttpStream */ struct java_http_stream_properties { jclass stream_class; From f774190145607ea04a8ffba51977bc559c04471d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 28 Oct 2021 13:52:44 -0700 Subject: [PATCH 026/142] tests added --- src/native/http_request_response.c | 7 +--- .../crt/test/Http2ClientConnectionTest.java | 33 ++++++++++++++++++- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 7b5c77483..8c7afb2df 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -644,12 +644,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio /* We marshalled each setting to two long integers, the long list will be number of settings times two */ const size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); - if (len % 2) { - aws_jni_throw_runtime_exception( - env, "Http2ClientConnection.http2ClientConnectionChangeSettings: failed to change settings"); - s_cleanup_http2_callback_data(callback_data); - return; - } + AWS_ASSERT(len % 2 == 0); const size_t settings_len = len / 2; struct aws_http2_setting settings[settings_len]; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 502c71bdb..a6e6d3e4c 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -11,14 +11,15 @@ import java.net.URI; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.*; import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.Http2ConnectionSetting; import software.amazon.awssdk.crt.http.Http2ClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.Http2ClientConnection.Http2ErrorCode; import software.amazon.awssdk.crt.CrtResource; -import software.amazon.awssdk.crt.Log; public class Http2ClientConnectionTest extends HttpClientTestFixture { protected final static String HOST = "https://httpbin.org"; @@ -49,6 +50,36 @@ public void testHttp2ConnectionGetVersion() throws Exception { CrtResource.waitForNoResources(); } + @Test + public void testHttp2ConnectionChangeSettings() throws Exception { + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean actuallyConnected = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, + TimeUnit.SECONDS);) { + actuallyConnected = true; + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + List settings = new ArrayList(); + settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.ENABLE_PUSH, 0)); + settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.ENABLE_PUSH, 0)); + conn.changeSettings(settings); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } + @Test public void testHttp2ConnectionPing() throws Exception { skipIfNetworkUnavailable(); From 6e2534cf88eb67265823e916d46f4ad797eb9ae7 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 28 Oct 2021 14:05:41 -0700 Subject: [PATCH 027/142] check the value range --- .../crt/http/Http2ClientConnection.java | 9 ++++-- .../crt/test/Http2ClientConnectionTest.java | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index dc4537f1f..b19ea0734 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -77,7 +77,8 @@ public CompletableFuture changeSettings(final List * Send a PING frame. Round-trip-time is calculated when PING ACK is received * from peer. * - * @param pingData 8 Bytes data with the PING frame or null for not include data in ping + * @param pingData 8 Bytes data with the PING frame or null for not include data + * in ping * * @return When this future completes without exception, the peer has * acknowledged the PING and future will be completed with the round @@ -125,8 +126,7 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM } public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams) { - http2ClientConnectionSendGoAway(getNativeHandle(), (long) Http2ErrorCode.getValue(), allowMoreStreams, - null); + http2ClientConnectionSendGoAway(getNativeHandle(), (long) Http2ErrorCode.getValue(), allowMoreStreams, null); } /** @@ -153,6 +153,9 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM * window */ public void updateConnectionWindow(long incrementSize) { + if (incrementSize > 4294967296L || incrementSize < 0) { + throw new IllegalArgumentException(); + } http2ClientConnectionUpdateConnectionWindow(getNativeHandle(), incrementSize); } diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index a6e6d3e4c..4b9512cfd 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -80,6 +80,35 @@ public void testHttp2ConnectionChangeSettings() throws Exception { CrtResource.waitForNoResources(); } + @Test + public void testHttp2ConnectionChangeSettingsEmpty() throws Exception { + /* empty settings is allowed to send */ + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean actuallyConnected = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, + TimeUnit.SECONDS);) { + actuallyConnected = true; + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + List settings = new ArrayList(); + conn.changeSettings(settings); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } + @Test public void testHttp2ConnectionPing() throws Exception { skipIfNetworkUnavailable(); From 4798e80a5a8f2aa539cb3e099168e0372d884db5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 28 Oct 2021 14:13:05 -0700 Subject: [PATCH 028/142] allocate array to the heap --- src/native/http_request_response.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 8c7afb2df..ad70626ee 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -646,8 +646,8 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio const size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); AWS_ASSERT(len % 2 == 0); const size_t settings_len = len / 2; - struct aws_http2_setting settings[settings_len]; - + struct aws_http2_setting *settings = aws_mem_calloc(allocator, settings_len, sizeof(struct aws_http2_setting)); + int success = false; jlong *marshalled_settings = (*env)->GetLongArrayElements(env, java_marshalled_settings, NULL); for (size_t i = 0; i < settings_len; i++) { jlong id = marshalled_settings[i * 2]; @@ -660,14 +660,15 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio native_conn, settings, settings_len, s_on_settings_completed, callback_data)) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionChangeSettings: failed to change settings"); - goto error; + goto done; } - - (*env)->ReleaseLongArrayElements(env, java_marshalled_settings, (jlong *)marshalled_settings, JNI_ABORT); - return; -error: + success = true; +done: + aws_mem_release(settings); (*env)->ReleaseLongArrayElements(env, java_marshalled_settings, (jlong *)marshalled_settings, JNI_ABORT); - s_cleanup_http2_callback_data(callback_data); + if (!success) { + s_cleanup_http2_callback_data(callback_data); + } return; } From 0b7efcc5a08f4e473ce39819493770508d869b2f Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 28 Oct 2021 14:18:15 -0700 Subject: [PATCH 029/142] you need allocator --- src/native/http_request_response.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index ad70626ee..5c5acfb6a 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -664,7 +664,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio } success = true; done: - aws_mem_release(settings); + aws_mem_release(allocator, settings); (*env)->ReleaseLongArrayElements(env, java_marshalled_settings, (jlong *)marshalled_settings, JNI_ABORT); if (!success) { s_cleanup_http2_callback_data(callback_data); From 33eb1446a158924f6e7a9165c90d17e06c66b6a6 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 29 Oct 2021 09:58:29 -0700 Subject: [PATCH 030/142] zero check --- src/native/http_request_response.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 5c5acfb6a..7ddfcd2d7 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -646,7 +646,8 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio const size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); AWS_ASSERT(len % 2 == 0); const size_t settings_len = len / 2; - struct aws_http2_setting *settings = aws_mem_calloc(allocator, settings_len, sizeof(struct aws_http2_setting)); + struct aws_http2_setting *settings = + settings_len ? aws_mem_calloc(allocator, settings_len, sizeof(struct aws_http2_setting)) : NULL; int success = false; jlong *marshalled_settings = (*env)->GetLongArrayElements(env, java_marshalled_settings, NULL); for (size_t i = 0; i < settings_len; i++) { From f157d8c1047045e3e44fe518288808bf42572a5d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 29 Oct 2021 14:04:32 -0700 Subject: [PATCH 031/142] http2 stream support --- .../crt/http/Http2ClientConnection.java | 29 ++++ .../amazon/awssdk/crt/http/Http2Stream.java | 11 +- .../awssdk/crt/http/HttpClientConnection.java | 16 ++- src/native/http_request_response.c | 85 +++++++++-- src/native/java_class_ids.c | 12 ++ src/native/java_class_ids.h | 7 + .../crt/test/Http2ClientConnectionTest.java | 8 +- .../crt/test/Http2RequestResponseTest.java | 134 +++++++++++------- .../crt/test/HttpRequestResponseFixture.java | 56 +++++++- 9 files changed, 279 insertions(+), 79 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index b19ea0734..a094692c1 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -10,6 +10,8 @@ import java.util.concurrent.CompletableFuture; import java.util.List; +import static software.amazon.awssdk.crt.CRT.awsLastError; + /** * This class wraps aws-c-http to provide the basic HTTP/2 request/response * functionality via the AWS Common Runtime. @@ -159,6 +161,33 @@ public void updateConnectionWindow(long incrementSize) { http2ClientConnectionUpdateConnectionWindow(getNativeHandle(), incrementSize); } + /** + * Schedules an HttpRequest on the Native EventLoop for this + * HttpClientConnection. + * + * @param request The Request to make to the Server. + * @param streamHandler The Stream Handler to be called from the Native + * EventLoop + * @throws CrtRuntimeException if stream creation fails + * @return The Http2Stream that represents this Request/Response Pair. It can be + * closed at any time during the request/response, but must be closed by + * the user thread making this request when it's done. + */ + @Override + public Http2Stream makeRequest(HttpRequest request, HttpStreamResponseHandler streamHandler) + throws CrtRuntimeException { + if (isNull()) { + throw new IllegalStateException("Http2ClientConnection has been closed, can't make requests on it."); + } + + Http2Stream stream = http2ClientConnectionMakeRequest(getNativeHandle(), request.marshalForJni(), + request.getBodyStream(), new HttpStreamResponseHandlerNativeAdapter(streamHandler)); + if (stream == null || stream.isNull()) { + throw new CrtRuntimeException(awsLastError()); + } + return stream; + } + /** * @TODO: bindings for getters of local/remote setting and goaway. */ diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java index ca3a592c1..bebbf7c2e 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java @@ -14,17 +14,18 @@ protected Http2Stream(long ptr) { } /** - * Reset the HTTP/2 stream. - * Note that if the stream closes before this async call is fully processed, the RST_STREAM frame will not be sent. + * Reset the HTTP/2 stream. Note that if the stream closes before this async + * call is fully processed, the RST_STREAM frame will not be sent. * * @param http2_stream HTTP/2 stream. - * @param http2_error aws_http2_error_code. Reason to reset the stream. + * @param http2_error aws_http2_error_code. Reason to reset the stream. */ public void resetStream(final Http2ClientConnection.Http2ErrorCode errorCode) { - throw new CrtRuntimeException("Unimplemented"); + http2StreamResetStream(getNativeHandle(), errorCode.getValue()); } /** - * @TODO getters for reset stream. Not sure anyone needs it. + * @TODO getters for reset stream. Not sure anyone needs it though. */ + private static native void http2StreamResetStream(long http_stream, int errorCode); } diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index b9a35a7ac..fd13c497b 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -14,6 +14,7 @@ import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.http.Http2Stream; import static software.amazon.awssdk.crt.CRT.awsLastError; @@ -79,11 +80,11 @@ public HttpStream makeRequest(HttpRequest request, HttpStreamResponseHandler str if (isNull()) { throw new IllegalStateException("HttpClientConnection has been closed, can't make requests on it."); } - - HttpStream stream = httpClientConnectionMakeRequest(getNativeHandle(), - request.marshalForJni(), - request.getBodyStream(), - new HttpStreamResponseHandlerNativeAdapter(streamHandler)); + HttpStream stream = getVersion() == ProtocolVersion.HTTP_2 + ? http2ClientConnectionMakeRequest(getNativeHandle(), request.marshalForJni(), request.getBodyStream(), + new HttpStreamResponseHandlerNativeAdapter(streamHandler)) + : httpClientConnectionMakeRequest(getNativeHandle(), request.marshalForJni(), request.getBodyStream(), + new HttpStreamResponseHandlerNativeAdapter(streamHandler)); if (stream == null || stream.isNull()) { throw new CrtRuntimeException(awsLastError()); } @@ -144,6 +145,11 @@ private static native HttpStream httpClientConnectionMakeRequest(long connection HttpRequestBodyStream bodyStream, HttpStreamResponseHandlerNativeAdapter responseHandler) throws CrtRuntimeException; + protected static native Http2Stream http2ClientConnectionMakeRequest(long connectionBinding, + byte[] marshalledRequest, + HttpRequestBodyStream bodyStream, + HttpStreamResponseHandlerNativeAdapter responseHandler) throws CrtRuntimeException; + private static native void httpClientConnectionShutdown(long connectionBinding) throws CrtRuntimeException; private static native void httpClientConnectionReleaseManaged(long connectionBinding) throws CrtRuntimeException; diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 7ddfcd2d7..0947a98ea 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -34,11 +34,24 @@ # endif #endif -static jobject s_java_http_stream_from_native_new(JNIEnv *env, void *opaque) { +static jobject s_java_http_stream_from_native_new(JNIEnv *env, void *opaque, enum aws_http_version version) { jlong jni_native_ptr = (jlong)opaque; AWS_ASSERT(jni_native_ptr); - return (*env)->NewObject( - env, http_stream_properties.stream_class, http_stream_properties.constructor, jni_native_ptr); + jobject stream = NULL; + switch (version) { + case AWS_HTTP_VERSION_2: + stream = (*env)->NewObject( + env, http2_stream_properties.stream_class, http2_stream_properties.constructor, jni_native_ptr); + break; + case AWS_HTTP_VERSION_1_0: + case AWS_HTTP_VERSION_1_1: + stream = (*env)->NewObject( + env, http_stream_properties.stream_class, http_stream_properties.constructor, jni_native_ptr); + break; + default: + aws_raise_error(AWS_ERROR_UNIMPLEMENTED); + } + return stream; } static void s_java_http_stream_from_native_delete(JNIEnv *env, jobject jHttpStream) { @@ -275,15 +288,14 @@ jobjectArray aws_java_http_headers_from_native(JNIEnv *env, struct aws_http_head return (ret); } -JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection_httpClientConnectionMakeRequest( +static jobject s_make_request_general( JNIEnv *env, - jclass jni_class, jlong jni_connection, jbyteArray marshalled_request, jobject jni_http_request_body_stream, - jobject jni_http_response_callback_handler) { + jobject jni_http_response_callback_handler, + enum aws_http_version version) { - (void)jni_class; struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; struct aws_http_connection *native_conn = connection_binding->connection; @@ -334,7 +346,7 @@ JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnect (void *)native_conn, (void *)callback_data->native_stream); - jHttpStream = s_java_http_stream_from_native_new(env, callback_data); + jHttpStream = s_java_http_stream_from_native_new(env, callback_data, version); } /* Check for errors that might have occurred while holding the lock. */ @@ -356,6 +368,40 @@ JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnect return jHttpStream; } +JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection_httpClientConnectionMakeRequest( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jbyteArray marshalled_request, + jobject jni_http_request_body_stream, + jobject jni_http_response_callback_handler) { + (void)jni_class; + return s_make_request_general( + env, + jni_connection, + marshalled_request, + jni_http_request_body_stream, + jni_http_response_callback_handler, + AWS_HTTP_VERSION_1_1); +} + +JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection_http2ClientConnectionMakeRequest( + JNIEnv *env, + jclass jni_class, + jlong jni_connection, + jbyteArray marshalled_request, + jobject jni_http_request_body_stream, + jobject jni_http_response_callback_handler) { + (void)jni_class; + return s_make_request_general( + env, + jni_connection, + marshalled_request, + jni_http_request_body_stream, + jni_http_response_callback_handler, + AWS_HTTP_VERSION_2); +} + struct http_stream_chunked_callback_data { struct http_stream_callback_data *stream_cb_data; struct aws_byte_buf chunk_data; @@ -542,6 +588,29 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_HttpStream_httpStrea aws_http_stream_update_window(stream, window_update); } +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2Stream_http2StreamResetStream( + JNIEnv *env, + jclass jni_class, + jlong jni_cb_data, + jint error_code) { + + (void)jni_class; + + struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct aws_http_stream *stream = cb_data->native_stream; + + if (stream == NULL) { + aws_jni_throw_runtime_exception(env, "HttpStream is null."); + return; + } + + AWS_LOGF_TRACE(AWS_LS_HTTP_STREAM, "Resetting Stream. stream: %p", (void *)stream); + if (aws_http2_stream_reset(stream, error_code)) { + aws_jni_throw_runtime_exception(env, "reset stream failed."); + return; + } +} + JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection_httpClientConnectionShutdown( JNIEnv *env, jclass jni_class, diff --git a/src/native/java_class_ids.c b/src/native/java_class_ids.c index 9fe653f8c..42b55295f 100644 --- a/src/native/java_class_ids.c +++ b/src/native/java_class_ids.c @@ -342,6 +342,17 @@ static void s_cache_http_stream(JNIEnv *env) { AWS_FATAL_ASSERT(http_stream_properties.close); } +struct java_http2_stream_properties http2_stream_properties; + +static void s_cache_http2_stream(JNIEnv *env) { + jclass cls = (*env)->FindClass(env, "software/amazon/awssdk/crt/http/Http2Stream"); + AWS_FATAL_ASSERT(cls); + http2_stream_properties.stream_class = (*env)->NewGlobalRef(env, cls); + + http2_stream_properties.constructor = (*env)->GetMethodID(env, cls, "", "(J)V"); + AWS_FATAL_ASSERT(http2_stream_properties.constructor); +} + struct java_http_stream_response_handler_native_adapter_properties http_stream_response_handler_properties; static void s_cache_http_stream_response_handler_native_adapter(JNIEnv *env) { @@ -708,6 +719,7 @@ void cache_java_class_ids(JNIEnv *env) { s_cache_http_client_connection_manager(env); s_cache_http_client_connection(env); s_cache_http_stream(env); + s_cache_http2_stream(env); s_cache_http_stream_response_handler_native_adapter(env); s_cache_http_stream_write_chunk_completion_properties(env); s_cache_event_stream_server_listener_properties(env); diff --git a/src/native/java_class_ids.h b/src/native/java_class_ids.h index 7f12fa25c..3a6f47039 100644 --- a/src/native/java_class_ids.h +++ b/src/native/java_class_ids.h @@ -158,6 +158,13 @@ struct java_http_stream_properties { }; extern struct java_http_stream_properties http_stream_properties; +/* Http2Stream */ +struct java_http2_stream_properties { + jclass stream_class; + jmethodID constructor; +}; +extern struct java_http2_stream_properties http2_stream_properties; + /* HttpStreamResponseHandler */ struct java_http_stream_response_handler_native_adapter_properties { jmethodID onResponseHeaders; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 4b9512cfd..794814bba 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -22,8 +22,12 @@ import software.amazon.awssdk.crt.CrtResource; public class Http2ClientConnectionTest extends HttpClientTestFixture { - protected final static String HOST = "https://httpbin.org"; - protected final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; + /** + * @TODO: maybe use a echo server to verify the frames are sent properly??? We + * have tests in C, not sure if it's worth to add it for java... + */ + private final static String HOST = "https://httpbin.org"; + private final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; @Test public void testHttp2ConnectionGetVersion() throws Exception { diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index f34ccf418..7b70cce58 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -5,52 +5,34 @@ package software.amazon.awssdk.crt.test; -import static software.amazon.awssdk.crt.utils.ByteBufferUtils.transferData; - import org.junit.Assert; import org.junit.Assume; import org.junit.Test; + import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; -import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.http.Http2Stream; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.Http2ClientConnection; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; +import software.amazon.awssdk.crt.http.Http2ClientConnection.Http2ErrorCode; +import software.amazon.awssdk.crt.CrtResource; import java.net.URI; import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; public class Http2RequestResponseTest extends HttpRequestResponseFixture { + private final static String HOST = "https://httpbin.org"; + private final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; public TestHttpResponse testHttp2Request(String method, String endpoint, String path, String requestBody, int expectedStatus) throws Exception { URI uri = new URI(endpoint); - - HttpHeader[] requestHeaders = null; - - /* TODO: Http2 headers and request */ - requestHeaders = new HttpHeader[] { new HttpHeader("host", uri.getHost()), - new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)) }; - - HttpRequestBodyStream bodyStream = null; - - final ByteBuffer bodyBytesIn = ByteBuffer.wrap(requestBody.getBytes(UTF8)); - - bodyStream = new HttpRequestBodyStream() { - @Override - public boolean sendRequestBody(ByteBuffer bodyBytesOut) { - transferData(bodyBytesIn, bodyBytesOut); - - return bodyBytesIn.remaining() == 0; - } - - @Override - public boolean resetPosition() { - bodyBytesIn.position(0); - - return true; - } - }; - - HttpRequest request = new HttpRequest(method, path, requestHeaders, bodyStream); + HttpRequest request = getHttpRequest(method, endpoint, path, requestBody); TestHttpResponse response = null; int numAttempts = 0; @@ -63,7 +45,7 @@ public boolean resetPosition() { numAttempts++; response = null; try { - response = getResponse(uri, request, null, HttpClientConnection.ProtocolVersion.HTTP_2); + response = getResponse(uri, request, null, EXPECTED_VERSION); } catch (Exception ex) { // do nothing just let it retry @@ -92,43 +74,43 @@ public boolean resetPosition() { } @Test - public void testHttpGet() throws Exception { + public void testHttp2Get() throws Exception { Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); - testHttp2Request("GET", "https://httpbin.org", "/delete", EMPTY_BODY, 405); - testHttp2Request("GET", "https://httpbin.org", "/get", EMPTY_BODY, 200); - testHttp2Request("GET", "https://httpbin.org", "/post", EMPTY_BODY, 405); - testHttp2Request("GET", "https://httpbin.org", "/put", EMPTY_BODY, 405); + testHttp2Request("GET", HOST, "/delete", EMPTY_BODY, 405); + testHttp2Request("GET", HOST, "/get", EMPTY_BODY, 200); + testHttp2Request("GET", HOST, "/post", EMPTY_BODY, 405); + testHttp2Request("GET", HOST, "/put", EMPTY_BODY, 405); } @Test - public void testHttpPost() throws Exception { + public void testHttp2Post() throws Exception { skipIfNetworkUnavailable(); - testHttp2Request("POST", "https://httpbin.org", "/delete", EMPTY_BODY, 405); - testHttp2Request("POST", "https://httpbin.org", "/get", EMPTY_BODY, 405); - testHttp2Request("POST", "https://httpbin.org", "/post", EMPTY_BODY, 200); - testHttp2Request("POST", "https://httpbin.org", "/put", EMPTY_BODY, 405); + testHttp2Request("POST", HOST, "/delete", EMPTY_BODY, 405); + testHttp2Request("POST", HOST, "/get", EMPTY_BODY, 405); + testHttp2Request("POST", HOST, "/post", EMPTY_BODY, 200); + testHttp2Request("POST", HOST, "/put", EMPTY_BODY, 405); } @Test - public void testHttpPut() throws Exception { + public void testHttp2Put() throws Exception { skipIfNetworkUnavailable(); - testHttp2Request("PUT", "https://httpbin.org", "/delete", EMPTY_BODY, 405); - testHttp2Request("PUT", "https://httpbin.org", "/get", EMPTY_BODY, 405); - testHttp2Request("PUT", "https://httpbin.org", "/post", EMPTY_BODY, 405); - testHttp2Request("PUT", "https://httpbin.org", "/put", EMPTY_BODY, 200); + testHttp2Request("PUT", HOST, "/delete", EMPTY_BODY, 405); + testHttp2Request("PUT", HOST, "/get", EMPTY_BODY, 405); + testHttp2Request("PUT", HOST, "/post", EMPTY_BODY, 405); + testHttp2Request("PUT", HOST, "/put", EMPTY_BODY, 200); } @Test - public void testHttpResponseStatusCodes() throws Exception { + public void testHttp2ResponseStatusCodes() throws Exception { skipIfNetworkUnavailable(); - testHttp2Request("GET", "https://httpbin.org", "/status/200", EMPTY_BODY, 200); - testHttp2Request("GET", "https://httpbin.org", "/status/300", EMPTY_BODY, 300); - testHttp2Request("GET", "https://httpbin.org", "/status/400", EMPTY_BODY, 400); - testHttp2Request("GET", "https://httpbin.org", "/status/500", EMPTY_BODY, 500); + testHttp2Request("GET", HOST, "/status/200", EMPTY_BODY, 200); + testHttp2Request("GET", HOST, "/status/300", EMPTY_BODY, 300); + testHttp2Request("GET", HOST, "/status/400", EMPTY_BODY, 400); + testHttp2Request("GET", HOST, "/status/500", EMPTY_BODY, 500); } @Test - public void testHttpDownload() throws Exception { + public void testHttp2Download() throws Exception { skipIfNetworkUnavailable(); /* cloudfront uses HTTP/2 */ TestHttpResponse response = testHttp2Request("GET", "https://d1cz66xoahf9cl.cloudfront.net/", @@ -139,4 +121,50 @@ public void testHttpDownload() throws Exception { Assert.assertEquals(TEST_DOC_SHA256, calculateBodyHash(body)); } + + @Test + public void testHttp2ResetStream() throws Exception { + /* + * Test that the binding works not the actual functionality. C part has the test + * for functionality + */ + skipIfNetworkUnavailable(); + + CompletableFuture shutdownComplete = null; + boolean actuallyConnected = false; + URI uri = new URI(HOST); + + try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, EXPECTED_VERSION)) { + shutdownComplete = connPool.getShutdownCompleteFuture(); + try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, + TimeUnit.SECONDS);) { + actuallyConnected = true; + Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); + HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + Http2Stream h2Stream = (Http2Stream) stream; + h2Stream.resetStream(Http2ErrorCode.INTERNAL_ERROR); + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + stream.close(); + } + }; + HttpRequest request = getHttpRequest("GET", HOST, "/get", EMPTY_BODY); + Http2Stream h2Stream = conn.makeRequest(request, streamHandler); + h2Stream.activate(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(actuallyConnected); + + shutdownComplete.get(60, TimeUnit.SECONDS); + + CrtResource.waitForNoResources(); + } } diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java index d55c5dd81..89bb46e2c 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -4,15 +4,20 @@ */ package software.amazon.awssdk.crt.test; +import static software.amazon.awssdk.crt.utils.ByteBufferUtils.transferData; + import org.junit.Assert; import software.amazon.awssdk.crt.CRT; import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.Http2ClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.http.Http2Stream; import java.net.URI; import java.nio.ByteBuffer; @@ -69,6 +74,40 @@ protected boolean shouldRetry(TestHttpResponse response) { return false; } + protected HttpRequest getHttpRequest(String method, String endpoint, String path, String requestBody) + throws Exception { + URI uri = new URI(endpoint); + + HttpHeader[] requestHeaders = null; + + /* TODO: Http2 headers and request */ + requestHeaders = new HttpHeader[] { new HttpHeader("host", uri.getHost()), + new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)) }; + + HttpRequestBodyStream bodyStream = null; + + final ByteBuffer bodyBytesIn = ByteBuffer.wrap(requestBody.getBytes(UTF8)); + + bodyStream = new HttpRequestBodyStream() { + @Override + public boolean sendRequestBody(ByteBuffer bodyBytesOut) { + transferData(bodyBytesIn, bodyBytesOut); + + return bodyBytesIn.remaining() == 0; + } + + @Override + public boolean resetPosition() { + bodyBytesIn.position(0); + + return true; + } + }; + + HttpRequest request = new HttpRequest(method, path, requestHeaders, bodyStream); + return request; + } + public static String byteArrayToHex(byte[] input) { StringBuilder output = new StringBuilder(input.length * 2); for (byte b : input) { @@ -127,12 +166,17 @@ public void onResponseComplete(HttpStream stream, int errorCode) { stream.close(); } }; - - HttpStream stream = conn.makeRequest(request, streamHandler); - stream.activate(); - - if (chunkedData != null) { - stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); + if (expectedVersion == HttpClientConnection.ProtocolVersion.HTTP_2) { + try (Http2ClientConnection h2Connection = (Http2ClientConnection) conn) { + Http2Stream h2Stream = h2Connection.makeRequest(request, streamHandler); + h2Stream.activate(); + } + } else { + HttpStream stream = conn.makeRequest(request, streamHandler); + stream.activate(); + if (chunkedData != null) { + stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); + } } // Give the request up to 60 seconds to complete, otherwise throw a // TimeoutException From 74c48f37d45b47384ba71f55c27beb9abb01905b Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 29 Oct 2021 14:13:54 -0700 Subject: [PATCH 032/142] add comment --- src/native/http_request_response.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 0947a98ea..5fae86e92 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -879,6 +879,7 @@ JNIEXPORT void JNICALL env, "Http2ClientConnection.http2ClientConnectionUpdateConnectionWindow: Invalid aws_http_connection"); return; } + /* We did range check in Java already. */ aws_http2_connection_update_window(native_conn, (uint32_t)increment_size); return; } From 8fd7696c8daee22db20dc3fbd9b43dabc1c1a1bb Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 29 Oct 2021 15:00:10 -0700 Subject: [PATCH 033/142] thanks mike for fixing the exception --- .../amazon/awssdk/crt/test/Http2ClientConnectionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 794814bba..1874a6683 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -161,7 +161,7 @@ public void testHttp2ConnectionPingExceptionPingDataLength() throws Exception { } } catch (CrtRuntimeException e) { exception = true; - Assert.assertTrue(e.getMessage().contains("AWS_ERROR_INVALID_ARGUMENT")); + Assert.assertEquals(e.errorName, "AWS_ERROR_INVALID_ARGUMENT"); } Assert.assertTrue(exception); From 7f8b90554f69a3ef78be7957dfd639c5e7dbca94 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 1 Nov 2021 17:09:44 -0700 Subject: [PATCH 034/142] http2 request --- crt/aws-c-http | 2 +- .../amazon/awssdk/crt/http/Http2Request.java | 26 ++++ .../amazon/awssdk/crt/http/HttpRequest.java | 112 +++--------------- .../awssdk/crt/http/HttpRequestBase.java | 100 ++++++++++++++++ src/native/http_request_utils.c | 35 ++++-- .../crt/test/HttpRequestResponseFixture.java | 4 +- 6 files changed, 175 insertions(+), 104 deletions(-) create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2Request.java create mode 100644 src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java diff --git a/crt/aws-c-http b/crt/aws-c-http index 7af5e6f1a..4825acffd 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 7af5e6f1a8d66243b67e4500b8a30952714a8623 +Subproject commit 4825acffd476ac6fdd5c431ca13e6303b8388832 diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java new file mode 100644 index 000000000..3a6e28055 --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java @@ -0,0 +1,26 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package software.amazon.awssdk.crt.http; + +import software.amazon.awssdk.crt.http.HttpClientConnection.ProtocolVersion; + +/** + * Represents a single Client Request to be sent on a HTTP connection + */ +public class Http2Request extends HttpRequestBase { + /** + * + */ + public Http2Request() { + this(new HttpHeader[] {}, null); + } + + public Http2Request(HttpHeader[] headers, HttpRequestBodyStream bodyStream) { + super(headers, bodyStream); + this.version = ProtocolVersion.HTTP_2; + } + +} diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java b/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java index e378eea4e..7f9d37dc8 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java @@ -4,73 +4,63 @@ */ package software.amazon.awssdk.crt.http; + import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.http.HttpClientConnection.ProtocolVersion; import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; /** * Represents a single Client Request to be sent on a HTTP connection */ -public class HttpRequest { - private final static Charset UTF8 = java.nio.charset.StandardCharsets.UTF_8; - private final static int BUFFER_INT_SIZE = 4; - private final String method; - private String encodedPath; - private List headers; - private HttpRequestBodyStream bodyStream; +public class HttpRequest extends HttpRequestBase { /** * - * @param method http verb to use + * @param method http verb to use * @param encodedPath path of the http request */ public HttpRequest(String method, String encodedPath) { - this(method, encodedPath, new HttpHeader[]{}, null); + this(method, encodedPath, new HttpHeader[] {}, null); } /** * - * @param method http verb to use + * @param method http verb to use * @param encodedPath path of the http request - * @param headers set of http request headers to include - * @param bodyStream (optional) interface to an object that will stream out the request body + * @param headers set of http request headers to include + * @param bodyStream (optional) interface to an object that will stream out the + * request body */ public HttpRequest(String method, String encodedPath, HttpHeader[] headers, HttpRequestBodyStream bodyStream) { - if (headers == null) { throw new IllegalArgumentException("Headers can be empty, but can't be null"); } + super(headers, bodyStream); this.method = method; this.encodedPath = encodedPath; - this.headers = Arrays.asList(headers); - this.bodyStream = bodyStream; } /** - * Package private. Used by JNI to convert native http representation to a Java object, because accessing this - * struct from JNI is too slow. + * Package private. Used by JNI to convert native http representation to a Java + * object, because accessing this struct from JNI is too slow. * - * Requests are marshalled as follows: + * Requests are marshalled as follows: * - * each string field is: - * [4-bytes BE] [variable length bytes specified by the previous field] - * Each request is then: - * [method][path][header name-value pairs] + * each string field is: [4-bytes BE] [variable length bytes specified by the + * previous field] Each request is then: [method][path][header name-value pairs] * * @param marshalledRequest serialized http request to be parsed. - * @param bodyStream body stream for the http request. + * @param bodyStream body stream for the http request. */ HttpRequest(ByteBuffer marshalledRequest, HttpRequestBodyStream bodyStream) { - if (marshalledRequest.remaining() < BUFFER_INT_SIZE * 2) { + if (marshalledRequest.remaining() < BUFFER_INT_SIZE * 3) { throw new CrtRuntimeException("Invalid marshalled request object."); } + this.version = ProtocolVersion.getEnumValueFromInteger(marshalledRequest.getInt()); int methodLength = marshalledRequest.getInt(); byte[] methodBlob = new byte[methodLength]; marshalledRequest.get(methodBlob); this.method = new String(methodBlob, UTF8); - if (marshalledRequest.remaining() < BUFFER_INT_SIZE) { + if (marshalledRequest.remaining() < BUFFER_INT_SIZE) { throw new CrtRuntimeException("Invalid marshalled request object."); } @@ -94,68 +84,4 @@ public String getEncodedPath() { public void setEncodedPath(final String encodedPath) { this.encodedPath = encodedPath; } - - public List getHeaders() { - return headers; - } - - public HttpHeader[] getHeadersAsArray() { - return headers.toArray(new HttpHeader[] {}); - } - - public void addHeader(final HttpHeader header) { - headers.add(header); - } - - public void addHeader(final String headerName, final String headerValue) { - headers.add(new HttpHeader(headerName, headerValue)); - } - - public void addHeaders(final HttpHeader[] headers) { - Collections.addAll(this.headers, headers); - - } - - public HttpRequestBodyStream getBodyStream() { - return bodyStream; - } - - /** - * @exclude Requests are marshalled as follows: - * - * each string field is: [4-bytes BE] [variable length bytes specified - * by the previous field] - * - * Each request is then: [method][path][header name-value pairs] - * @return encoded blob of headers - */ - public byte[] marshalForJni() { - int size = 0; - size += BUFFER_INT_SIZE + method.length(); - size += BUFFER_INT_SIZE + encodedPath.length(); - size += (BUFFER_INT_SIZE * 2) * headers.size(); - - for(HttpHeader header : headers) { - if (header.getNameBytes().length > 0) { - size += header.getNameBytes().length + header.getValueBytes().length; - } - } - - ByteBuffer buffer = ByteBuffer.allocate(size); - buffer.putInt(method.length()); - buffer.put(method.getBytes(UTF8)); - buffer.putInt(encodedPath.length()); - buffer.put(encodedPath.getBytes(UTF8)); - - for(HttpHeader header : headers) { - if (header.getNameBytes().length > 0) { - buffer.putInt(header.getNameBytes().length); - buffer.put(header.getNameBytes()); - buffer.putInt(header.getValueBytes().length); - buffer.put(header.getValueBytes()); - } - } - - return buffer.array(); - } } diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java new file mode 100644 index 000000000..a086b2529 --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java @@ -0,0 +1,100 @@ +package software.amazon.awssdk.crt.http; + +import java.util.Arrays; +import java.util.List; + +import software.amazon.awssdk.crt.http.HttpClientConnection.ProtocolVersion; +import java.util.Arrays; +import java.nio.charset.Charset; +import java.nio.ByteBuffer; +import java.util.Collections; + +public class HttpRequestBase { + protected final static Charset UTF8 = java.nio.charset.StandardCharsets.UTF_8; + protected final static int BUFFER_INT_SIZE = 4; + protected List headers; + protected HttpRequestBodyStream bodyStream; + protected ProtocolVersion version = ProtocolVersion.HTTP_1_1; + protected String method; + protected String encodedPath; + + protected HttpRequestBase() { + } + + protected HttpRequestBase(HttpHeader[] headers, HttpRequestBodyStream bodyStream) { + if (headers == null) { + throw new IllegalArgumentException("Headers can be empty, but can't be null"); + } + this.headers = Arrays.asList(headers); + this.bodyStream = bodyStream; + } + + /** + * @exclude Requests are marshalled as follows: + * + * version is as int: [4-bytes BE] + * + * each string field is: [4-bytes BE] [variable length bytes specified + * by the previous field] + * + * Each request is then: [version][method][path][header name-value + * pairs] + * @return encoded blob of headers + */ + public byte[] marshalForJni() { + int size = 0; + size += BUFFER_INT_SIZE; /* version */ + size += BUFFER_INT_SIZE + method.length(); + size += BUFFER_INT_SIZE + encodedPath.length(); + size += (BUFFER_INT_SIZE * 2) * headers.size(); + + for (HttpHeader header : headers) { + if (header.getNameBytes().length > 0) { + size += header.getNameBytes().length + header.getValueBytes().length; + } + } + + ByteBuffer buffer = ByteBuffer.allocate(size); + buffer.putInt(version.getValue()); + buffer.putInt(method.length()); + buffer.put(method.getBytes(UTF8)); + buffer.putInt(encodedPath.length()); + buffer.put(encodedPath.getBytes(UTF8)); + + for (HttpHeader header : headers) { + if (header.getNameBytes().length > 0) { + buffer.putInt(header.getNameBytes().length); + buffer.put(header.getNameBytes()); + buffer.putInt(header.getValueBytes().length); + buffer.put(header.getValueBytes()); + } + } + + return buffer.array(); + } + + public HttpRequestBodyStream getBodyStream() { + return bodyStream; + } + + public List getHeaders() { + return headers; + } + + public HttpHeader[] getHeadersAsArray() { + return headers.toArray(new HttpHeader[] {}); + } + + public void addHeader(final HttpHeader header) { + headers.add(header); + } + + public void addHeader(final String headerName, final String headerValue) { + headers.add(new HttpHeader(headerName, headerValue)); + } + + public void addHeaders(final HttpHeader[] headers) { + Collections.addAll(this.headers, headers); + + } +} diff --git a/src/native/http_request_utils.c b/src/native/http_request_utils.c index 5d8dd8cb5..2bf173291 100644 --- a/src/native/http_request_utils.c +++ b/src/native/http_request_utils.c @@ -214,6 +214,14 @@ int aws_marshal_http_headers_to_dynamic_buffer( return AWS_OP_SUCCESS; } +static inline enum aws_http_version s_unmarshal_http_request_to_get_version(struct aws_byte_cursor *request_blob) { + uint32_t version = 0; + if (!aws_byte_cursor_read_be32(request_blob, &version)) { + aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } + return version; +} + static inline int s_unmarshal_http_request(struct aws_http_message *message, struct aws_byte_cursor *request_blob) { uint32_t field_len = 0; @@ -282,7 +290,12 @@ int aws_apply_java_http_request_changes_to_native_request( struct aws_byte_cursor marshalled_cur = aws_byte_cursor_from_array((uint8_t *)marshalled_request_data, marshalled_request_length); - result = s_unmarshal_http_request(message, &marshalled_cur); + enum aws_http_version version = s_unmarshal_http_request_to_get_version(&marshalled_cur); + if (version != aws_http_message_get_protocol_version(message)) { + result = aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } else { + result = s_unmarshal_http_request(message, &marshalled_cur); + } (*env)->ReleasePrimitiveArrayCritical(env, marshalled_request, marshalled_request_data, 0); if (jni_body_stream) { @@ -305,17 +318,22 @@ struct aws_http_message *aws_http_request_new_from_java_http_request( jbyteArray marshalled_request, jobject jni_body_stream) { const char *exception_message = NULL; - struct aws_http_message *request = aws_http_message_new_request(aws_jni_get_allocator()); - if (request == NULL) { - aws_jni_throw_runtime_exception(env, "aws_http_request_new_from_java_http_request: Unable to allocate request"); - return NULL; - } const size_t marshalled_request_length = (*env)->GetArrayLength(env, marshalled_request); jbyte *marshalled_request_data = (*env)->GetPrimitiveArrayCritical(env, marshalled_request, NULL); struct aws_byte_cursor marshalled_cur = aws_byte_cursor_from_array((uint8_t *)marshalled_request_data, marshalled_request_length); - int result = s_unmarshal_http_request(request, &marshalled_cur); + enum aws_http_version version = s_unmarshal_http_request_to_get_version(&marshalled_cur); + struct aws_http_message *request = version == AWS_HTTP_VERSION_2 + ? aws_http2_message_new_request(aws_jni_get_allocator()) + : aws_http_message_new_request(aws_jni_get_allocator()); + + int result = AWS_OP_SUCCESS; + if (version != aws_http_message_get_protocol_version(request)) { + result = aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } else { + result = s_unmarshal_http_request(request, &marshalled_cur); + } (*env)->ReleasePrimitiveArrayCritical(env, marshalled_request, marshalled_request_data, 0); if (result) { @@ -358,10 +376,11 @@ static inline int s_marshall_http_request(const struct aws_http_message *message AWS_FATAL_ASSERT(!aws_http_message_get_request_path(message, &path)); - if (aws_byte_buf_reserve_relative(request_buf, sizeof(int) + sizeof(int) + method.len + path.len)) { + if (aws_byte_buf_reserve_relative(request_buf, sizeof(int) + sizeof(int) + sizeof(int) + method.len + path.len)) { return AWS_OP_ERR; } + aws_byte_buf_write_be32(request_buf, (uint32_t)aws_http_message_get_protocol_version(message)); aws_byte_buf_write_be32(request_buf, (uint32_t)method.len); aws_byte_buf_write_from_whole_cursor(request_buf, method); aws_byte_buf_write_be32(request_buf, (uint32_t)path.len); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java index 89bb46e2c..eb63676ef 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -81,8 +81,8 @@ protected HttpRequest getHttpRequest(String method, String endpoint, String path HttpHeader[] requestHeaders = null; /* TODO: Http2 headers and request */ - requestHeaders = new HttpHeader[] { new HttpHeader("host", uri.getHost()), - new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)) }; + requestHeaders = new HttpHeader[] { new HttpHeader("Host", uri.getHost()), + new HttpHeader("Content-Length", Integer.toString(requestBody.getBytes(UTF8).length)) }; HttpRequestBodyStream bodyStream = null; From 4d88b7119ad4d46e927899261e31ae31e3ae2003 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 2 Nov 2021 13:34:49 -0700 Subject: [PATCH 035/142] http2 request --- crt/aws-c-http | 2 +- .../crt/http/Http2ClientConnection.java | 21 +++----- .../amazon/awssdk/crt/http/Http2Request.java | 10 +++- .../awssdk/crt/http/HttpClientConnection.java | 2 +- .../awssdk/crt/http/HttpRequestBase.java | 9 +++- src/native/http_request_utils.c | 34 ++++++------- .../crt/test/Http2RequestResponseTest.java | 51 +++++++++++++++++-- .../crt/test/HttpRequestResponseFixture.java | 45 ++-------------- .../crt/test/HttpRequestResponseTest.java | 21 ++------ 9 files changed, 99 insertions(+), 96 deletions(-) diff --git a/crt/aws-c-http b/crt/aws-c-http index 4825acffd..d8cff2e16 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 4825acffd476ac6fdd5c431ca13e6303b8388832 +Subproject commit d8cff2e16b125c5aad8bda21cf1bd85b8147968b diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index a094692c1..fa6fdbd59 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -29,19 +29,9 @@ public class Http2ClientConnection extends HttpClientConnection { * (RFC-7540 7). */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), - INTERNAL_ERROR(2), - FLOW_CONTROL_ERROR(3), - SETTINGS_TIMEOUT(4), - STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), - REFUSED_STREAM(7), - CANCEL(8), - COMPRESSION_ERROR(9), - CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), - INADEQUATE_SECURITY(12), - HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); private int errorCode; @@ -163,7 +153,8 @@ public void updateConnectionWindow(long incrementSize) { /** * Schedules an HttpRequest on the Native EventLoop for this - * HttpClientConnection. + * HttpClientConnection. The HTTP/1.1 request will be transformed to HTTP/2 + * request under the hood. * * @param request The Request to make to the Server. * @param streamHandler The Stream Handler to be called from the Native @@ -174,7 +165,7 @@ public void updateConnectionWindow(long incrementSize) { * the user thread making this request when it's done. */ @Override - public Http2Stream makeRequest(HttpRequest request, HttpStreamResponseHandler streamHandler) + public Http2Stream makeRequest(HttpRequestBase request, HttpStreamResponseHandler streamHandler) throws CrtRuntimeException { if (isNull()) { throw new IllegalStateException("Http2ClientConnection has been closed, can't make requests on it."); diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java index 3a6e28055..282c18517 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java @@ -12,12 +12,20 @@ */ public class Http2Request extends HttpRequestBase { /** - * + * An empty HTTP/2 Request. */ public Http2Request() { this(new HttpHeader[] {}, null); } + /** + * An empty HTTP/2 Request with headers and body stream. + * + * @param headers set of http request headers to include, note: pseudo + * headers should be set to make a good HTTP/2 request. + * @param bodyStream (optional) interface to an object that will stream out the + * request body + */ public Http2Request(HttpHeader[] headers, HttpRequestBodyStream bodyStream) { super(headers, bodyStream); this.version = ProtocolVersion.HTTP_2; diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index fd13c497b..e0539d8f2 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -76,7 +76,7 @@ protected HttpClientConnection(long connectionBinding) { * @return The HttpStream that represents this Request/Response Pair. It can be closed at any time during the * request/response, but must be closed by the user thread making this request when it's done. */ - public HttpStream makeRequest(HttpRequest request, HttpStreamResponseHandler streamHandler) throws CrtRuntimeException { + public HttpStream makeRequest(HttpRequestBase request, HttpStreamResponseHandler streamHandler) throws CrtRuntimeException { if (isNull()) { throw new IllegalStateException("HttpClientConnection has been closed, can't make requests on it."); } diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java index a086b2529..8088d17a5 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java @@ -1,10 +1,10 @@ package software.amazon.awssdk.crt.http; import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import software.amazon.awssdk.crt.http.HttpClientConnection.ProtocolVersion; -import java.util.Arrays; import java.nio.charset.Charset; import java.nio.ByteBuffer; import java.util.Collections; @@ -18,6 +18,9 @@ public class HttpRequestBase { protected String method; protected String encodedPath; + /** + * Only used for create request from native side. + */ protected HttpRequestBase() { } @@ -25,7 +28,9 @@ protected HttpRequestBase(HttpHeader[] headers, HttpRequestBodyStream bodyStream if (headers == null) { throw new IllegalArgumentException("Headers can be empty, but can't be null"); } - this.headers = Arrays.asList(headers); + method = ""; + encodedPath = ""; + this.headers = new ArrayList(Arrays.asList(headers)); this.bodyStream = bodyStream; } diff --git a/src/native/http_request_utils.c b/src/native/http_request_utils.c index 2bf173291..9f3c9a0f4 100644 --- a/src/native/http_request_utils.c +++ b/src/native/http_request_utils.c @@ -224,29 +224,29 @@ static inline enum aws_http_version s_unmarshal_http_request_to_get_version(stru static inline int s_unmarshal_http_request(struct aws_http_message *message, struct aws_byte_cursor *request_blob) { uint32_t field_len = 0; + if (aws_http_message_get_protocol_version(message) != AWS_HTTP_VERSION_2) { + if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } - if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { - return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - } - - struct aws_byte_cursor method = aws_byte_cursor_advance(request_blob, field_len); + struct aws_byte_cursor method = aws_byte_cursor_advance(request_blob, field_len); - int result = aws_http_message_set_request_method(message, method); - if (result != AWS_OP_SUCCESS) { - return AWS_OP_ERR; - } + int result = aws_http_message_set_request_method(message, method); + if (result != AWS_OP_SUCCESS) { + return AWS_OP_ERR; + } - if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { - return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - } + if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } - struct aws_byte_cursor path = aws_byte_cursor_advance(request_blob, field_len); + struct aws_byte_cursor path = aws_byte_cursor_advance(request_blob, field_len); - result = aws_http_message_set_request_path(message, path); - if (result != AWS_OP_SUCCESS) { - return AWS_OP_ERR; + result = aws_http_message_set_request_path(message, path); + if (result != AWS_OP_SUCCESS) { + return AWS_OP_ERR; + } } - while (request_blob->len) { if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index 7b70cce58..8a04109ac 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -9,9 +9,10 @@ import org.junit.Assume; import org.junit.Test; +import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpHeader; -import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.Http2Request; import software.amazon.awssdk.crt.http.HttpStream; import software.amazon.awssdk.crt.http.Http2Stream; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; @@ -19,6 +20,8 @@ import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.Http2ClientConnection.Http2ErrorCode; import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import static software.amazon.awssdk.crt.utils.ByteBufferUtils.transferData; import java.net.URI; import java.nio.ByteBuffer; @@ -29,10 +32,44 @@ public class Http2RequestResponseTest extends HttpRequestResponseFixture { private final static String HOST = "https://httpbin.org"; private final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; + private Http2Request getHttp2Request(String method, String endpoint, String path, String requestBody) + throws Exception { + URI uri = new URI(endpoint); + + HttpHeader[] requestHeaders = null; + + requestHeaders = new HttpHeader[] { new HttpHeader(":method", method), new HttpHeader(":path", path), + new HttpHeader(":scheme", uri.getScheme()), new HttpHeader(":authority", uri.getHost()), + new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)), }; + + HttpRequestBodyStream bodyStream = null; + + final ByteBuffer bodyBytesIn = ByteBuffer.wrap(requestBody.getBytes(UTF8)); + + bodyStream = new HttpRequestBodyStream() { + @Override + public boolean sendRequestBody(ByteBuffer bodyBytesOut) { + transferData(bodyBytesIn, bodyBytesOut); + + return bodyBytesIn.remaining() == 0; + } + + @Override + public boolean resetPosition() { + bodyBytesIn.position(0); + + return true; + } + }; + + Http2Request request = new Http2Request(requestHeaders, bodyStream); + return request; + } + public TestHttpResponse testHttp2Request(String method, String endpoint, String path, String requestBody, int expectedStatus) throws Exception { URI uri = new URI(endpoint); - HttpRequest request = getHttpRequest(method, endpoint, path, requestBody); + Http2Request request = getHttp2Request(method, endpoint, path, requestBody); TestHttpResponse response = null; int numAttempts = 0; @@ -139,6 +176,7 @@ public void testHttp2ResetStream() throws Exception { try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, TimeUnit.SECONDS);) { actuallyConnected = true; + CompletableFuture streamComplete = new CompletableFuture<>(); Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { @Override @@ -151,11 +189,17 @@ public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blo @Override public void onResponseComplete(HttpStream stream, int errorCode) { stream.close(); + if (errorCode != 0) { + streamComplete.completeExceptionally(new CrtRuntimeException(errorCode)); + } else { + streamComplete.complete(null); + } } }; - HttpRequest request = getHttpRequest("GET", HOST, "/get", EMPTY_BODY); + Http2Request request = getHttp2Request("GET", HOST, "/get", EMPTY_BODY); Http2Stream h2Stream = conn.makeRequest(request, streamHandler); h2Stream.activate(); + streamComplete.get(); } } catch (Exception e) { throw new RuntimeException(e); @@ -167,4 +211,5 @@ public void onResponseComplete(HttpStream stream, int errorCode) { CrtResource.waitForNoResources(); } + } diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java index eb63676ef..99190faa6 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -4,17 +4,16 @@ */ package software.amazon.awssdk.crt.test; -import static software.amazon.awssdk.crt.utils.ByteBufferUtils.transferData; - import org.junit.Assert; import software.amazon.awssdk.crt.CRT; import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.http.Http2ClientConnection; +import software.amazon.awssdk.crt.http.Http2Request; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; -import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import software.amazon.awssdk.crt.http.HttpRequestBase; import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpStream; import software.amazon.awssdk.crt.http.Http2Stream; @@ -74,40 +73,6 @@ protected boolean shouldRetry(TestHttpResponse response) { return false; } - protected HttpRequest getHttpRequest(String method, String endpoint, String path, String requestBody) - throws Exception { - URI uri = new URI(endpoint); - - HttpHeader[] requestHeaders = null; - - /* TODO: Http2 headers and request */ - requestHeaders = new HttpHeader[] { new HttpHeader("Host", uri.getHost()), - new HttpHeader("Content-Length", Integer.toString(requestBody.getBytes(UTF8).length)) }; - - HttpRequestBodyStream bodyStream = null; - - final ByteBuffer bodyBytesIn = ByteBuffer.wrap(requestBody.getBytes(UTF8)); - - bodyStream = new HttpRequestBodyStream() { - @Override - public boolean sendRequestBody(ByteBuffer bodyBytesOut) { - transferData(bodyBytesIn, bodyBytesOut); - - return bodyBytesIn.remaining() == 0; - } - - @Override - public boolean resetPosition() { - bodyBytesIn.position(0); - - return true; - } - }; - - HttpRequest request = new HttpRequest(method, path, requestHeaders, bodyStream); - return request; - } - public static String byteArrayToHex(byte[] input) { StringBuilder output = new StringBuilder(input.length * 2); for (byte b : input) { @@ -122,7 +87,7 @@ public String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmExc return byteArrayToHex(digest.digest()); } - public TestHttpResponse getResponse(URI uri, HttpRequest request, byte[] chunkedData, + public TestHttpResponse getResponse(URI uri, HttpRequestBase request, byte[] chunkedData, HttpClientConnection.ProtocolVersion expectedVersion) throws Exception { boolean actuallyConnected = false; @@ -168,11 +133,11 @@ public void onResponseComplete(HttpStream stream, int errorCode) { }; if (expectedVersion == HttpClientConnection.ProtocolVersion.HTTP_2) { try (Http2ClientConnection h2Connection = (Http2ClientConnection) conn) { - Http2Stream h2Stream = h2Connection.makeRequest(request, streamHandler); + Http2Stream h2Stream = h2Connection.makeRequest((Http2Request) request, streamHandler); h2Stream.activate(); } } else { - HttpStream stream = conn.makeRequest(request, streamHandler); + HttpStream stream = conn.makeRequest((HttpRequest) request, streamHandler); stream.activate(); if (chunkedData != null) { stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java index 232ee53a5..98404891b 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java @@ -184,22 +184,11 @@ private void testHttpUpload(boolean chunked) throws Exception { /** * Example Json Response Body from httpbin.org: * - * { - * "args": {}, - * "data": "This is a sample to prove that http downloads and - * uploads work. It doesn't really matter what's in here, we mainly just need to - * verify the downloads and uploads work.", - * "files": {}, - * "form": {}, - * "headers": { - * "Content-Length": "166", - * "Host": "httpbin.org" - * }, - * "json": null, - * "method": "PUT", - * "origin": "1.2.3.4, 5.6.7.8", - * "url": "https://httpbin.org/anything" - * } + * { "args": {}, "data": "This is a sample to prove that http downloads and + * uploads work. It doesn't really matter what's in here, we mainly just need to + * verify the downloads and uploads work.", "files": {}, "form": {}, "headers": + * { "Content-Length": "166", "Host": "httpbin.org" }, "json": null, "method": + * "PUT", "origin": "1.2.3.4, 5.6.7.8", "url": "https://httpbin.org/anything" } * */ From ecf504e5ab52920bdaf4af522d4e45b9694eafd7 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 2 Nov 2021 13:35:53 -0700 Subject: [PATCH 036/142] update the test --- .../awssdk/crt/test/Http2RequestResponseTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index 7b70cce58..f1d69a630 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -9,6 +9,7 @@ import org.junit.Assume; import org.junit.Test; +import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; @@ -19,6 +20,8 @@ import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.Http2ClientConnection.Http2ErrorCode; import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import static software.amazon.awssdk.crt.utils.ByteBufferUtils.transferData; import java.net.URI; import java.nio.ByteBuffer; @@ -139,6 +142,7 @@ public void testHttp2ResetStream() throws Exception { try (Http2ClientConnection conn = (Http2ClientConnection) connPool.acquireConnection().get(60, TimeUnit.SECONDS);) { actuallyConnected = true; + CompletableFuture streamComplete = new CompletableFuture<>(); Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { @Override @@ -151,11 +155,17 @@ public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blo @Override public void onResponseComplete(HttpStream stream, int errorCode) { stream.close(); + if (errorCode != 0) { + streamComplete.completeExceptionally(new CrtRuntimeException(errorCode)); + } else { + streamComplete.complete(null); + } } }; HttpRequest request = getHttpRequest("GET", HOST, "/get", EMPTY_BODY); Http2Stream h2Stream = conn.makeRequest(request, streamHandler); h2Stream.activate(); + streamComplete.get(); } } catch (Exception e) { throw new RuntimeException(e); @@ -167,4 +177,5 @@ public void onResponseComplete(HttpStream stream, int errorCode) { CrtResource.waitForNoResources(); } + } From 40f93775a85df738de5cb88eb0b72dd819b07cce Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 2 Nov 2021 13:41:37 -0700 Subject: [PATCH 037/142] auto format --- .../crt/test/HttpRequestResponseTest.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java index 98404891b..232ee53a5 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java @@ -184,11 +184,22 @@ private void testHttpUpload(boolean chunked) throws Exception { /** * Example Json Response Body from httpbin.org: * - * { "args": {}, "data": "This is a sample to prove that http downloads and - * uploads work. It doesn't really matter what's in here, we mainly just need to - * verify the downloads and uploads work.", "files": {}, "form": {}, "headers": - * { "Content-Length": "166", "Host": "httpbin.org" }, "json": null, "method": - * "PUT", "origin": "1.2.3.4, 5.6.7.8", "url": "https://httpbin.org/anything" } + * { + * "args": {}, + * "data": "This is a sample to prove that http downloads and + * uploads work. It doesn't really matter what's in here, we mainly just need to + * verify the downloads and uploads work.", + * "files": {}, + * "form": {}, + * "headers": { + * "Content-Length": "166", + * "Host": "httpbin.org" + * }, + * "json": null, + * "method": "PUT", + * "origin": "1.2.3.4, 5.6.7.8", + * "url": "https://httpbin.org/anything" + * } * */ From 895edc62c5540d33d5e830c82ba9192199205469 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 2 Nov 2021 13:43:38 -0700 Subject: [PATCH 038/142] formatting --- .../awssdk/crt/http/Http2ClientConnection.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index fa6fdbd59..cce07e636 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -29,9 +29,19 @@ public class Http2ClientConnection extends HttpClientConnection { * (RFC-7540 7). */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), + INTERNAL_ERROR(2), + FLOW_CONTROL_ERROR(3), + SETTINGS_TIMEOUT(4), + STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), + REFUSED_STREAM(7), + CANCEL(8), + COMPRESSION_ERROR(9), + CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), + INADEQUATE_SECURITY(12), + HTTP_1_1_REQUIRED(13); private int errorCode; From 3727158696d39dfb73401074e708349d078e75b2 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Mon, 18 May 2020 10:10:41 -0700 Subject: [PATCH 039/142] elasticurl wip --- pom.xml | 6 + .../amazon/awssdk/crt/http/HttpVersion.java | 34 +++ .../amazon/awssdk/crt/test/Elasticurl.java | 266 ++++++++++++++++++ 3 files changed, 306 insertions(+) create mode 100644 src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java create mode 100644 src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java diff --git a/pom.xml b/pom.xml index ccac76db2..937e888dc 100644 --- a/pom.xml +++ b/pom.xml @@ -278,6 +278,12 @@ 1.10.19 test + + commons-cli + commons-cli + 1.4 + test + diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java b/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java new file mode 100644 index 000000000..74bed3b66 --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.crt.http; + +public enum HttpVersion { + + UNKNOWN(0), + HTTP1_0(1), + HTTP1_1(2), + HTTP2(3); + + private int value; + + HttpVersion(int value) { + this.value = value; + } + + public int getValue() { + return value; + } +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java new file mode 100644 index 000000000..14d78efd0 --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -0,0 +1,266 @@ +package software.amazon.awssdk.crt.test; + +import java.net.URI; +import java.net.URL; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import software.amazon.awssdk.crt.Log; +import software.amazon.awssdk.crt.Log.LogLevel; +import software.amazon.awssdk.crt.http.HttpVersion; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.io.TlsContextOptions; + +public class Elasticurl { + + static void exit() { + System.exit(1); + } + + static void exit(String msg) { + System.out.println(msg); + exit(); + } + + static CommandLine parseArgs(String args[]) { + Options cliOpts = new Options(); + + cliOpts.addOption("h", "help", false, "show this help message and exit"); + cliOpts.addOption(Option.builder() + .longOpt("cacert") + .hasArg() + .argName("file") + .desc("path to a CA certificate file.") + .build()); + cliOpts.addOption(Option.builder() + .longOpt("capath") + .hasArg() + .argName("dir") + .desc("path to a directory containing CA files.") + .build()); + cliOpts.addOption(Option.builder() + .longOpt("cert") + .hasArg() + .argName("file") + .desc("path to a PEM encoded certificate to use with mTLS.") + .build()); + cliOpts.addOption(Option.builder() + .longOpt("key") + .hasArg() + .argName("file") + .desc("path to a PEM encoded private key that matches cert.") + .build()); + cliOpts.addOption(Option.builder() + .longOpt("connect_timeout") + .hasArg() + .argName("int") + .desc("time in milliseconds to wait for a connection.") + .build()); + cliOpts.addOption(Option.builder("H") + .longOpt("header") + .hasArgs() + .argName("str") + .desc("line to send as a header in format 'name:value'. May be specified multiple times.") + .build()); + cliOpts.addOption(Option.builder("d") + .longOpt("data") + .hasArg() + .argName("str") + .desc("data to send in POST or PUT.") + .build()); + cliOpts.addOption(Option.builder() + .longOpt("data_file") + .hasArg() + .argName("file") + .desc("file to send in POST or PUT.") + .build()); + cliOpts.addOption(Option.builder("M") + .longOpt("method") + .hasArg() + .argName("str") + .desc("request method. Default is GET)") + .build()); + cliOpts.addOption("G", "get", false, "uses GET for request method."); + cliOpts.addOption("P", "post", false, "uses POST for request method."); + cliOpts.addOption("I", "head", false, "uses HEAD for request method."); + cliOpts.addOption("i", "include", false, "includes headers in output."); + cliOpts.addOption("k", "insecure", false, "turns off x.509 validation."); + cliOpts.addOption(Option.builder("o") + .longOpt("output") + .hasArg() + .argName("file") + .desc("dumps content-body to FILE instead of stdout.") + .build()); + cliOpts.addOption(Option.builder("t") + .longOpt("trace") + .hasArg() + .argName("file") + .desc("dumps logs to FILE instead of stderr.") + .build()); + cliOpts.addOption(Option.builder("p") + .longOpt("alpn") + .hasArgs() + .argName("str") + .desc("protocol for ALPN. May be specified multiple times.") + .build()); + cliOpts.addOption(null, "http1_1", false, "HTTP/1.1 connection required."); + cliOpts.addOption(null, "http2", false, "HTTP/2 connection required."); + cliOpts.addOption(Option.builder("v") + .longOpt("verbose") + .hasArg() + .argName("str") + .desc("logging level (ERROR|INFO|DEBUG|TRACE) default is none.") + .build()); + + CommandLineParser cliParser = new DefaultParser(); + CommandLine cli = null; + try { + cli = cliParser.parse(cliOpts, args); + + if (cli.hasOption("help") || cli.getArgs().length == 0) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("elasticurl [OPTIONS]... URL", cliOpts); + exit(); + } + + } catch (ParseException e) { + exit(e.getMessage()); + } + + return cli; + } + + public static void main(String args[]) throws Exception { + CommandLine cli = parseArgs(args); + + // enable logging + String verbose = cli.getOptionValue("verbose"); + if (verbose != null) { + LogLevel logLevel = LogLevel.None; + if (verbose == "ERROR") { + logLevel = LogLevel.Error; + } else if (verbose == "INFO") { + logLevel = LogLevel.Info; + } else if (verbose == "DEBUG") { + logLevel = LogLevel.Debug; + } else if (verbose == "TRACE") { + logLevel = LogLevel.Trace; + } else { + exit(logLevel + " unsupported value for verbose option"); + } + + String trace = cli.getOptionValue("trace"); + if (trace != null) { + Log.initLoggingToFile(logLevel, trace); + } else { + Log.initLoggingToStderr(logLevel); + } + } + + if (cli.getArgs().length == 0) { + exit("missing URL"); + } + + URI uri = new URI(cli.getArgs()[0]); + boolean useTls = true; + int port = 443; + if (uri.getScheme() == "http") { + useTls = true; + port = 80; + } + + HttpVersion requiredVersion = HttpVersion.UNKNOWN; + if (cli.hasOption("http1_1")) { + requiredVersion = HttpVersion.HTTP1_1; + } else if (cli.hasOption("http2")) { + requiredVersion = HttpVersion.HTTP2; + } + + TlsContextOptions tlsOpts = null; + TlsContext tlsCtx = null; + try { + // set up TLS (if https) + if (useTls) { + String cert = cli.getOptionValue("cert"); + String key = cli.getOptionValue("key"); + if (cert != null && key != null) { + tlsOpts = TlsContextOptions.createWithMtlsFromPath(cert, key); + } else { + tlsOpts = TlsContextOptions.createDefaultClient(); + } + + String caPath = cli.getOptionValue("capath"); + String caCert = cli.getOptionValue("cacert"); + if (caPath != null || caCert != null) { + tlsOpts.overrideDefaultTrustStoreFromPath(caPath, caCert); + } + + if (cli.hasOption("insecure")) { + tlsOpts.verifyPeer = false; + } + + String[] alpn = cli.getOptionValues("alpn"); + if (alpn == null) { + if (requiredVersion == HttpVersion.HTTP1_1) { + alpn = new String[]{"http/1.1"}; + } else if (requiredVersion == HttpVersion.HTTP2) { + alpn = new String[]{"h2"}; + } else { + alpn = new String[]{"h2", "http/1.1"}; + } + } + tlsOpts.alpnList = Arrays.asList(alpn); + + tlsCtx = new TlsContext(tlsOpts); + } + + CompletableFuture connMgrShutdownComplete = null; + + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions socketOpts = new SocketOptions()) { + + HttpClientConnectionManagerOptions connMgrOpts = new HttpClientConnectionManagerOptions() + .withClientBootstrap(bootstrap) + .withSocketOptions(socketOpts) + .withTlsContext(tlsCtx) + .withUri(uri) + .withPort(port); + + try (HttpClientConnectionManager connMgr = HttpClientConnectionManager.create(connMgrOpts)) { + connMgrShutdownComplete = connMgr.getShutdownCompleteFuture(); + try () { YOU ARE HERE + } + } finally { + if (connMgrShutdownComplete != null) { + connMgrShutdownComplete.get(); + } + } + } + + } finally { + if (tlsCtx != null) { + tlsCtx.close(); + } + + if (tlsOpts != null) { + tlsOpts.close(); + } + } + } +} From dcbb787c32cb7e757e8f7e420532d75944a86d8d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 3 Nov 2021 15:20:09 -0700 Subject: [PATCH 040/142] use async callback instead --- .../crt/http/Http2ClientConnection.java | 59 ++++---- src/native/http_request_response.c | 126 ++++++++---------- src/native/java_class_ids.c | 8 ++ src/native/java_class_ids.h | 5 +- src/native/mqtt_connection.c | 47 +++---- .../crt/test/Http2ClientConnectionTest.java | 14 +- 6 files changed, 135 insertions(+), 124 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index a094692c1..08d1bc4a9 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -5,6 +5,7 @@ package software.amazon.awssdk.crt.http; +import software.amazon.awssdk.crt.AsyncCallback; import software.amazon.awssdk.crt.CrtRuntimeException; import java.util.concurrent.CompletableFuture; @@ -29,19 +30,9 @@ public class Http2ClientConnection extends HttpClientConnection { * (RFC-7540 7). */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), - INTERNAL_ERROR(2), - FLOW_CONTROL_ERROR(3), - SETTINGS_TIMEOUT(4), - STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), - REFUSED_STREAM(7), - CANCEL(8), - COMPRESSION_ERROR(9), - CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), - INADEQUATE_SECURITY(12), - HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); private int errorCode; @@ -69,10 +60,20 @@ public Http2ClientConnection(long connectionBinding) { * acknowledged the settings and the change has been applied. */ public CompletableFuture changeSettings(final List settings) { - CompletableFuture completionFuture = new CompletableFuture<>(); - http2ClientConnectionChangeSettings(getNativeHandle(), completionFuture, - Http2ConnectionSetting.marshallSettingsForJNI(settings)); - return completionFuture; + CompletableFuture future = new CompletableFuture<>(); + if (isNull()) { + future.completeExceptionally( + new IllegalStateException("Http2ClientConnection has been closed, can't change settings on it.")); + return future; + } + AsyncCallback changeSettingsCompleted = AsyncCallback.wrapFuture(future, null); + try { + http2ClientConnectionChangeSettings(getNativeHandle(), changeSettingsCompleted, + Http2ConnectionSetting.marshallSettingsForJNI(settings)); + } catch (CrtRuntimeException ex) { + future.completeExceptionally(ex); + } + return future; } /** @@ -88,14 +89,22 @@ public CompletableFuture changeSettings(final List */ public CompletableFuture sendPing(final byte[] pingData) { CompletableFuture completionFuture = new CompletableFuture<>(); - http2ClientConnectionSendPing(getNativeHandle(), completionFuture, pingData); + if (isNull()) { + completionFuture.completeExceptionally( + new IllegalStateException("Http2ClientConnection has been closed, can't send ping on it.")); + return completionFuture; + } + AsyncCallback pingCompleted = AsyncCallback.wrapFuture(completionFuture, 0L); + try { + http2ClientConnectionSendPing(getNativeHandle(), pingCompleted, pingData); + } catch (CrtRuntimeException ex) { + completionFuture.completeExceptionally(ex); + } return completionFuture; } public CompletableFuture sendPing() { - CompletableFuture completionFuture = new CompletableFuture<>(); - http2ClientConnectionSendPing(getNativeHandle(), completionFuture, null); - return completionFuture; + return this.sendPing(null); } /** @@ -128,7 +137,7 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM } public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowMoreStreams) { - http2ClientConnectionSendGoAway(getNativeHandle(), (long) Http2ErrorCode.getValue(), allowMoreStreams, null); + this.sendGoAway(Http2ErrorCode, allowMoreStreams, null); } /** @@ -156,7 +165,7 @@ public void sendGoAway(final Http2ErrorCode Http2ErrorCode, final boolean allowM */ public void updateConnectionWindow(long incrementSize) { if (incrementSize > 4294967296L || incrementSize < 0) { - throw new IllegalArgumentException(); + throw new IllegalArgumentException("increment size cannot exceed 4294967296"); } http2ClientConnectionUpdateConnectionWindow(getNativeHandle(), incrementSize); } @@ -196,9 +205,9 @@ public Http2Stream makeRequest(HttpRequest request, HttpStreamResponseHandler st ******************************************************************************/ private static native void http2ClientConnectionChangeSettings(long connectionBinding, - CompletableFuture future, long[] marshalledSettings) throws CrtRuntimeException; + AsyncCallback completedCallback, long[] marshalledSettings) throws CrtRuntimeException; - private static native void http2ClientConnectionSendPing(long connectionBinding, CompletableFuture future, + private static native void http2ClientConnectionSendPing(long connectionBinding, AsyncCallback completedCallback, byte[] pingData) throws CrtRuntimeException; private static native void http2ClientConnectionSendGoAway(long connectionBinding, long h2ErrorCode, diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 5fae86e92..e33ba74a3 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -645,36 +645,46 @@ JNIEXPORT jshort JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnecti } struct s_aws_http2_callback_data { JavaVM *jvm; - jobject java_result_future; + jobject async_callback; }; static void s_cleanup_http2_callback_data(struct s_aws_http2_callback_data *callback_data) { JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); - (*env)->DeleteGlobalRef(env, callback_data->java_result_future); - + if (callback_data->async_callback) { + (*env)->DeleteGlobalRef(env, callback_data->async_callback); + } aws_mem_release(aws_jni_get_allocator(), callback_data); } +static struct s_aws_http2_callback_data *s_new_http2_callback_data( + JNIEnv *env, + struct aws_allocator *allocator, + jobject async_callback) { + struct s_aws_http2_callback_data *callback_data = + aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_callback_data)); + + jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); + AWS_FATAL_ASSERT(jvmresult == 0); + callback_data->async_callback = async_callback ? (*env)->NewGlobalRef(env, async_callback) : NULL; + AWS_FATAL_ASSERT(callback_data->async_callback != NULL); + + return callback_data; +} + static void s_on_settings_completed(struct aws_http_connection *http2_connection, int error_code, void *user_data) { (void)http2_connection; struct s_aws_http2_callback_data *callback_data = user_data; JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); if (error_code) { jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); - (*env)->CallBooleanMethod( - env, - callback_data->java_result_future, - completable_future_properties.complete_exceptionally_method_id, - crt_exception); - aws_jni_check_and_clear_exception(env); + (*env)->CallVoidMethod(env, callback_data->async_callback, async_callback_properties.on_failure, crt_exception); (*env)->DeleteLocalRef(env, crt_exception); - goto done; + } else { + (*env)->CallVoidMethod(env, callback_data->async_callback, async_callback_properties.on_success); } - (*env)->CallBooleanMethod( - env, callback_data->java_result_future, completable_future_properties.complete_method_id, NULL); -done: + AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); s_cleanup_http2_callback_data(callback_data); } @@ -682,34 +692,26 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio JNIEnv *env, jclass jni_class, jlong jni_connection, - jobject java_result_future, + jobject java_async_callback, jlongArray java_marshalled_settings) { (void)jni_class; - struct aws_allocator *allocator = aws_jni_get_allocator(); - struct s_aws_http2_callback_data *callback_data = - aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_callback_data)); - - jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); - AWS_FATAL_ASSERT(jvmresult == 0); struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; struct aws_http_connection *native_conn = connection_binding->connection; if (!native_conn) { - aws_jni_throw_runtime_exception( + aws_jni_throw_null_pointer_exception( env, "Http2ClientConnection.http2ClientConnectionChangeSettings: Invalid aws_http_connection"); - s_cleanup_http2_callback_data(callback_data); return; } - - callback_data->java_result_future = (*env)->NewGlobalRef(env, java_result_future); - if (callback_data->java_result_future == NULL) { - aws_jni_throw_runtime_exception( - env, "Http2ClientConnection.http2ClientConnectionChangeSettings: failed to obtain ref to future"); - s_cleanup_http2_callback_data(callback_data); + if (!java_async_callback) { + aws_jni_throw_null_pointer_exception( + env, "Http2ClientConnection.http2ClientConnectionChangeSettings: Invalid async callback"); return; } + struct aws_allocator *allocator = aws_jni_get_allocator(); + struct s_aws_http2_callback_data *callback_data = s_new_http2_callback_data(env, allocator, java_async_callback); /* We marshalled each setting to two long integers, the long list will be number of settings times two */ const size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); @@ -752,64 +754,49 @@ static void s_on_ping_completed( JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); if (error_code) { jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); - (*env)->CallBooleanMethod( - env, - callback_data->java_result_future, - completable_future_properties.complete_exceptionally_method_id, - crt_exception); - aws_jni_check_and_clear_exception(env); + (*env)->CallVoidMethod(env, callback_data->async_callback, async_callback_properties.on_failure, crt_exception); (*env)->DeleteLocalRef(env, crt_exception); - goto done; + } else { + jobject java_round_trip_time_ns = (*env)->NewObject( + env, boxed_long_properties.long_class, boxed_long_properties.constructor, (jlong)round_trip_time_ns); + (*env)->CallVoidMethod( + env, + callback_data->async_callback, + async_callback_properties.on_success_with_object, + java_round_trip_time_ns); + (*env)->DeleteLocalRef(env, java_round_trip_time_ns); } - /* Create a java.lang.long object to complete the future */ - jobject java_round_trip_time_ns = NULL; - jclass cls = (*env)->FindClass(env, "java/lang/Long"); - jmethodID longConstructor = (*env)->GetMethodID(env, cls, "", "(J)V"); - java_round_trip_time_ns = (*env)->NewObject(env, cls, longConstructor, (jlong)round_trip_time_ns); - - (*env)->CallBooleanMethod( - env, - callback_data->java_result_future, - completable_future_properties.complete_method_id, - java_round_trip_time_ns); - (*env)->DeleteLocalRef(env, java_round_trip_time_ns); -done: + AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); s_cleanup_http2_callback_data(callback_data); } JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionSendPing( JNIEnv *env, jclass jni_class, jlong jni_connection, - jobject java_result_future, + jobject java_async_callback, jbyteArray ping_data) { (void)jni_class; - bool success = false; - struct aws_allocator *allocator = aws_jni_get_allocator(); - struct aws_byte_cursor *ping_cur_pointer = NULL; - struct aws_byte_cursor ping_cur; - AWS_ZERO_STRUCT(ping_cur); - struct s_aws_http2_callback_data *callback_data = - aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_callback_data)); - - jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); - AWS_FATAL_ASSERT(jvmresult == 0); - struct aws_http_connection_binding *connection_binding = (struct aws_http_connection_binding *)jni_connection; struct aws_http_connection *native_conn = connection_binding->connection; if (!native_conn) { aws_jni_throw_runtime_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid aws_http_connection"); - goto done; + return; } - - callback_data->java_result_future = (*env)->NewGlobalRef(env, java_result_future); - if (callback_data->java_result_future == NULL) { - aws_jni_throw_runtime_exception( - env, "Http2ClientConnection.http2ClientConnectionSendPing: failed to obtain ref to future"); - goto done; + if (!java_async_callback) { + aws_jni_throw_null_pointer_exception( + env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid async callback"); + return; } + bool success = false; + struct aws_allocator *allocator = aws_jni_get_allocator(); + struct aws_byte_cursor *ping_cur_pointer = NULL; + struct aws_byte_cursor ping_cur; + AWS_ZERO_STRUCT(ping_cur); + struct s_aws_http2_callback_data *callback_data = s_new_http2_callback_data(env, allocator, java_async_callback); + if (ping_data) { ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); ping_cur_pointer = &ping_cur; @@ -823,10 +810,9 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio if (ping_cur_pointer) { aws_jni_byte_cursor_from_jbyteArray_release(env, ping_data, ping_cur); } - if (success) { - return; + if (!success) { + s_cleanup_http2_callback_data(callback_data); } - s_cleanup_http2_callback_data(callback_data); return; } diff --git a/src/native/java_class_ids.c b/src/native/java_class_ids.c index 260002236..693ebc994 100644 --- a/src/native/java_class_ids.c +++ b/src/native/java_class_ids.c @@ -107,6 +107,10 @@ struct java_boxed_long_properties boxed_long_properties; static void s_cache_boxed_long(JNIEnv *env) { jclass boxed_long_class = (*env)->FindClass(env, "java/lang/Long"); AWS_FATAL_ASSERT(boxed_long_class); + boxed_long_properties.long_class = (*env)->NewGlobalRef(env, boxed_long_class); + + boxed_long_properties.constructor = (*env)->GetMethodID(env, boxed_long_class, "", "(J)V"); + AWS_FATAL_ASSERT(boxed_long_properties.constructor); boxed_long_properties.long_value_method_id = (*env)->GetMethodID(env, boxed_long_class, "longValue", "()J"); AWS_FATAL_ASSERT(boxed_long_properties.long_value_method_id); @@ -288,6 +292,10 @@ static void s_cache_async_callback(JNIEnv *env) { jclass cls = (*env)->FindClass(env, "software/amazon/awssdk/crt/AsyncCallback"); AWS_FATAL_ASSERT(cls); + async_callback_properties.on_success_with_object = + (*env)->GetMethodID(env, cls, "onSuccess", "(Ljava/lang/Object;)V"); + AWS_FATAL_ASSERT(async_callback_properties.on_success_with_object); + async_callback_properties.on_success = (*env)->GetMethodID(env, cls, "onSuccess", "()V"); AWS_FATAL_ASSERT(async_callback_properties.on_success); diff --git a/src/native/java_class_ids.h b/src/native/java_class_ids.h index e7fbbeb11..2b861e251 100644 --- a/src/native/java_class_ids.h +++ b/src/native/java_class_ids.h @@ -43,8 +43,10 @@ struct java_predicate_properties { }; extern struct java_predicate_properties predicate_properties; -/* java/lang/Integer */ +/* java/lang/Long */ struct java_boxed_long_properties { + jclass long_class; + jmethodID constructor; jmethodID long_value_method_id; }; extern struct java_boxed_long_properties boxed_long_properties; @@ -126,6 +128,7 @@ extern struct java_credentials_handler_properties credentials_handler_properties /* AsyncCallback */ struct java_async_callback_properties { + jmethodID on_success_with_object; jmethodID on_success; jmethodID on_failure; }; diff --git a/src/native/mqtt_connection.c b/src/native/mqtt_connection.c index c9f234780..dd4aebcc3 100644 --- a/src/native/mqtt_connection.c +++ b/src/native/mqtt_connection.c @@ -97,16 +97,13 @@ static void s_mqtt_jni_connection_release(struct mqtt_jni_connection *connection } } -static struct mqtt_jni_async_callback *mqtt_jni_async_callback_new( +static struct mqtt_jni_async_callback *s_mqtt_jni_async_callback_new( struct mqtt_jni_connection *connection, jobject async_callback) { struct aws_allocator *allocator = aws_jni_get_allocator(); + /* allocate cannot fail */ struct mqtt_jni_async_callback *callback = aws_mem_calloc(allocator, 1, sizeof(struct mqtt_jni_async_callback)); - if (!callback) { - /* caller will throw when they get a null */ - return NULL; - } JNIEnv *env = aws_jni_get_thread_env(connection->jvm); @@ -118,7 +115,7 @@ static struct mqtt_jni_async_callback *mqtt_jni_async_callback_new( return callback; } -static void mqtt_jni_async_callback_destroy(struct mqtt_jni_async_callback *callback) { +static void s_mqtt_jni_async_callback_destroy(struct mqtt_jni_async_callback *callback) { AWS_FATAL_ASSERT(callback && callback->connection); JNIEnv *env = aws_jni_get_thread_env(callback->connection->jvm); if (callback->async_callback) { @@ -179,7 +176,7 @@ static void s_on_connection_complete( } } - mqtt_jni_async_callback_destroy(connect_callback); + s_mqtt_jni_async_callback_destroy(connect_callback); s_mqtt_jni_connection_release(connection); } @@ -241,7 +238,7 @@ static void s_on_connection_disconnected(struct aws_mqtt_client_connection *clie s_on_connection_interrupted_internal(connect_callback->connection, 0, connect_callback->async_callback); - mqtt_jni_async_callback_destroy(connect_callback); + s_mqtt_jni_async_callback_destroy(connect_callback); AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); s_mqtt_jni_connection_release(jni_connection); @@ -290,7 +287,7 @@ static void s_mqtt_connection_destroy(JNIEnv *env, struct mqtt_jni_connection *c } if (connection->on_message) { - mqtt_jni_async_callback_destroy(connection->on_message); + s_mqtt_jni_async_callback_destroy(connection->on_message); } if (connection->java_mqtt_connection) { @@ -401,7 +398,7 @@ void JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttClien goto cleanup; } - struct mqtt_jni_async_callback *connect_callback = mqtt_jni_async_callback_new(connection, NULL); + struct mqtt_jni_async_callback *connect_callback = s_mqtt_jni_async_callback_new(connection, NULL); if (connect_callback == NULL) { aws_jni_throw_runtime_exception(env, "MqttClientConnection.mqtt_connect: Failed to create async callback"); goto cleanup; @@ -448,7 +445,7 @@ void JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttClien int result = aws_mqtt_client_connection_connect(connection->client_connection, &connect_options); if (result != AWS_OP_SUCCESS) { s_mqtt_jni_connection_release(connection); - mqtt_jni_async_callback_destroy(connect_callback); + s_mqtt_jni_async_callback_destroy(connect_callback); aws_jni_throw_runtime_exception( env, "MqttClientConnection.mqtt_connect: aws_mqtt_client_connection_connect failed"); } @@ -474,7 +471,7 @@ void JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttClien return; } - struct mqtt_jni_async_callback *disconnect_callback = mqtt_jni_async_callback_new(connection, jni_ack); + struct mqtt_jni_async_callback *disconnect_callback = s_mqtt_jni_async_callback_new(connection, jni_ack); if (disconnect_callback == NULL) { aws_jni_throw_runtime_exception(env, "MqttClientConnection.mqtt_disconnect: Failed to create async callback"); return; @@ -545,7 +542,7 @@ static void s_on_op_complete( s_deliver_ack_success(callback); } - mqtt_jni_async_callback_destroy(callback); + s_mqtt_jni_async_callback_destroy(callback); } static void s_on_ack( @@ -562,7 +559,7 @@ static void s_on_ack( static void s_cleanup_handler(void *user_data) { struct mqtt_jni_async_callback *handler = user_data; - mqtt_jni_async_callback_destroy(handler); + s_mqtt_jni_async_callback_destroy(handler); } static void s_on_subscription_delivered( @@ -615,7 +612,7 @@ jshort JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttCli return 0; } - struct mqtt_jni_async_callback *handler = mqtt_jni_async_callback_new(connection, jni_handler); + struct mqtt_jni_async_callback *handler = s_mqtt_jni_async_callback_new(connection, jni_handler); if (!handler) { aws_jni_throw_runtime_exception(env, "MqttClientConnection.mqtt_subscribe: Unable to allocate handler"); return 0; @@ -624,7 +621,7 @@ jshort JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttCli /* from here, any failure requires error_cleanup */ struct mqtt_jni_async_callback *sub_ack = NULL; if (jni_ack) { - sub_ack = mqtt_jni_async_callback_new(connection, jni_ack); + sub_ack = s_mqtt_jni_async_callback_new(connection, jni_ack); if (!sub_ack) { aws_jni_throw_runtime_exception(env, "MqttClientConnection.mqtt_subscribe: Unable to allocate sub ack"); goto error_cleanup; @@ -654,11 +651,11 @@ jshort JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttCli error_cleanup: if (handler) { - mqtt_jni_async_callback_destroy(handler); + s_mqtt_jni_async_callback_destroy(handler); } if (sub_ack) { - mqtt_jni_async_callback_destroy(sub_ack); + s_mqtt_jni_async_callback_destroy(sub_ack); } return 0; @@ -682,7 +679,7 @@ void JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttClien return; } - struct mqtt_jni_async_callback *handler = mqtt_jni_async_callback_new(connection, jni_handler); + struct mqtt_jni_async_callback *handler = s_mqtt_jni_async_callback_new(connection, jni_handler); if (!handler) { aws_jni_throw_runtime_exception( env, "MqttClientConnection.mqttClientConnectionOnMessage: Unable to allocate handler"); @@ -697,7 +694,7 @@ void JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttClien } if (connection->on_message) { - mqtt_jni_async_callback_destroy(connection->on_message); + s_mqtt_jni_async_callback_destroy(connection->on_message); } connection->on_message = handler; @@ -706,7 +703,7 @@ void JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttClien error_cleanup: if (handler) { - mqtt_jni_async_callback_destroy(handler); + s_mqtt_jni_async_callback_destroy(handler); } } @@ -727,7 +724,7 @@ jshort JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttCli return 0; } - struct mqtt_jni_async_callback *unsub_ack = mqtt_jni_async_callback_new(connection, jni_ack); + struct mqtt_jni_async_callback *unsub_ack = s_mqtt_jni_async_callback_new(connection, jni_ack); if (!unsub_ack) { aws_jni_throw_runtime_exception(env, "MqttClientConnection.mqtt_unsubscribe: Unable to allocate unsub ack"); goto error_cleanup; @@ -747,7 +744,7 @@ jshort JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttCli error_cleanup: if (unsub_ack) { - mqtt_jni_async_callback_destroy(unsub_ack); + s_mqtt_jni_async_callback_destroy(unsub_ack); } return 0; } @@ -777,7 +774,7 @@ jshort JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttCli return 0; } - struct mqtt_jni_async_callback *pub_ack = mqtt_jni_async_callback_new(connection, jni_ack); + struct mqtt_jni_async_callback *pub_ack = s_mqtt_jni_async_callback_new(connection, jni_ack); if (!pub_ack) { aws_jni_throw_runtime_exception(env, "MqttClientConnection.mqtt_publish: Unable to allocate pub ack"); goto error_cleanup; @@ -804,7 +801,7 @@ jshort JNICALL Java_software_amazon_awssdk_crt_mqtt_MqttClientConnection_mqttCli error_cleanup: if (pub_ack) { - mqtt_jni_async_callback_destroy(pub_ack); + s_mqtt_jni_async_callback_destroy(pub_ack); } return 0; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 1874a6683..5a7ff21a6 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -10,6 +10,7 @@ import java.net.URI; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.*; @@ -159,9 +160,16 @@ public void testHttp2ConnectionPingExceptionPingDataLength() throws Exception { long time = conn.sendPing("123".getBytes()).get(5, TimeUnit.SECONDS); Assert.assertNotNull(time); } - } catch (CrtRuntimeException e) { - exception = true; - Assert.assertEquals(e.errorName, "AWS_ERROR_INVALID_ARGUMENT"); + } catch (ExecutionException e) { + try { + throw e.getCause(); + } catch (CrtRuntimeException causeException) { + exception = true; + Assert.assertEquals(causeException.errorName, "AWS_ERROR_INVALID_ARGUMENT"); + } catch (Throwable throwable) { + /* Unexpected exception */ + throwable.printStackTrace(); + } } Assert.assertTrue(exception); From 73d937e38f4c068679ead000f342a1edb09b54bd Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 3 Nov 2021 15:55:31 -0700 Subject: [PATCH 041/142] address comments from Mike --- .../crt/http/Http2ClientConnection.java | 7 ++++--- .../awssdk/crt/http/HttpClientConnection.java | 21 ++++++------------- src/native/http_request_response.c | 18 +++++++++------- .../crt/test/Http2ClientConnectionTest.java | 5 +---- .../crt/test/HttpRequestResponseFixture.java | 16 +++++--------- 5 files changed, 27 insertions(+), 40 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index 08d1bc4a9..121a77aaf 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -191,9 +191,6 @@ public Http2Stream makeRequest(HttpRequest request, HttpStreamResponseHandler st Http2Stream stream = http2ClientConnectionMakeRequest(getNativeHandle(), request.marshalForJni(), request.getBodyStream(), new HttpStreamResponseHandlerNativeAdapter(streamHandler)); - if (stream == null || stream.isNull()) { - throw new CrtRuntimeException(awsLastError()); - } return stream; } @@ -204,6 +201,10 @@ public Http2Stream makeRequest(HttpRequest request, HttpStreamResponseHandler st * Native methods ******************************************************************************/ + private static native Http2Stream http2ClientConnectionMakeRequest(long connectionBinding, byte[] marshalledRequest, + HttpRequestBodyStream bodyStream, HttpStreamResponseHandlerNativeAdapter responseHandler) + throws CrtRuntimeException; + private static native void http2ClientConnectionChangeSettings(long connectionBinding, AsyncCallback completedCallback, long[] marshalledSettings) throws CrtRuntimeException; diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index fd13c497b..4b4637fea 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -14,7 +14,6 @@ import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpStream; -import software.amazon.awssdk.crt.http.Http2Stream; import static software.amazon.awssdk.crt.CRT.awsLastError; @@ -76,18 +75,15 @@ protected HttpClientConnection(long connectionBinding) { * @return The HttpStream that represents this Request/Response Pair. It can be closed at any time during the * request/response, but must be closed by the user thread making this request when it's done. */ - public HttpStream makeRequest(HttpRequest request, HttpStreamResponseHandler streamHandler) throws CrtRuntimeException { + public HttpStream makeRequest(HttpRequest request, HttpStreamResponseHandler streamHandler) + throws CrtRuntimeException { if (isNull()) { throw new IllegalStateException("HttpClientConnection has been closed, can't make requests on it."); } - HttpStream stream = getVersion() == ProtocolVersion.HTTP_2 - ? http2ClientConnectionMakeRequest(getNativeHandle(), request.marshalForJni(), request.getBodyStream(), - new HttpStreamResponseHandlerNativeAdapter(streamHandler)) - : httpClientConnectionMakeRequest(getNativeHandle(), request.marshalForJni(), request.getBodyStream(), - new HttpStreamResponseHandlerNativeAdapter(streamHandler)); - if (stream == null || stream.isNull()) { - throw new CrtRuntimeException(awsLastError()); - } + HttpStream stream = httpClientConnectionMakeRequest(getNativeHandle(), + request.marshalForJni(), + request.getBodyStream(), + new HttpStreamResponseHandlerNativeAdapter(streamHandler)); return stream; } @@ -145,11 +141,6 @@ private static native HttpStream httpClientConnectionMakeRequest(long connection HttpRequestBodyStream bodyStream, HttpStreamResponseHandlerNativeAdapter responseHandler) throws CrtRuntimeException; - protected static native Http2Stream http2ClientConnectionMakeRequest(long connectionBinding, - byte[] marshalledRequest, - HttpRequestBodyStream bodyStream, - HttpStreamResponseHandlerNativeAdapter responseHandler) throws CrtRuntimeException; - private static native void httpClientConnectionShutdown(long connectionBinding) throws CrtRuntimeException; private static native void httpClientConnectionReleaseManaged(long connectionBinding) throws CrtRuntimeException; diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index e33ba74a3..2ac78aef0 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -51,6 +51,9 @@ static jobject s_java_http_stream_from_native_new(JNIEnv *env, void *opaque, enu default: aws_raise_error(AWS_ERROR_UNIMPLEMENTED); } + if (!stream) { + aws_jni_throw_runtime_exception(env, "Failed to create Java stream from native stream"); + } return stream; } @@ -300,12 +303,12 @@ static jobject s_make_request_general( struct aws_http_connection *native_conn = connection_binding->connection; if (!native_conn) { - aws_jni_throw_runtime_exception(env, "HttpClientConnection.MakeRequest: Invalid aws_http_connection"); + aws_jni_throw_null_pointer_exception(env, "HttpClientConnection.MakeRequest: Invalid aws_http_connection"); return (jobject)NULL; } if (!jni_http_response_callback_handler) { - aws_jni_throw_runtime_exception( + aws_jni_throw_illegal_argument_exception( env, "HttpClientConnection.MakeRequest: Invalid jni_http_response_callback_handler"); return (jobject)NULL; } @@ -362,6 +365,7 @@ static jobject s_make_request_general( callback_data will clean itself up when stream completes. */ aws_http_connection_close(native_conn); aws_http_stream_release(callback_data->native_stream); + /* Java exception has already been raised. */ return NULL; } @@ -385,7 +389,7 @@ JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnect AWS_HTTP_VERSION_1_1); } -JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection_http2ClientConnectionMakeRequest( +JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionMakeRequest( JNIEnv *env, jclass jni_class, jlong jni_connection, @@ -600,7 +604,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2Stream_http2Str struct aws_http_stream *stream = cb_data->native_stream; if (stream == NULL) { - aws_jni_throw_runtime_exception(env, "HttpStream is null."); + aws_jni_throw_null_pointer_exception(env, "Http2Stream is null."); return; } @@ -706,7 +710,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio return; } if (!java_async_callback) { - aws_jni_throw_null_pointer_exception( + aws_jni_throw_illegal_argument_exception( env, "Http2ClientConnection.http2ClientConnectionChangeSettings: Invalid async callback"); return; } @@ -781,12 +785,12 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio struct aws_http_connection *native_conn = connection_binding->connection; if (!native_conn) { - aws_jni_throw_runtime_exception( + aws_jni_throw_null_pointer_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid aws_http_connection"); return; } if (!java_async_callback) { - aws_jni_throw_null_pointer_exception( + aws_jni_throw_illegal_argument_exception( env, "Http2ClientConnection.http2ClientConnectionSendPing: Invalid async callback"); return; } diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 5a7ff21a6..8cac3b6da 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -23,10 +23,7 @@ import software.amazon.awssdk.crt.CrtResource; public class Http2ClientConnectionTest extends HttpClientTestFixture { - /** - * @TODO: maybe use a echo server to verify the frames are sent properly??? We - * have tests in C, not sure if it's worth to add it for java... - */ + private final static String HOST = "https://httpbin.org"; private final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java index 89bb46e2c..86a67ea5d 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -166,18 +166,12 @@ public void onResponseComplete(HttpStream stream, int errorCode) { stream.close(); } }; - if (expectedVersion == HttpClientConnection.ProtocolVersion.HTTP_2) { - try (Http2ClientConnection h2Connection = (Http2ClientConnection) conn) { - Http2Stream h2Stream = h2Connection.makeRequest(request, streamHandler); - h2Stream.activate(); - } - } else { - HttpStream stream = conn.makeRequest(request, streamHandler); - stream.activate(); - if (chunkedData != null) { - stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); - } + HttpStream stream = conn.makeRequest(request, streamHandler); + stream.activate(); + if (chunkedData != null) { + stream.writeChunk(chunkedData, true).get(5, TimeUnit.SECONDS); } + // Give the request up to 60 seconds to complete, otherwise throw a // TimeoutException reqCompleted.get(60, TimeUnit.SECONDS); From d1a9e13cdc3bf2a220d03b9e7c1172f51173bd07 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 3 Nov 2021 16:00:56 -0700 Subject: [PATCH 042/142] Remove s from the name of structure --- src/native/http_request_response.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 2ac78aef0..1eefbefee 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -647,12 +647,12 @@ JNIEXPORT jshort JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnecti } return (jshort)aws_http_connection_get_version(native_conn); } -struct s_aws_http2_callback_data { +struct aws_http2_callback_data { JavaVM *jvm; jobject async_callback; }; -static void s_cleanup_http2_callback_data(struct s_aws_http2_callback_data *callback_data) { +static void s_cleanup_http2_callback_data(struct aws_http2_callback_data *callback_data) { JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); @@ -662,12 +662,12 @@ static void s_cleanup_http2_callback_data(struct s_aws_http2_callback_data *call aws_mem_release(aws_jni_get_allocator(), callback_data); } -static struct s_aws_http2_callback_data *s_new_http2_callback_data( +static struct aws_http2_callback_data *s_new_http2_callback_data( JNIEnv *env, struct aws_allocator *allocator, jobject async_callback) { - struct s_aws_http2_callback_data *callback_data = - aws_mem_calloc(allocator, 1, sizeof(struct s_aws_http2_callback_data)); + struct aws_http2_callback_data *callback_data = + aws_mem_calloc(allocator, 1, sizeof(struct aws_http2_callback_data)); jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); AWS_FATAL_ASSERT(jvmresult == 0); @@ -679,7 +679,7 @@ static struct s_aws_http2_callback_data *s_new_http2_callback_data( static void s_on_settings_completed(struct aws_http_connection *http2_connection, int error_code, void *user_data) { (void)http2_connection; - struct s_aws_http2_callback_data *callback_data = user_data; + struct aws_http2_callback_data *callback_data = user_data; JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); if (error_code) { jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); @@ -715,7 +715,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio return; } struct aws_allocator *allocator = aws_jni_get_allocator(); - struct s_aws_http2_callback_data *callback_data = s_new_http2_callback_data(env, allocator, java_async_callback); + struct aws_http2_callback_data *callback_data = s_new_http2_callback_data(env, allocator, java_async_callback); /* We marshalled each setting to two long integers, the long list will be number of settings times two */ const size_t len = (*env)->GetArrayLength(env, java_marshalled_settings); @@ -754,7 +754,7 @@ static void s_on_ping_completed( int error_code, void *user_data) { (void)http2_connection; - struct s_aws_http2_callback_data *callback_data = user_data; + struct aws_http2_callback_data *callback_data = user_data; JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); if (error_code) { jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); @@ -799,7 +799,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio struct aws_byte_cursor *ping_cur_pointer = NULL; struct aws_byte_cursor ping_cur; AWS_ZERO_STRUCT(ping_cur); - struct s_aws_http2_callback_data *callback_data = s_new_http2_callback_data(env, allocator, java_async_callback); + struct aws_http2_callback_data *callback_data = s_new_http2_callback_data(env, allocator, java_async_callback); if (ping_data) { ping_cur = aws_jni_byte_cursor_from_jbyteArray_acquire(env, ping_data); From d2b70c6d221c8e2eef11c77edea320b291649f71 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 3 Nov 2021 16:31:10 -0700 Subject: [PATCH 043/142] use the tag --- crt/aws-c-http | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crt/aws-c-http b/crt/aws-c-http index d8cff2e16..0426a065b 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit d8cff2e16b125c5aad8bda21cf1bd85b8147968b +Subproject commit 0426a065b77e99ce8dbbe2e89f6e9fae35840728 From feec4a939e5d107a9b63db0ef474b41a7bc25276 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 3 Nov 2021 16:37:35 -0700 Subject: [PATCH 044/142] address comments --- src/native/http_request_utils.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/native/http_request_utils.c b/src/native/http_request_utils.c index 9f3c9a0f4..1c56a55b8 100644 --- a/src/native/http_request_utils.c +++ b/src/native/http_request_utils.c @@ -225,6 +225,7 @@ static inline enum aws_http_version s_unmarshal_http_request_to_get_version(stru static inline int s_unmarshal_http_request(struct aws_http_message *message, struct aws_byte_cursor *request_blob) { uint32_t field_len = 0; if (aws_http_message_get_protocol_version(message) != AWS_HTTP_VERSION_2) { + /* HTTP/1 puts method and path first, but those are just headers in HTTP/2 */ if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } @@ -298,6 +299,12 @@ int aws_apply_java_http_request_changes_to_native_request( } (*env)->ReleasePrimitiveArrayCritical(env, marshalled_request, marshalled_request_data, 0); + if (result) { + aws_jni_throw_runtime_exception( + env, "HttpRequest.applyChangesToNativeRequest: %s\n", aws_error_debug_str(aws_last_error())); + return result; + } + if (jni_body_stream) { struct aws_input_stream *body_stream = aws_input_stream_new_from_java_http_request_body_stream(aws_jni_get_allocator(), env, jni_body_stream); @@ -305,11 +312,6 @@ int aws_apply_java_http_request_changes_to_native_request( aws_http_message_set_body_stream(message, body_stream); } - if (result) { - aws_jni_throw_runtime_exception( - env, "HttpRequest.applyChangesToNativeRequest: %s\n", aws_error_debug_str(aws_last_error())); - } - return result; } From 5d9d9edda06798a30c6fa29579ac68cb3e82a735 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 3 Nov 2021 16:53:53 -0700 Subject: [PATCH 045/142] the empty path and method should be read out --- src/native/http_request_utils.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/native/http_request_utils.c b/src/native/http_request_utils.c index 1c56a55b8..e05dd1f49 100644 --- a/src/native/http_request_utils.c +++ b/src/native/http_request_utils.c @@ -225,7 +225,7 @@ static inline enum aws_http_version s_unmarshal_http_request_to_get_version(stru static inline int s_unmarshal_http_request(struct aws_http_message *message, struct aws_byte_cursor *request_blob) { uint32_t field_len = 0; if (aws_http_message_get_protocol_version(message) != AWS_HTTP_VERSION_2) { - /* HTTP/1 puts method and path first, but those are just headers in HTTP/2 */ + /* HTTP/1 puts method and path first, but those are empty in HTTP/2 */ if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } @@ -247,6 +247,14 @@ static inline int s_unmarshal_http_request(struct aws_http_message *message, str if (result != AWS_OP_SUCCESS) { return AWS_OP_ERR; } + } else { + /* Read empty method and path from the marshalled request */ + if (!aws_byte_cursor_read_be32(request_blob, &field_len) || field_len) { + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } + if (!aws_byte_cursor_read_be32(request_blob, &field_len) || field_len) { + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } } while (request_blob->len) { if (!aws_byte_cursor_read_be32(request_blob, &field_len)) { From 0f8506a6f2c596cc1f983aa1a881dac3d3af65ce Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 3 Nov 2021 17:12:10 -0700 Subject: [PATCH 046/142] add comments around to make it more clear --- src/native/http_request_utils.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/native/http_request_utils.c b/src/native/http_request_utils.c index e05dd1f49..ab0138447 100644 --- a/src/native/http_request_utils.c +++ b/src/native/http_request_utils.c @@ -213,7 +213,20 @@ int aws_marshal_http_headers_to_dynamic_buffer( return AWS_OP_SUCCESS; } - +/** + * Unmarshal the request from java. + * + * Version is as int: [4-bytes BE] + * + * Each string field is: [4-bytes BE] [variable length bytes specified + * by the previous field] + * + * Each request is like: [version][method][path][header name-value + * pairs] + * + * s_unmarshal_http_request_to_get_version to get the version field, which is a 4 byte int. + * s_unmarshal_http_request_without_version to get the whole request without version field. + */ static inline enum aws_http_version s_unmarshal_http_request_to_get_version(struct aws_byte_cursor *request_blob) { uint32_t version = 0; if (!aws_byte_cursor_read_be32(request_blob, &version)) { @@ -222,7 +235,9 @@ static inline enum aws_http_version s_unmarshal_http_request_to_get_version(stru return version; } -static inline int s_unmarshal_http_request(struct aws_http_message *message, struct aws_byte_cursor *request_blob) { +static inline int s_unmarshal_http_request_without_version( + struct aws_http_message *message, + struct aws_byte_cursor *request_blob) { uint32_t field_len = 0; if (aws_http_message_get_protocol_version(message) != AWS_HTTP_VERSION_2) { /* HTTP/1 puts method and path first, but those are empty in HTTP/2 */ @@ -303,7 +318,7 @@ int aws_apply_java_http_request_changes_to_native_request( if (version != aws_http_message_get_protocol_version(message)) { result = aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } else { - result = s_unmarshal_http_request(message, &marshalled_cur); + result = s_unmarshal_http_request_without_version(message, &marshalled_cur); } (*env)->ReleasePrimitiveArrayCritical(env, marshalled_request, marshalled_request_data, 0); @@ -342,7 +357,7 @@ struct aws_http_message *aws_http_request_new_from_java_http_request( if (version != aws_http_message_get_protocol_version(request)) { result = aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } else { - result = s_unmarshal_http_request(request, &marshalled_cur); + result = s_unmarshal_http_request_without_version(request, &marshalled_cur); } (*env)->ReleasePrimitiveArrayCritical(env, marshalled_request, marshalled_request_data, 0); From 57f1a642909491ec9b6c2cec9b52d2bab01b1c66 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 4 Nov 2021 15:28:35 -0700 Subject: [PATCH 047/142] It's working! TIME TO FIND Out how to run java now --- pom.xml | 6 + .../amazon/awssdk/crt/test/Elasticurl.java | 292 ++++++++++++------ 2 files changed, 196 insertions(+), 102 deletions(-) diff --git a/pom.xml b/pom.xml index 937e888dc..181b6e303 100644 --- a/pom.xml +++ b/pom.xml @@ -284,6 +284,12 @@ 1.4 test + + org.apache.commons + commons-compress + 1.19 + test + diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index 14d78efd0..e0a33d3e4 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -1,9 +1,15 @@ package software.amazon.awssdk.crt.test; +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; import java.net.URI; -import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -16,14 +22,23 @@ import software.amazon.awssdk.crt.Log; import software.amazon.awssdk.crt.Log.LogLevel; import software.amazon.awssdk.crt.http.HttpVersion; +import software.amazon.awssdk.crt.http.Http2Request; +import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpRequestBase; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.EventLoopGroup; import software.amazon.awssdk.crt.io.HostResolver; import software.amazon.awssdk.crt.io.SocketOptions; import software.amazon.awssdk.crt.io.TlsContext; import software.amazon.awssdk.crt.io.TlsContextOptions; +import software.amazon.awssdk.crt.utils.ByteBufferUtils; public class Elasticurl { @@ -40,91 +55,39 @@ static CommandLine parseArgs(String args[]) { Options cliOpts = new Options(); cliOpts.addOption("h", "help", false, "show this help message and exit"); - cliOpts.addOption(Option.builder() - .longOpt("cacert") - .hasArg() - .argName("file") - .desc("path to a CA certificate file.") - .build()); - cliOpts.addOption(Option.builder() - .longOpt("capath") - .hasArg() - .argName("dir") - .desc("path to a directory containing CA files.") - .build()); - cliOpts.addOption(Option.builder() - .longOpt("cert") - .hasArg() - .argName("file") - .desc("path to a PEM encoded certificate to use with mTLS.") - .build()); - cliOpts.addOption(Option.builder() - .longOpt("key") - .hasArg() - .argName("file") - .desc("path to a PEM encoded private key that matches cert.") - .build()); - cliOpts.addOption(Option.builder() - .longOpt("connect_timeout") - .hasArg() - .argName("int") - .desc("time in milliseconds to wait for a connection.") - .build()); - cliOpts.addOption(Option.builder("H") - .longOpt("header") - .hasArgs() - .argName("str") - .desc("line to send as a header in format 'name:value'. May be specified multiple times.") - .build()); - cliOpts.addOption(Option.builder("d") - .longOpt("data") - .hasArg() - .argName("str") - .desc("data to send in POST or PUT.") - .build()); - cliOpts.addOption(Option.builder() - .longOpt("data_file") - .hasArg() - .argName("file") - .desc("file to send in POST or PUT.") - .build()); - cliOpts.addOption(Option.builder("M") - .longOpt("method") - .hasArg() - .argName("str") - .desc("request method. Default is GET)") - .build()); + cliOpts.addOption(Option.builder().longOpt("cacert").hasArg().argName("file") + .desc("path to a CA certificate file.").build()); + cliOpts.addOption(Option.builder().longOpt("capath").hasArg().argName("dir") + .desc("path to a directory containing CA files.").build()); + cliOpts.addOption(Option.builder().longOpt("cert").hasArg().argName("file") + .desc("path to a PEM encoded certificate to use with mTLS.").build()); + cliOpts.addOption(Option.builder().longOpt("key").hasArg().argName("file") + .desc("path to a PEM encoded private key that matches cert.").build()); + cliOpts.addOption(Option.builder().longOpt("connect_timeout").hasArg().argName("int") + .desc("time in milliseconds to wait for a connection.").build()); + cliOpts.addOption(Option.builder("H").longOpt("header").hasArgs().argName("str") + .desc("line to send as a header in format 'name:value'. May be specified multiple times.").build()); + cliOpts.addOption(Option.builder("d").longOpt("data").hasArg().argName("str") + .desc("data to send in POST or PUT.").build()); + cliOpts.addOption(Option.builder().longOpt("data_file").hasArg().argName("file") + .desc("file to send in POST or PUT.").build()); + cliOpts.addOption(Option.builder("M").longOpt("method").hasArg().argName("str") + .desc("request method. Default is GET)").build()); cliOpts.addOption("G", "get", false, "uses GET for request method."); cliOpts.addOption("P", "post", false, "uses POST for request method."); cliOpts.addOption("I", "head", false, "uses HEAD for request method."); cliOpts.addOption("i", "include", false, "includes headers in output."); cliOpts.addOption("k", "insecure", false, "turns off x.509 validation."); - cliOpts.addOption(Option.builder("o") - .longOpt("output") - .hasArg() - .argName("file") - .desc("dumps content-body to FILE instead of stdout.") - .build()); - cliOpts.addOption(Option.builder("t") - .longOpt("trace") - .hasArg() - .argName("file") - .desc("dumps logs to FILE instead of stderr.") - .build()); - cliOpts.addOption(Option.builder("p") - .longOpt("alpn") - .hasArgs() - .argName("str") - .desc("protocol for ALPN. May be specified multiple times.") - .build()); + cliOpts.addOption(Option.builder("o").longOpt("output").hasArg().argName("file") + .desc("dumps content-body to FILE instead of stdout.").build()); + cliOpts.addOption(Option.builder("t").longOpt("trace").hasArg().argName("file") + .desc("dumps logs to FILE instead of stderr.").build()); + cliOpts.addOption(Option.builder("p").longOpt("alpn").hasArgs().argName("str") + .desc("protocol for ALPN. May be specified multiple times.").build()); cliOpts.addOption(null, "http1_1", false, "HTTP/1.1 connection required."); cliOpts.addOption(null, "http2", false, "HTTP/2 connection required."); - cliOpts.addOption(Option.builder("v") - .longOpt("verbose") - .hasArg() - .argName("str") - .desc("logging level (ERROR|INFO|DEBUG|TRACE) default is none.") - .build()); + cliOpts.addOption(Option.builder("v").longOpt("verbose").hasArg().argName("str") + .desc("logging level (ERROR|INFO|DEBUG|TRACE) default is none.").build()); CommandLineParser cliParser = new DefaultParser(); CommandLine cli = null; @@ -144,6 +107,80 @@ static CommandLine parseArgs(String args[]) { return cli; } + private static HttpRequestBase buildHttpRequest(CommandLine cli, HttpVersion requiredVersion, URI uri) + throws Exception { + String method = cli.getOptionValue("method"); + if (cli.hasOption("get")) { + method = "GET"; + } else if (cli.hasOption("post")) { + method = "POST"; + } else if (cli.hasOption("head")) { + method = "HEAD"; + } + if (method == null) { + method = "GET"; + } + String pathAndQuery = uri.getQuery() == null ? uri.getPath() + "?" + uri.getQuery() : uri.getPath(); + + /* body */ + ByteBuffer tmpPayload = null; + if (cli.getOptionValue("data") != null) { + tmpPayload = ByteBuffer.wrap(cli.getOptionValue("data").getBytes()); + } else if (cli.getOptionValue("data_file") != null) { + Path path = Paths.get(cli.getOptionValue("data_file")); + tmpPayload = ByteBuffer.wrap(Files.readAllBytes(path)); + } + HttpRequestBodyStream payloadStream = null; + if (tmpPayload != null) { + final ByteBuffer payload = tmpPayload; + payloadStream = new HttpRequestBodyStream() { + @Override + public boolean sendRequestBody(ByteBuffer outBuffer) { + ByteBufferUtils.transferData(payload, outBuffer); + return payload.remaining() == 0; + } + + @Override + public boolean resetPosition() { + return true; + } + + @Override + public long getLength() { + return payload.capacity(); + } + }; + } + /* initial headers */ + HttpHeader[] headers = new HttpHeader[] {}; + HttpRequestBase request = requiredVersion == HttpVersion.HTTP2 ? new Http2Request(headers, payloadStream) + : new HttpRequest(method, pathAndQuery, headers, payloadStream); + + /* Version specific headers */ + if (requiredVersion == HttpVersion.HTTP2) { + request.addHeader(new HttpHeader(":method", method)); + request.addHeader(new HttpHeader(":scheme", uri.getScheme())); + request.addHeader(new HttpHeader(":authority", uri.getAuthority())); + request.addHeader(new HttpHeader(":path", pathAndQuery)); + } else { + request.addHeader(new HttpHeader("Host", uri.getHost())); + } + + /* General headers */ + request.addHeader(new HttpHeader("accept", "*/*")); + request.addHeader(new HttpHeader("user-agent", "elasticurl 1.0, Powered by the AWS Common Runtime.")); + + /* Customized headers */ + String[] customizedHeaders = cli.getOptionValues("header"); + if (customizedHeaders != null) { + for (String header : customizedHeaders) { + String[] pair = header.split(":"); + request.addHeader(new HttpHeader(pair[0], pair[1])); + } + } + return request; + } + public static void main(String args[]) throws Exception { CommandLine cli = parseArgs(args); @@ -151,13 +188,13 @@ public static void main(String args[]) throws Exception { String verbose = cli.getOptionValue("verbose"); if (verbose != null) { LogLevel logLevel = LogLevel.None; - if (verbose == "ERROR") { + if (verbose.equals("ERROR")) { logLevel = LogLevel.Error; - } else if (verbose == "INFO") { + } else if (verbose.equals("INFO")) { logLevel = LogLevel.Info; - } else if (verbose == "DEBUG") { + } else if (verbose.equals("DEBUG")) { logLevel = LogLevel.Debug; - } else if (verbose == "TRACE") { + } else if (verbose.equals("TRACE")) { logLevel = LogLevel.Trace; } else { exit(logLevel + " unsupported value for verbose option"); @@ -216,11 +253,11 @@ public static void main(String args[]) throws Exception { String[] alpn = cli.getOptionValues("alpn"); if (alpn == null) { if (requiredVersion == HttpVersion.HTTP1_1) { - alpn = new String[]{"http/1.1"}; + alpn = new String[] { "http/1.1" }; } else if (requiredVersion == HttpVersion.HTTP2) { - alpn = new String[]{"h2"}; + alpn = new String[] { "h2" }; } else { - alpn = new String[]{"h2", "http/1.1"}; + alpn = new String[] { "h2", "http/1.1" }; } } tlsOpts.alpnList = Arrays.asList(alpn); @@ -228,28 +265,79 @@ public static void main(String args[]) throws Exception { tlsCtx = new TlsContext(tlsOpts); } - CompletableFuture connMgrShutdownComplete = null; - + CompletableFuture connMgrShutdownComplete = null; + final BufferedOutputStream out = cli.getOptionValue("output") == null ? new BufferedOutputStream(System.out) + : new BufferedOutputStream(new FileOutputStream(cli.getOptionValue("output"))); try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); - HostResolver resolver = new HostResolver(eventLoopGroup); - ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); - SocketOptions socketOpts = new SocketOptions()) { - + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions socketOpts = new SocketOptions()) { + if (cli.getOptionValue("connect_timeout") != null) { + int timeout = Integer.parseInt(cli.getOptionValue("connect_timeout")); + socketOpts.connectTimeoutMs = timeout; + } HttpClientConnectionManagerOptions connMgrOpts = new HttpClientConnectionManagerOptions() - .withClientBootstrap(bootstrap) - .withSocketOptions(socketOpts) - .withTlsContext(tlsCtx) - .withUri(uri) - .withPort(port); + .withClientBootstrap(bootstrap).withSocketOptions(socketOpts).withTlsContext(tlsCtx) + .withUri(uri).withPort(port); try (HttpClientConnectionManager connMgr = HttpClientConnectionManager.create(connMgrOpts)) { connMgrShutdownComplete = connMgr.getShutdownCompleteFuture(); - try () { YOU ARE HERE - } - } finally { - if (connMgrShutdownComplete != null) { - connMgrShutdownComplete.get(); + try (HttpClientConnection conn = connMgr.acquireConnection().get(60, TimeUnit.SECONDS)) { + + final CompletableFuture reqCompleted = new CompletableFuture<>(); + HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { + boolean statusWritten = false; + + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + if (blockType == 1) { + /* Ignore informational headers */ + return; + } + if (cli.hasOption("include")) { + if (!statusWritten) { + System.out.println(String.format("Response Status: %d", responseStatusCode)); + statusWritten = true; + } + for (HttpHeader header : nextHeaders) { + System.out.println(header.getName() + ": " + header.getValue()); + } + } + } + + @Override + public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + try { + out.write(bodyBytesIn, 0, bodyBytesIn.length); + out.flush(); + } catch (Exception e) { + exit("Failed to write the body"); + } + return bodyBytesIn.length; + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + reqCompleted.complete(null); + stream.close(); + } + }; + HttpRequestBase request = buildHttpRequest(cli, requiredVersion, uri); + HttpStream stream = conn.makeRequest(request, streamHandler); + stream.activate(); + + // Give the request up to 60 seconds to complete, otherwise throw a + // TimeoutException + reqCompleted.get(60, TimeUnit.SECONDS); } + } catch (Exception e) { + throw new RuntimeException(e); + } + } finally { + out.close(); + if (connMgrShutdownComplete != null) { + connMgrShutdownComplete.get(); } } From c795b788c13b408150a5730f4fc11c40bc2e8bc9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 4 Nov 2021 16:02:47 -0700 Subject: [PATCH 048/142] I don't know... --- .builder/actions/aws_crt_java_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index c68b60146..b0fbcf5d0 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -16,5 +16,16 @@ def run(self, env): # Failed actions.append("exit 1") os.system("cat log.txt") + actions.append( + ['java', + '-classpath', + '\{source_dir\}/target/test-classes:\{source_dir\}/target/classes:~/.m2/repository/commons-cli/commons-cli/1.4/commons-cli-1.4.jar', + 'software.amazon.awssdk.crt.test.Elasticurl', + '-v', + 'TRACE', + '--http2', + '-o', + 'elastigirl_h2.png', + 'https://d1cz66xoahf9cl.cloudfront.net/elastigirl.png']) return Builder.Script(actions, name='aws-crt-java-test') From 13945b145dc8ace5a665f7eab24ff486f37da4a7 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 4 Nov 2021 16:28:05 -0700 Subject: [PATCH 049/142] let's see --- .builder/actions/aws_crt_java_test.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index b0fbcf5d0..d8df4b78f 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -10,19 +10,25 @@ def run(self, env): env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # Failed - actions.append("exit 1") - os.system("cat log.txt") + # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # # Failed + # actions.append("exit 1") + # os.system("cat log.txt") + # Three level up of the currect file path .builder/actions/__FILE__ + source_path = os.path.dirname( + os.path.dirname((os.path.dirname(__file__)))) + home_dir = os.path.expanduser("~") + print(source_path) actions.append( ['java', '-classpath', - '\{source_dir\}/target/test-classes:\{source_dir\}/target/classes:~/.m2/repository/commons-cli/commons-cli/1.4/commons-cli-1.4.jar', + '{source}/target/test-classes:{source}/target/classes:{home}/.m2/repository/commons-cli/commons-cli/1.4/commons-cli-1.4.jar'.format( + source=source_path, home=home_dir), 'software.amazon.awssdk.crt.test.Elasticurl', '-v', - 'TRACE', + 'ERROR', '--http2', '-o', 'elastigirl_h2.png', From 65481359ff3cebd8bc18e4163fb9fb9ec0208aa8 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 4 Nov 2021 16:40:36 -0700 Subject: [PATCH 050/142] next test --- .builder/actions/aws_crt_java_test.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index d8df4b78f..853c2ff74 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -22,16 +22,11 @@ def run(self, env): home_dir = os.path.expanduser("~") print(source_path) actions.append( - ['java', - '-classpath', - '{source}/target/test-classes:{source}/target/classes:{home}/.m2/repository/commons-cli/commons-cli/1.4/commons-cli-1.4.jar'.format( - source=source_path, home=home_dir), - 'software.amazon.awssdk.crt.test.Elasticurl', - '-v', - 'ERROR', - '--http2', - '-o', - 'elastigirl_h2.png', - 'https://d1cz66xoahf9cl.cloudfront.net/elastigirl.png']) + ["mvn", + "-e", + "exec:java", + "-Dexec.classpathScope=\"test\"", + "-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"", + "-Dexec.args=\"-v ERROR --http2 -i -o elastigril_h2.png https://d1cz66xoahf9cl.cloudfront.net/elastigirl.png\""]) return Builder.Script(actions, name='aws-crt-java-test') From 7c650458b5df6792247a3e15a0f81c8ee338b111 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 4 Nov 2021 17:16:04 -0700 Subject: [PATCH 051/142] use a runner to run elasticurl --- .builder/actions/aws_crt_java_test.py | 28 ++++++++-------------- .builder/actions/java_elasticurl_runner.py | 10 ++++++++ 2 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 .builder/actions/java_elasticurl_runner.py diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index 853c2ff74..2d6b28092 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -1,5 +1,6 @@ import Builder +import sys import os @@ -9,24 +10,15 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - - # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # # Failed - # actions.append("exit 1") - # os.system("cat log.txt") - # Three level up of the currect file path .builder/actions/__FILE__ - source_path = os.path.dirname( - os.path.dirname((os.path.dirname(__file__)))) - home_dir = os.path.expanduser("~") - print(source_path) + if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # Failed + actions.append("exit 1") + os.system("cat log.txt") + python = sys.executable actions.append( - ["mvn", - "-e", - "exec:java", - "-Dexec.classpathScope=\"test\"", - "-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"", - "-Dexec.args=\"-v ERROR --http2 -i -o elastigril_h2.png https://d1cz66xoahf9cl.cloudfront.net/elastigirl.png\""]) + [python, 'crt/aws-c-http/integration-testing/http_client_test.py', + python, '.builder/actions/java_elasticurl_runner.py']) return Builder.Script(actions, name='aws-crt-java-test') diff --git a/.builder/actions/java_elasticurl_runner.py b/.builder/actions/java_elasticurl_runner.py new file mode 100644 index 000000000..f61ec011b --- /dev/null +++ b/.builder/actions/java_elasticurl_runner.py @@ -0,0 +1,10 @@ +import sys +import os + +# Runner for elasticurl integration tests +elasticurl_args = sys.argv[1:] + +java_command = "mvn -e exec:java -Dexec.classpathScope=\"test\" -Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\" -Dexec.args=\"{args}\"".format( + args=" ".join(elasticurl_args)) + +os.system(java_command) From 958f8a9ace48db790c525dfec8cd44b018f27004 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 10:39:17 -0700 Subject: [PATCH 052/142] update the script --- .builder/actions/java_elasticurl_runner.py | 42 +++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/.builder/actions/java_elasticurl_runner.py b/.builder/actions/java_elasticurl_runner.py index f61ec011b..2d7baa056 100644 --- a/.builder/actions/java_elasticurl_runner.py +++ b/.builder/actions/java_elasticurl_runner.py @@ -1,10 +1,44 @@ import sys -import os +import subprocess +from subprocess import Popen, PIPE +TIMEOUT = 100 # Runner for elasticurl integration tests elasticurl_args = sys.argv[1:] -java_command = "mvn -e exec:java -Dexec.classpathScope=\"test\" -Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\" -Dexec.args=\"{args}\"".format( - args=" ".join(elasticurl_args)) +mvn_args = " ".join(elasticurl_args) -os.system(java_command) +java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', + '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] + +print(java_command) + +command_string = " ".join(java_command) +print(command_string) + + +def run_command(args_str): + # gather all stderr and stdout to a single string that we print only if things go wrong + process = subprocess.Popen( + args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + timedout = False + try: + output = process.communicate(timeout=TIMEOUT)[0] + except subprocess.TimeoutExpired: + timedout = True + process.kill() + output = process.communicate()[0] + finally: + if process.returncode != 0 or timedout: + print(args_str) + for line in output.splitlines(): + print(line.decode()) + if timedout: + raise RuntimeError("Timeout happened after {secs} secs from: {cmd}".format( + secs=TIMEOUT, cmd=args_str)) + else: + raise RuntimeError("Return code {code} from: {cmd}".format( + code=process.returncode, cmd=args_str)) + + +run_command(command_string) From e2503c9862a825119d8ab3cc79e98b037b403048 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 11:19:21 -0700 Subject: [PATCH 053/142] The builder folder is magic --- .builder/actions/aws_crt_java_test.py | 14 +++++++------- .../java_elasticurl_runner.py | 0 .../amazon/awssdk/crt/test/Elasticurl.java | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) rename .builder/{actions => integration-testing}/java_elasticurl_runner.py (100%) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index 2d6b28092..3a026c244 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -10,15 +10,15 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # Failed - actions.append("exit 1") - os.system("cat log.txt") + # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # # Failed + # actions.append("exit 1") + # os.system("cat log.txt") python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', - python, '.builder/actions/java_elasticurl_runner.py']) + python, '.builder/integration-testing/java_elasticurl_runner.py']) return Builder.Script(actions, name='aws-crt-java-test') diff --git a/.builder/actions/java_elasticurl_runner.py b/.builder/integration-testing/java_elasticurl_runner.py similarity index 100% rename from .builder/actions/java_elasticurl_runner.py rename to .builder/integration-testing/java_elasticurl_runner.py diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index e0a33d3e4..225704078 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -19,6 +19,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.Log; import software.amazon.awssdk.crt.Log.LogLevel; import software.amazon.awssdk.crt.http.HttpVersion; @@ -319,7 +320,11 @@ public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { @Override public void onResponseComplete(HttpStream stream, int errorCode) { - reqCompleted.complete(null); + if (errorCode!=0) { + reqCompleted.completeExceptionally(new CrtRuntimeException(errorCode)); + } else{ + reqCompleted.complete(null); + } stream.close(); } }; From 2a6f61f87332c99e23b513090a5b93979b08a67b Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 11:21:23 -0700 Subject: [PATCH 054/142] hmmm --- .builder/actions/aws_crt_java_test.py | 2 +- .../java_elasticurl_runner.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {.builder/integration-testing => integration-testing}/java_elasticurl_runner.py (100%) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index 3a026c244..9e38e6401 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -19,6 +19,6 @@ def run(self, env): python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', - python, '.builder/integration-testing/java_elasticurl_runner.py']) + python, 'integration-testing/java_elasticurl_runner.py']) return Builder.Script(actions, name='aws-crt-java-test') diff --git a/.builder/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py similarity index 100% rename from .builder/integration-testing/java_elasticurl_runner.py rename to integration-testing/java_elasticurl_runner.py From c8175041efe43625701b0465de455f69fa0ec793 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 11:25:58 -0700 Subject: [PATCH 055/142] normal test first --- .builder/actions/aws_crt_java_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index 9e38e6401..89c888535 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -10,12 +10,12 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # # Failed - # actions.append("exit 1") - # os.system("cat log.txt") + if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # Failed + actions.append("exit 1") + os.system("cat log.txt") python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', From aacde74c6a070b5fba9714e29f8bd905163127ab Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 12:36:27 -0700 Subject: [PATCH 056/142] runner to take care the "\"" stuff --- integration-testing/java_elasticurl_runner.py | 11 ++++++++--- .../software/amazon/awssdk/crt/test/Elasticurl.java | 7 +++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index 2d7baa056..6e904ed4c 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -4,17 +4,22 @@ TIMEOUT = 100 # Runner for elasticurl integration tests + elasticurl_args = sys.argv[1:] +for index, arg in enumerate(elasticurl_args): + if " " in arg: + elasticurl_args[index] = "\\\"{}\\\"".format(arg) + if arg[0] == "\"" and arg[-1] == "\"": + elasticurl_args[index] = "\\\"{}\\\"".format(arg[1:-1]) + mvn_args = " ".join(elasticurl_args) + java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] -print(java_command) - command_string = " ".join(java_command) -print(command_string) def run_command(args_str): diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index 225704078..db5430a88 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -121,7 +121,10 @@ private static HttpRequestBase buildHttpRequest(CommandLine cli, HttpVersion req if (method == null) { method = "GET"; } - String pathAndQuery = uri.getQuery() == null ? uri.getPath() + "?" + uri.getQuery() : uri.getPath(); + String pathAndQuery = uri.getQuery() == null ? uri.getPath() : uri.getPath() + "?" + uri.getQuery(); + if (pathAndQuery.length() == 0) { + pathAndQuery = "/"; + } /* body */ ByteBuffer tmpPayload = null; @@ -176,7 +179,7 @@ public long getLength() { if (customizedHeaders != null) { for (String header : customizedHeaders) { String[] pair = header.split(":"); - request.addHeader(new HttpHeader(pair[0], pair[1])); + request.addHeader(new HttpHeader(pair[0].trim(), pair[1].trim())); } } return request; From ec83d5c1c0bec390ab64285639ff43c10cb55371 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 12:46:59 -0700 Subject: [PATCH 057/142] protocol version -> Http version --- .../amazon/awssdk/crt/http/Http2Request.java | 4 +- .../awssdk/crt/http/HttpClientConnection.java | 48 ++----------------- .../crt/http/HttpClientConnectionManager.java | 6 +-- .../HttpClientConnectionManagerOptions.java | 10 ++-- .../amazon/awssdk/crt/http/HttpRequest.java | 4 +- .../awssdk/crt/http/HttpRequestBase.java | 4 +- .../amazon/awssdk/crt/http/HttpVersion.java | 44 ++++++++++------- .../crt/test/Http2ClientConnectionTest.java | 2 +- .../crt/test/Http2RequestResponseTest.java | 3 +- .../crt/test/HttpClientTestFixture.java | 6 +-- .../crt/test/HttpRequestResponseFixture.java | 3 +- .../crt/test/HttpRequestResponseTest.java | 6 +-- 12 files changed, 57 insertions(+), 83 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java index 282c18517..97daec506 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Request.java @@ -5,7 +5,7 @@ package software.amazon.awssdk.crt.http; -import software.amazon.awssdk.crt.http.HttpClientConnection.ProtocolVersion; +import software.amazon.awssdk.crt.http.HttpVersion; /** * Represents a single Client Request to be sent on a HTTP connection @@ -28,7 +28,7 @@ public Http2Request() { */ public Http2Request(HttpHeader[] headers, HttpRequestBodyStream bodyStream) { super(headers, bodyStream); - this.version = ProtocolVersion.HTTP_2; + this.version = HttpVersion.HTTP_2; } } diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index f616fc8d3..f0ca94b18 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -6,14 +6,10 @@ package software.amazon.awssdk.crt.http; import java.util.concurrent.CompletableFuture; -import java.util.Map; -import java.util.HashMap; import software.amazon.awssdk.crt.CRT; import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.CrtRuntimeException; -import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; -import software.amazon.awssdk.crt.http.HttpStream; import static software.amazon.awssdk.crt.CRT.awsLastError; @@ -28,42 +24,6 @@ * threads. */ public class HttpClientConnection extends CrtResource { - /** - * HTTP protocol version. - */ - public enum ProtocolVersion { - HTTP_1_0(1), HTTP_1_1(2), HTTP_2(3); - - private int protocolVersion; - private static Map enumMapping = buildEnumMapping(); - - ProtocolVersion(int value) { - protocolVersion = value; - } - - public static ProtocolVersion getEnumValueFromInteger(int value) { - ProtocolVersion enumValue = enumMapping.get(value); - if (enumValue != null) { - return enumValue; - } - - throw new RuntimeException("Illegal signature type value in signing configuration"); - } - - private static Map buildEnumMapping() { - Map enumMapping = new HashMap(); - enumMapping.put(HTTP_1_0.getValue(), HTTP_1_0); - enumMapping.put(HTTP_1_1.getValue(), HTTP_1_1); - enumMapping.put(HTTP_2.getValue(), HTTP_2); - - return enumMapping; - } - - public int getValue() { - return protocolVersion; - } - } - protected HttpClientConnection(long connectionBinding) { acquireNativeHandle(connectionBinding); } @@ -121,9 +81,9 @@ public void shutdown() { httpClientConnectionShutdown(getNativeHandle()); } - public ProtocolVersion getVersion() { + public HttpVersion getVersion() { short version = httpClientConnectionGetVersion(getNativeHandle()); - return ProtocolVersion.getEnumValueFromInteger((int) version); + return HttpVersion.getEnumValueFromInteger((int) version); }; /** Called from Native when a new connection is acquired **/ @@ -133,8 +93,8 @@ private static void onConnectionAcquired(CompletableFuture acquireFuture.completeExceptionally(new HttpException(errorCode)); return; } - if (ProtocolVersion.getEnumValueFromInteger( - (int) httpClientConnectionGetVersion(nativeConnectionBinding)) == ProtocolVersion.HTTP_2) { + if (HttpVersion.getEnumValueFromInteger( + (int) httpClientConnectionGetVersion(nativeConnectionBinding)) == HttpVersion.HTTP_2) { HttpClientConnection h2Conn = new Http2ClientConnection(nativeConnectionBinding); acquireFuture.complete(h2Conn); } else { diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java index c9c55a21c..75a3dbc61 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java @@ -32,7 +32,7 @@ public class HttpClientConnectionManager extends CrtResource { private final int port; private final int maxConnections; private final CompletableFuture shutdownComplete = new CompletableFuture<>(); - private final HttpClientConnection.ProtocolVersion expectedProtocolVersion; + private final HttpVersion expectedHttpVersion; public static HttpClientConnectionManager create(HttpClientConnectionManagerOptions options) { return new HttpClientConnectionManager(options); @@ -77,7 +77,7 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options) this.uri = uri; this.port = port; this.maxConnections = maxConnections; - this.expectedProtocolVersion = options.getExpectedProtocolVersion(); + this.expectedHttpVersion = options.getExpectedHttpVersion(); int proxyConnectionType = 0; String proxyHost = null; @@ -124,7 +124,7 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options) options.getMaxConnectionIdleInMilliseconds(), monitoringThroughputThresholdInBytesPerSecond, monitoringFailureIntervalInSeconds, - expectedProtocolVersion.getValue())); + expectedHttpVersion.getValue())); /* we don't need to add a reference to socketOptions since it's copied during connection manager construction */ addReferenceTo(clientBootstrap); diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java index 142d2b9a2..aa79f754f 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java @@ -29,7 +29,7 @@ public class HttpClientConnectionManagerOptions { private boolean manualWindowManagement = false; private HttpMonitoringOptions monitoringOptions; private long maxConnectionIdleInMilliseconds = 0; - private HttpClientConnection.ProtocolVersion expectedProtocolVersion = HttpClientConnection.ProtocolVersion.HTTP_1_1; + private HttpVersion expectedHttpVersion = HttpVersion.HTTP_1_1; public HttpClientConnectionManagerOptions() { } @@ -203,16 +203,16 @@ public HttpClientConnectionManagerOptions withManualWindowManagement(boolean man * * @return this */ - public HttpClientConnectionManagerOptions withExpectedProtocolVersion(HttpClientConnection.ProtocolVersion expectedProtocolVersion) { - this.expectedProtocolVersion = expectedProtocolVersion; + public HttpClientConnectionManagerOptions withExpectedHttpVersion(HttpVersion expectedHttpVersion) { + this.expectedHttpVersion = expectedHttpVersion; return this; } /** * @return Return the expected HTTP protocol version. */ - public HttpClientConnection.ProtocolVersion getExpectedProtocolVersion() { - return expectedProtocolVersion; + public HttpVersion getExpectedHttpVersion() { + return expectedHttpVersion; } /** diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java b/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java index 7f9d37dc8..be996cde8 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpRequest.java @@ -6,7 +6,7 @@ package software.amazon.awssdk.crt.http; import software.amazon.awssdk.crt.CrtRuntimeException; -import software.amazon.awssdk.crt.http.HttpClientConnection.ProtocolVersion; +import software.amazon.awssdk.crt.http.HttpVersion; import java.nio.ByteBuffer; @@ -54,7 +54,7 @@ public HttpRequest(String method, String encodedPath, HttpHeader[] headers, Http if (marshalledRequest.remaining() < BUFFER_INT_SIZE * 3) { throw new CrtRuntimeException("Invalid marshalled request object."); } - this.version = ProtocolVersion.getEnumValueFromInteger(marshalledRequest.getInt()); + this.version = HttpVersion.getEnumValueFromInteger(marshalledRequest.getInt()); int methodLength = marshalledRequest.getInt(); byte[] methodBlob = new byte[methodLength]; diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java index 8088d17a5..9585f4a25 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java @@ -4,7 +4,7 @@ import java.util.ArrayList; import java.util.List; -import software.amazon.awssdk.crt.http.HttpClientConnection.ProtocolVersion; +import software.amazon.awssdk.crt.http.HttpVersion; import java.nio.charset.Charset; import java.nio.ByteBuffer; import java.util.Collections; @@ -14,7 +14,7 @@ public class HttpRequestBase { protected final static int BUFFER_INT_SIZE = 4; protected List headers; protected HttpRequestBodyStream bodyStream; - protected ProtocolVersion version = ProtocolVersion.HTTP_1_1; + protected HttpVersion version = HttpVersion.HTTP_1_1; protected String method; protected String encodedPath; diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java b/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java index 74bed3b66..9b0311b5b 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java @@ -1,33 +1,45 @@ -/* - * Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. */ package software.amazon.awssdk.crt.http; +import java.util.Map; +import java.util.HashMap; + public enum HttpVersion { UNKNOWN(0), - HTTP1_0(1), - HTTP1_1(2), - HTTP2(3); + HTTP_1_0(1), + HTTP_1_1(2), + HTTP_2(3); private int value; + private static Map enumMapping = buildEnumMapping(); HttpVersion(int value) { this.value = value; } + public static HttpVersion getEnumValueFromInteger(int value) { + HttpVersion enumValue = enumMapping.get(value); + if (enumValue != null) { + return enumValue; + } + + throw new RuntimeException("Illegal signature type value in signing configuration"); + } + + private static Map buildEnumMapping() { + Map enumMapping = new HashMap(); + enumMapping.put(HTTP_1_0.getValue(), HTTP_1_0); + enumMapping.put(HTTP_1_1.getValue(), HTTP_1_1); + enumMapping.put(HTTP_2.getValue(), HTTP_2); + + return enumMapping; + } + public int getValue() { return value; } diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index 8cac3b6da..c4f185ec8 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -25,7 +25,7 @@ public class Http2ClientConnectionTest extends HttpClientTestFixture { private final static String HOST = "https://httpbin.org"; - private final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; + private final static HttpVersion EXPECTED_VERSION = HttpVersion.HTTP_2; @Test public void testHttp2ConnectionGetVersion() throws Exception { diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index 8a04109ac..9af47e841 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -11,6 +11,7 @@ import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpVersion; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.Http2Request; import software.amazon.awssdk.crt.http.HttpStream; @@ -30,7 +31,7 @@ public class Http2RequestResponseTest extends HttpRequestResponseFixture { private final static String HOST = "https://httpbin.org"; - private final static HttpClientConnection.ProtocolVersion EXPECTED_VERSION = HttpClientConnection.ProtocolVersion.HTTP_2; + private final static HttpVersion EXPECTED_VERSION = HttpVersion.HTTP_2; private Http2Request getHttp2Request(String method, String endpoint, String path, String requestBody) throws Exception { diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java index 0a2cb93f3..784ebad64 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java @@ -5,9 +5,9 @@ package software.amazon.awssdk.crt.test; -import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpVersion; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.EventLoopGroup; import software.amazon.awssdk.crt.io.HostResolver; @@ -58,8 +58,8 @@ public TlsContext createHttpClientTlsContext(TlsContextOptions tlsOpts) { } public HttpClientConnectionManager createConnectionPoolManager(URI uri, - HttpClientConnection.ProtocolVersion expectedVersion) { - if (expectedVersion == HttpClientConnection.ProtocolVersion.HTTP_2) { + HttpVersion expectedVersion) { + if (expectedVersion == HttpVersion.HTTP_2) { return createHTTP2ConnectionPoolManager(uri); } else { return createHTTP1ConnectionPoolManager(uri); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java index 725c044e9..c619843b8 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseFixture.java @@ -11,6 +11,7 @@ import software.amazon.awssdk.crt.http.Http2Request; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpVersion; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; import software.amazon.awssdk.crt.http.HttpRequestBase; @@ -88,7 +89,7 @@ public String calculateBodyHash(ByteBuffer bodyBuffer) throws NoSuchAlgorithmExc } public TestHttpResponse getResponse(URI uri, HttpRequestBase request, byte[] chunkedData, - HttpClientConnection.ProtocolVersion expectedVersion) throws Exception { + HttpVersion expectedVersion) throws Exception { boolean actuallyConnected = false; final CompletableFuture reqCompleted = new CompletableFuture<>(); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java index 232ee53a5..aa9380463 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java @@ -76,9 +76,9 @@ public boolean resetPosition() { try { if (useChunkedEncoding) { response = getResponse(uri, request, requestBody.getBytes(UTF8), - HttpClientConnection.ProtocolVersion.HTTP_1_1); + HttpVersion.HTTP_1_1); } else { - response = getResponse(uri, request, null, HttpClientConnection.ProtocolVersion.HTTP_1_1); + response = getResponse(uri, request, null, HttpVersion.HTTP_1_1); } } catch (Exception ex) { // do nothing just let it retry @@ -247,7 +247,7 @@ public void testHttpRequestUnActivated() throws Exception { CompletableFuture shutdownComplete = null; try (HttpClientConnectionManager connPool = createConnectionPoolManager(uri, - HttpClientConnection.ProtocolVersion.HTTP_1_1)) { + HttpVersion.HTTP_1_1)) { shutdownComplete = connPool.getShutdownCompleteFuture(); try (HttpClientConnection conn = connPool.acquireConnection().get(60, TimeUnit.SECONDS)) { HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { From 14528c663f356826f7939ee110a266cf36738807 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 12:56:07 -0700 Subject: [PATCH 058/142] more compile error --- .../software/amazon/awssdk/crt/test/Elasticurl.java | 12 ++++++------ .../awssdk/crt/test/Http2ClientConnectionTest.java | 1 + .../awssdk/crt/test/HttpRequestResponseTest.java | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index db5430a88..b621710af 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -157,11 +157,11 @@ public long getLength() { } /* initial headers */ HttpHeader[] headers = new HttpHeader[] {}; - HttpRequestBase request = requiredVersion == HttpVersion.HTTP2 ? new Http2Request(headers, payloadStream) + HttpRequestBase request = requiredVersion == HttpVersion.HTTP_2 ? new Http2Request(headers, payloadStream) : new HttpRequest(method, pathAndQuery, headers, payloadStream); /* Version specific headers */ - if (requiredVersion == HttpVersion.HTTP2) { + if (requiredVersion == HttpVersion.HTTP_2) { request.addHeader(new HttpHeader(":method", method)); request.addHeader(new HttpHeader(":scheme", uri.getScheme())); request.addHeader(new HttpHeader(":authority", uri.getAuthority())); @@ -226,9 +226,9 @@ public static void main(String args[]) throws Exception { HttpVersion requiredVersion = HttpVersion.UNKNOWN; if (cli.hasOption("http1_1")) { - requiredVersion = HttpVersion.HTTP1_1; + requiredVersion = HttpVersion.HTTP_1_1; } else if (cli.hasOption("http2")) { - requiredVersion = HttpVersion.HTTP2; + requiredVersion = HttpVersion.HTTP_2; } TlsContextOptions tlsOpts = null; @@ -256,9 +256,9 @@ public static void main(String args[]) throws Exception { String[] alpn = cli.getOptionValues("alpn"); if (alpn == null) { - if (requiredVersion == HttpVersion.HTTP1_1) { + if (requiredVersion == HttpVersion.HTTP_1_1) { alpn = new String[] { "http/1.1" }; - } else if (requiredVersion == HttpVersion.HTTP2) { + } else if (requiredVersion == HttpVersion.HTTP_2) { alpn = new String[] { "h2" }; } else { alpn = new String[] { "h2", "http/1.1" }; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index c4f185ec8..f163c3330 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -15,6 +15,7 @@ import java.util.*; import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.http.HttpVersion; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.Http2ConnectionSetting; import software.amazon.awssdk.crt.http.Http2ClientConnection; diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java index aa9380463..f514567f7 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java @@ -12,6 +12,7 @@ import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpVersion; import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpRequest; import software.amazon.awssdk.crt.http.HttpRequestBodyStream; From 97ef42c64cf9a1d2a6bfe7eaba139ce2d43f56c5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 5 Nov 2021 16:48:49 -0700 Subject: [PATCH 059/142] had fun --- integration-testing/java_elasticurl_runner.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index 6e904ed4c..f972aafea 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -1,5 +1,6 @@ import sys import subprocess +import shlex from subprocess import Popen, PIPE TIMEOUT = 100 @@ -9,7 +10,7 @@ for index, arg in enumerate(elasticurl_args): if " " in arg: elasticurl_args[index] = "\\\"{}\\\"".format(arg) - if arg[0] == "\"" and arg[-1] == "\"": + elif arg[0] == "\"" and arg[-1] == "\"": elasticurl_args[index] = "\\\"{}\\\"".format(arg[1:-1]) @@ -21,11 +22,13 @@ command_string = " ".join(java_command) +args = shlex.split(command_string) -def run_command(args_str): + +def run_command(args): # gather all stderr and stdout to a single string that we print only if things go wrong process = subprocess.Popen( - args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -34,6 +37,7 @@ def run_command(args_str): process.kill() output = process.communicate()[0] finally: + args_str = subprocess.list2cmdline(args) if process.returncode != 0 or timedout: print(args_str) for line in output.splitlines(): @@ -46,4 +50,4 @@ def run_command(args_str): code=process.returncode, cmd=args_str)) -run_command(command_string) +run_command(args) From 50a501ab9cc8e8371774c99c917536bee85cd0da Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 8 Nov 2021 10:12:40 -0800 Subject: [PATCH 060/142] why it's failling harder --- integration-testing/java_elasticurl_runner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index f972aafea..fbcdbf9c1 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -22,13 +22,13 @@ command_string = " ".join(java_command) -args = shlex.split(command_string) +# args = shlex.split(command_string) -def run_command(args): +def run_command(args_str): # gather all stderr and stdout to a single string that we print only if things go wrong process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -37,7 +37,7 @@ def run_command(args): process.kill() output = process.communicate()[0] finally: - args_str = subprocess.list2cmdline(args) + # args_str = subprocess.list2cmdline(args) if process.returncode != 0 or timedout: print(args_str) for line in output.splitlines(): @@ -50,4 +50,4 @@ def run_command(args): code=process.returncode, cmd=args_str)) -run_command(args) +run_command(command_string) From c2fb4af32601b5b7f32f2875de5a5577a96616bd Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 8 Nov 2021 10:22:57 -0800 Subject: [PATCH 061/142] add some debug info --- integration-testing/java_elasticurl_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index fbcdbf9c1..beef23010 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -23,6 +23,7 @@ command_string = " ".join(java_command) # args = shlex.split(command_string) +print("command to run:"+command_string) def run_command(args_str): From aca437851d91c93af0b0d5eb1d1da78b88f0d1e8 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 8 Nov 2021 10:24:44 -0800 Subject: [PATCH 062/142] why --- integration-testing/java_elasticurl_runner.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index beef23010..15691a01f 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -22,14 +22,16 @@ command_string = " ".join(java_command) -# args = shlex.split(command_string) -print("command to run:"+command_string) +args = shlex.split(command_string) -def run_command(args_str): +def run_command(args): # gather all stderr and stdout to a single string that we print only if things go wrong + args_str = subprocess.list2cmdline(args) + print(args) + print(args_str) process = subprocess.Popen( - args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -38,7 +40,6 @@ def run_command(args_str): process.kill() output = process.communicate()[0] finally: - # args_str = subprocess.list2cmdline(args) if process.returncode != 0 or timedout: print(args_str) for line in output.splitlines(): @@ -51,4 +52,4 @@ def run_command(args_str): code=process.returncode, cmd=args_str)) -run_command(command_string) +run_command(args) From ffe074c612ec2f855772eddc0ec7df4394a935ea Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 8 Nov 2021 10:34:59 -0800 Subject: [PATCH 063/142] use the string as shell command --- integration-testing/java_elasticurl_runner.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index 15691a01f..3b53dd89d 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -19,19 +19,15 @@ java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] - +print(java_command) command_string = " ".join(java_command) - -args = shlex.split(command_string) +# args = shlex.split(command_line) -def run_command(args): +def run_command(args_str): # gather all stderr and stdout to a single string that we print only if things go wrong - args_str = subprocess.list2cmdline(args) - print(args) - print(args_str) process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -52,4 +48,4 @@ def run_command(args): code=process.returncode, cmd=args_str)) -run_command(args) +run_command(command_string) From f842a7fa341c8592ab5b3ddc3326b5465e348452 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 8 Nov 2021 10:46:40 -0800 Subject: [PATCH 064/142] if windows --- integration-testing/java_elasticurl_runner.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index 3b53dd89d..7c394eb3a 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -1,4 +1,5 @@ import sys +import os import subprocess import shlex from subprocess import Popen, PIPE @@ -20,14 +21,16 @@ java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] print(java_command) +if os.name == 'nt': + java_command[0] = 'mvn.cmd' command_string = " ".join(java_command) -# args = shlex.split(command_line) +args = shlex.split(command_string) -def run_command(args_str): +def run_command(args): # gather all stderr and stdout to a single string that we print only if things go wrong process = subprocess.Popen( - args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -36,6 +39,7 @@ def run_command(args_str): process.kill() output = process.communicate()[0] finally: + args_str = subprocess.list2cmdline(args) if process.returncode != 0 or timedout: print(args_str) for line in output.splitlines(): @@ -48,4 +52,4 @@ def run_command(args_str): code=process.returncode, cmd=args_str)) -run_command(command_string) +run_command(args) From 4d6ab131e7ed7de6344bd68e8bf504f98a47b921 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 8 Nov 2021 10:50:24 -0800 Subject: [PATCH 065/142] remove shell --- integration-testing/java_elasticurl_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index 7c394eb3a..c61028039 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -28,9 +28,10 @@ def run_command(args): + print(args) # gather all stderr and stdout to a single string that we print only if things go wrong process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] From f369483a9c57682109fde228afe61dc352509e5b Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 8 Nov 2021 11:50:36 -0800 Subject: [PATCH 066/142] last shot about subprocess --- integration-testing/java_elasticurl_runner.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index c61028039..57c64dc53 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -2,7 +2,7 @@ import os import subprocess import shlex -from subprocess import Popen, PIPE +import subprocess TIMEOUT = 100 # Runner for elasticurl integration tests @@ -24,14 +24,13 @@ if os.name == 'nt': java_command[0] = 'mvn.cmd' command_string = " ".join(java_command) -args = shlex.split(command_string) -def run_command(args): - print(args) +def run_command(args_str): + print(args_str) # gather all stderr and stdout to a single string that we print only if things go wrong process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -40,7 +39,6 @@ def run_command(args): process.kill() output = process.communicate()[0] finally: - args_str = subprocess.list2cmdline(args) if process.returncode != 0 or timedout: print(args_str) for line in output.splitlines(): @@ -53,4 +51,4 @@ def run_command(args): code=process.returncode, cmd=args_str)) -run_command(args) +run_command(command_string) From 62623289d62ce92c66cfacdde9813ed5b5a0abd6 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 12 Nov 2021 15:03:11 -0800 Subject: [PATCH 067/142] use sh for testing --- .builder/actions/aws_crt_java_test.py | 14 ++--- integration-testing/java_elasticurl_runner.py | 54 ------------------- integration-testing/java_elasticurl_runner.sh | 3 ++ 3 files changed, 10 insertions(+), 61 deletions(-) delete mode 100644 integration-testing/java_elasticurl_runner.py create mode 100644 integration-testing/java_elasticurl_runner.sh diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index 89c888535..ea79dc102 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -10,15 +10,15 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # Failed - actions.append("exit 1") - os.system("cat log.txt") + # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # # Failed + # actions.append("exit 1") + # os.system("cat log.txt") python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', - python, 'integration-testing/java_elasticurl_runner.py']) + 'sh', 'integration-testing/java_elasticurl_runner.sh']) return Builder.Script(actions, name='aws-crt-java-test') diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py deleted file mode 100644 index 57c64dc53..000000000 --- a/integration-testing/java_elasticurl_runner.py +++ /dev/null @@ -1,54 +0,0 @@ -import sys -import os -import subprocess -import shlex -import subprocess - -TIMEOUT = 100 -# Runner for elasticurl integration tests - -elasticurl_args = sys.argv[1:] -for index, arg in enumerate(elasticurl_args): - if " " in arg: - elasticurl_args[index] = "\\\"{}\\\"".format(arg) - elif arg[0] == "\"" and arg[-1] == "\"": - elasticurl_args[index] = "\\\"{}\\\"".format(arg[1:-1]) - - -mvn_args = " ".join(elasticurl_args) - - -java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', - '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] -print(java_command) -if os.name == 'nt': - java_command[0] = 'mvn.cmd' -command_string = " ".join(java_command) - - -def run_command(args_str): - print(args_str) - # gather all stderr and stdout to a single string that we print only if things go wrong - process = subprocess.Popen( - args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - timedout = False - try: - output = process.communicate(timeout=TIMEOUT)[0] - except subprocess.TimeoutExpired: - timedout = True - process.kill() - output = process.communicate()[0] - finally: - if process.returncode != 0 or timedout: - print(args_str) - for line in output.splitlines(): - print(line.decode()) - if timedout: - raise RuntimeError("Timeout happened after {secs} secs from: {cmd}".format( - secs=TIMEOUT, cmd=args_str)) - else: - raise RuntimeError("Return code {code} from: {cmd}".format( - code=process.returncode, cmd=args_str)) - - -run_command(command_string) diff --git a/integration-testing/java_elasticurl_runner.sh b/integration-testing/java_elasticurl_runner.sh new file mode 100644 index 000000000..bd6efe602 --- /dev/null +++ b/integration-testing/java_elasticurl_runner.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export JAVA_PROGRAM_ARGS=`echo "$@"` +mvn -e exec:java -Dexec.classpathScope="test" -Dexec.mainClass="software.amazon.awssdk.crt.test.Elasticurl" -Dexec.args="$JAVA_PROGRAM_ARGS" From c580eacbb11be250665325aab02e42430177f5ed Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 12 Nov 2021 15:05:19 -0800 Subject: [PATCH 068/142] bring the accidental comment out back --- .builder/actions/aws_crt_java_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index ea79dc102..38c6790e4 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -10,12 +10,12 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # # Failed - # actions.append("exit 1") - # os.system("cat log.txt") + if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # Failed + actions.append("exit 1") + os.system("cat log.txt") python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', From 26f79eb5d451fe78a0315ad35e008d71ed58b9b2 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 12 Nov 2021 15:48:34 -0800 Subject: [PATCH 069/142] maybe this? --- .builder/actions/aws_crt_java_test.py | 14 +++--- integration-testing/java_elasticurl_runner.py | 45 +++++++++++++++++++ integration-testing/java_elasticurl_runner.sh | 3 -- 3 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 integration-testing/java_elasticurl_runner.py delete mode 100644 integration-testing/java_elasticurl_runner.sh diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index 38c6790e4..9e38e6401 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -10,15 +10,15 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # Failed - actions.append("exit 1") - os.system("cat log.txt") + # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # # Failed + # actions.append("exit 1") + # os.system("cat log.txt") python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', - 'sh', 'integration-testing/java_elasticurl_runner.sh']) + python, 'integration-testing/java_elasticurl_runner.py']) return Builder.Script(actions, name='aws-crt-java-test') diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py new file mode 100644 index 000000000..f068a4b7c --- /dev/null +++ b/integration-testing/java_elasticurl_runner.py @@ -0,0 +1,45 @@ +import sys +import os +import subprocess +import shlex +import subprocess + +TIMEOUT = 100 +# Runner for elasticurl integration tests + +mvn_args = " ".join(map(shlex.quote, sys.argv[1:])) + +java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', + '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] +print(java_command) +if os.name == 'nt': + java_command[0] = 'mvn.cmd' +command_string = " ".join(java_command) + + +def run_command(args_str): + print(args_str) + # gather all stderr and stdout to a single string that we print only if things go wrong + process = subprocess.Popen( + args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + timedout = False + try: + output = process.communicate(timeout=TIMEOUT)[0] + except subprocess.TimeoutExpired: + timedout = True + process.kill() + output = process.communicate()[0] + finally: + if process.returncode != 0 or timedout: + print(args_str) + for line in output.splitlines(): + print(line.decode()) + if timedout: + raise RuntimeError("Timeout happened after {secs} secs from: {cmd}".format( + secs=TIMEOUT, cmd=args_str)) + else: + raise RuntimeError("Return code {code} from: {cmd}".format( + code=process.returncode, cmd=args_str)) + + +run_command(command_string) diff --git a/integration-testing/java_elasticurl_runner.sh b/integration-testing/java_elasticurl_runner.sh deleted file mode 100644 index bd6efe602..000000000 --- a/integration-testing/java_elasticurl_runner.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -export JAVA_PROGRAM_ARGS=`echo "$@"` -mvn -e exec:java -Dexec.classpathScope="test" -Dexec.mainClass="software.amazon.awssdk.crt.test.Elasticurl" -Dexec.args="$JAVA_PROGRAM_ARGS" From 8bc6ffe28013bcc6f4465b15cea014aaac3721b6 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 12 Nov 2021 15:48:53 -0800 Subject: [PATCH 070/142] normal test --- .builder/actions/aws_crt_java_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.builder/actions/aws_crt_java_test.py b/.builder/actions/aws_crt_java_test.py index 9e38e6401..89c888535 100644 --- a/.builder/actions/aws_crt_java_test.py +++ b/.builder/actions/aws_crt_java_test.py @@ -10,12 +10,12 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - # if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ - # -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ - # -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): - # # Failed - # actions.append("exit 1") - # os.system("cat log.txt") + if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + -DrerunFailingTestsCount=5 -DskipAfterFailureCount=1 \ + -Daws.crt.memory.tracing=2 -Daws.crt.debugnative=true"): + # Failed + actions.append("exit 1") + os.system("cat log.txt") python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', From 8c5068bfada03bbdb589c874586b9d1688a9ab21 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 12 Nov 2021 15:57:59 -0800 Subject: [PATCH 071/142] try not use shell? --- integration-testing/java_elasticurl_runner.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index f068a4b7c..8a2969ebc 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -11,17 +11,14 @@ java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] -print(java_command) + if os.name == 'nt': java_command[0] = 'mvn.cmd' -command_string = " ".join(java_command) - -def run_command(args_str): - print(args_str) +def run_command(args): # gather all stderr and stdout to a single string that we print only if things go wrong process = subprocess.Popen( - args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -31,7 +28,7 @@ def run_command(args_str): output = process.communicate()[0] finally: if process.returncode != 0 or timedout: - print(args_str) + args_str = subprocess.list2cmdline(args) for line in output.splitlines(): print(line.decode()) if timedout: @@ -42,4 +39,4 @@ def run_command(args_str): code=process.returncode, cmd=args_str)) -run_command(command_string) +run_command(java_command) From 9f496a44d15b8066156221513a4c094bc7cedddf Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 12 Nov 2021 16:05:31 -0800 Subject: [PATCH 072/142] Revert "try not use shell?" This reverts commit 8c5068bfada03bbdb589c874586b9d1688a9ab21. --- integration-testing/java_elasticurl_runner.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index 8a2969ebc..f068a4b7c 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -11,14 +11,17 @@ java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] - +print(java_command) if os.name == 'nt': java_command[0] = 'mvn.cmd' +command_string = " ".join(java_command) + -def run_command(args): +def run_command(args_str): + print(args_str) # gather all stderr and stdout to a single string that we print only if things go wrong process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + args_str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) timedout = False try: output = process.communicate(timeout=TIMEOUT)[0] @@ -28,7 +31,7 @@ def run_command(args): output = process.communicate()[0] finally: if process.returncode != 0 or timedout: - args_str = subprocess.list2cmdline(args) + print(args_str) for line in output.splitlines(): print(line.decode()) if timedout: @@ -39,4 +42,4 @@ def run_command(args): code=process.returncode, cmd=args_str)) -run_command(java_command) +run_command(command_string) From 406059f7ca1242387a8f343d79928855925d66de Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 15 Nov 2021 09:49:56 -0800 Subject: [PATCH 073/142] clean things up --- integration-testing/java_elasticurl_runner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration-testing/java_elasticurl_runner.py b/integration-testing/java_elasticurl_runner.py index f068a4b7c..09e984f52 100644 --- a/integration-testing/java_elasticurl_runner.py +++ b/integration-testing/java_elasticurl_runner.py @@ -2,7 +2,6 @@ import os import subprocess import shlex -import subprocess TIMEOUT = 100 # Runner for elasticurl integration tests @@ -11,8 +10,9 @@ java_command = ['mvn', '-e', 'exec:java', '-Dexec.classpathScope=\"test\"', '-Dexec.mainClass=\"software.amazon.awssdk.crt.test.Elasticurl\"', '-Dexec.args=\"{}\"'.format(mvn_args)] -print(java_command) + if os.name == 'nt': + # Windows uses mvn.cmd instead java_command[0] = 'mvn.cmd' command_string = " ".join(java_command) @@ -31,7 +31,6 @@ def run_command(args_str): output = process.communicate()[0] finally: if process.returncode != 0 or timedout: - print(args_str) for line in output.splitlines(): print(line.decode()) if timedout: From 6ed625a09e30f28849569449345684227dccb9a9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 15 Nov 2021 10:07:33 -0800 Subject: [PATCH 074/142] add warn level log --- src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index b621710af..9c4ca900b 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -88,7 +88,7 @@ static CommandLine parseArgs(String args[]) { cliOpts.addOption(null, "http1_1", false, "HTTP/1.1 connection required."); cliOpts.addOption(null, "http2", false, "HTTP/2 connection required."); cliOpts.addOption(Option.builder("v").longOpt("verbose").hasArg().argName("str") - .desc("logging level (ERROR|INFO|DEBUG|TRACE) default is none.").build()); + .desc("logging level (ERROR|WARN|INFO|DEBUG|TRACE) default is none.").build()); CommandLineParser cliParser = new DefaultParser(); CommandLine cli = null; @@ -194,6 +194,8 @@ public static void main(String args[]) throws Exception { LogLevel logLevel = LogLevel.None; if (verbose.equals("ERROR")) { logLevel = LogLevel.Error; + } else if (verbose.equals("WARN")) { + logLevel = LogLevel.Warn; } else if (verbose.equals("INFO")) { logLevel = LogLevel.Info; } else if (verbose.equals("DEBUG")) { From ad1bf7fae8d5ce6f76b5ebf1873742ba482a9e3d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 15 Nov 2021 10:12:38 -0800 Subject: [PATCH 075/142] do we really need compress? --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 181b6e303..937e888dc 100644 --- a/pom.xml +++ b/pom.xml @@ -284,12 +284,6 @@ 1.4 test - - org.apache.commons - commons-compress - 1.19 - test - From 3be8129be4d8abcb0c2e7ca47e0c912934ab1e5e Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 15 Nov 2021 10:46:41 -0800 Subject: [PATCH 076/142] missing unknown --- src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java b/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java index 9b0311b5b..d70e7312c 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpVersion.java @@ -33,6 +33,7 @@ public static HttpVersion getEnumValueFromInteger(int value) { private static Map buildEnumMapping() { Map enumMapping = new HashMap(); + enumMapping.put(UNKNOWN.getValue(), UNKNOWN); enumMapping.put(HTTP_1_0.getValue(), HTTP_1_0); enumMapping.put(HTTP_1_1.getValue(), HTTP_1_1); enumMapping.put(HTTP_2.getValue(), HTTP_2); From c4a4c91c5c7f012f7f9707701579722b8c2ad139 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 30 Nov 2021 12:31:56 -0800 Subject: [PATCH 077/142] settings --- crt/aws-c-http | 2 +- .../crt/http/Http2ClientConnection.java | 21 ++---- .../crt/http/Http2ConnectionSetting.java | 27 +------ .../crt/http/Http2ConnectionSettings.java | 71 +++++++++++++++++++ 4 files changed, 79 insertions(+), 42 deletions(-) create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java diff --git a/crt/aws-c-http b/crt/aws-c-http index 7af5e6f1a..0426a065b 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 7af5e6f1a8d66243b67e4500b8a30952714a8623 +Subproject commit 0426a065b77e99ce8dbbe2e89f6e9fae35840728 diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index 0e745f7ca..db288e119 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -26,19 +26,9 @@ public class Http2ClientConnection extends HttpClientConnection { * (RFC-7540 7). */ public enum Http2ErrorCode { - PROTOCOL_ERROR(1), - INTERNAL_ERROR(2), - FLOW_CONTROL_ERROR(3), - SETTINGS_TIMEOUT(4), - STREAM_CLOSED(5), - FRAME_SIZE_ERROR(6), - REFUSED_STREAM(7), - CANCEL(8), - COMPRESSION_ERROR(9), - CONNECT_ERROR(10), - ENHANCE_YOUR_CALM(11), - INADEQUATE_SECURITY(12), - HTTP_1_1_REQUIRED(13); + PROTOCOL_ERROR(1), INTERNAL_ERROR(2), FLOW_CONTROL_ERROR(3), SETTINGS_TIMEOUT(4), STREAM_CLOSED(5), + FRAME_SIZE_ERROR(6), REFUSED_STREAM(7), CANCEL(8), COMPRESSION_ERROR(9), CONNECT_ERROR(10), + ENHANCE_YOUR_CALM(11), INADEQUATE_SECURITY(12), HTTP_1_1_REQUIRED(13); private int errorCode; @@ -65,7 +55,7 @@ public Http2ClientConnection(long connectionBinding) { * @return When this future completes without exception, the peer has * acknowledged the settings and the change has been applied. */ - public CompletableFuture changeSettings(final Http2ConnectionSetting settings[]) { + public CompletableFuture updateSettings(final Http2ConnectionSetting settings[]) { throw new CrtRuntimeException("Unimplemented"); } @@ -73,7 +63,8 @@ public CompletableFuture changeSettings(final Http2ConnectionSetting setti * Send a PING frame. Round-trip-time is calculated when PING ACK is received * from peer. * - * @param pingData 8 Bytes data with the PING frame or null for not include data in ping + * @param pingData 8 Bytes data with the PING frame or null for not include data + * in ping * * @return When this future completes without exception, the peer has * acknowledged the PING and future will be completed with the round diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index 7ced33d2e..8e5169aad 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -12,11 +12,7 @@ public class Http2ConnectionSetting { * Predefined settings identifiers (RFC-7540 6.5.2). */ public enum ID { - HEADER_TABLE_SIZE(1), - ENABLE_PUSH(2), - MAX_CONCURRENT_STREAMS(3), - INITIAL_WINDOW_SIZE(4), - MAX_FRAME_SIZE(5), + HEADER_TABLE_SIZE(1), ENABLE_PUSH(2), MAX_CONCURRENT_STREAMS(3), INITIAL_WINDOW_SIZE(4), MAX_FRAME_SIZE(5), MAX_HEADER_LIST_SIZE(6); private int settingID; @@ -53,25 +49,4 @@ public Http2ConnectionSetting(ID id, long value) { this.id = id; this.value = value; } - - /** - * @exclude Marshals a list of settings into an array for Jni to deal with. - * - * @param settings list of headers to write to the headers block - * @return a long[] that with the [id, value, id, value, *] - */ - public static long[] marshallSettingsForJNI(List settings) { - /* Each setting is two long */ - int totalLength = settings.size(); - - long marshalledSettings[] = new long[totalLength * 2]; - - for (int i = 0; i < totalLength; i++) { - marshalledSettings[i * 2] = settings.get(i).id.getValue(); - marshalledSettings[i * 2 + 1] = settings.get(i).value; - } - - return marshalledSettings; - } - } diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java new file mode 100644 index 000000000..7b0ecde21 --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java @@ -0,0 +1,71 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package software.amazon.awssdk.crt.http; + +import java.util.List; +import java.util.ArrayList; + +public class Http2ConnectionSettings { + + private List settings; + + public Http2ConnectionSettings(List settings) { + this.settings = new ArrayList(settings); + } + + public Http2ConnectionSettings() { + this.settings = new ArrayList(); + } + + public void addSetting(Http2ConnectionSetting setting) throws Exception { + settings.add(setting); + } + + public void enablePush(boolean push) throws Exception { + settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.ENABLE_PUSH, push ? 1 : 0)); + } + + public void headerTableSize(long headerTableSize) throws Exception { + settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.HEADER_TABLE_SIZE, headerTableSize)); + } + + public void maxConcurrentStreams(long maxConcurrentStreams) throws Exception { + settings.add( + new Http2ConnectionSetting(Http2ConnectionSetting.ID.MAX_CONCURRENT_STREAMS, maxConcurrentStreams)); + } + + public void initialWindowSize(long initialWindowSize) throws Exception { + settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.INITIAL_WINDOW_SIZE, initialWindowSize)); + } + + public void maxFrameSize(long maxFrameSize) throws Exception { + settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.MAX_FRAME_SIZE, maxFrameSize)); + } + + public void maxHeaderListSize(long maxHeaderListSize) throws Exception { + settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.MAX_HEADER_LIST_SIZE, maxHeaderListSize)); + } + + /** + * @exclude Marshals a list of settings into an array for Jni to deal with. + * + * @param settings list of headers to write to the headers block + * @return a long[] that with the [id, value, id, value, *] + */ + public static long[] marshallSettingsForJNI() { + /* Each setting is two long */ + int totalLength = this.settings.size(); + + long marshalledSettings[] = new long[totalLength * 2]; + + for (int i = 0; i < totalLength; i++) { + marshalledSettings[i * 2] = settings.get(i).id.getValue(); + marshalledSettings[i * 2 + 1] = settings.get(i).value; + } + + return marshalledSettings; + } +} From 191162cb3dde496e49ac1191e5b78271ad62b7bc Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 1 Dec 2021 16:22:21 -0800 Subject: [PATCH 078/142] settings --- crt/aws-c-http | 2 +- .../crt/http/Http2ConnectionSetting.java | 20 +++++++++++++++++++ .../crt/http/Http2ConnectionSettings.java | 19 ------------------ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/crt/aws-c-http b/crt/aws-c-http index 0426a065b..4825acffd 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 0426a065b77e99ce8dbbe2e89f6e9fae35840728 +Subproject commit 4825acffd476ac6fdd5c431ca13e6303b8388832 diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java index 8e5169aad..358a6ba3a 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSetting.java @@ -49,4 +49,24 @@ public Http2ConnectionSetting(ID id, long value) { this.id = id; this.value = value; } + + /** + * @exclude Marshals a list of settings into an array for Jni to deal with. + * + * @param settings list of headers to write to the headers block + * @return a long[] that with the [id, value, id, value, *] + */ + public static long[] marshallSettingsForJNI(final List settings) { + /* Each setting is two long */ + int totalLength = settings.size(); + + long marshalledSettings[] = new long[totalLength * 2]; + + for (int i = 0; i < totalLength; i++) { + marshalledSettings[i * 2] = settings.get(i).id.getValue(); + marshalledSettings[i * 2 + 1] = settings.get(i).value; + } + + return marshalledSettings; + } } diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java index 7b0ecde21..cfaf2a544 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java @@ -49,23 +49,4 @@ public void maxHeaderListSize(long maxHeaderListSize) throws Exception { settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.MAX_HEADER_LIST_SIZE, maxHeaderListSize)); } - /** - * @exclude Marshals a list of settings into an array for Jni to deal with. - * - * @param settings list of headers to write to the headers block - * @return a long[] that with the [id, value, id, value, *] - */ - public static long[] marshallSettingsForJNI() { - /* Each setting is two long */ - int totalLength = this.settings.size(); - - long marshalledSettings[] = new long[totalLength * 2]; - - for (int i = 0; i < totalLength; i++) { - marshalledSettings[i * 2] = settings.get(i).id.getValue(); - marshalledSettings[i * 2 + 1] = settings.get(i).value; - } - - return marshalledSettings; - } } From f677a1432b40884ff726b12410c42141ec433a3d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 1 Dec 2021 16:31:18 -0800 Subject: [PATCH 079/142] update name --- .../amazon/awssdk/crt/http/Http2ClientConnection.java | 10 ++++------ src/native/http_request_response.c | 8 ++++---- .../awssdk/crt/test/Http2ClientConnectionTest.java | 8 ++++---- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java index a67620890..4aa056bb8 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2ClientConnection.java @@ -11,8 +11,6 @@ import java.util.concurrent.CompletableFuture; import java.util.List; -import static software.amazon.awssdk.crt.CRT.awsLastError; - /** * This class wraps aws-c-http to provide the basic HTTP/2 request/response * functionality via the AWS Common Runtime. @@ -59,16 +57,16 @@ public Http2ClientConnection(long connectionBinding) { * @return When this future completes without exception, the peer has * acknowledged the settings and the change has been applied. */ - public CompletableFuture changeSettings(final List settings) { + public CompletableFuture updateSettings(final List settings) { CompletableFuture future = new CompletableFuture<>(); if (isNull()) { future.completeExceptionally( new IllegalStateException("Http2ClientConnection has been closed, can't change settings on it.")); return future; } - AsyncCallback changeSettingsCompleted = AsyncCallback.wrapFuture(future, null); + AsyncCallback updateSettingsCompleted = AsyncCallback.wrapFuture(future, null); try { - http2ClientConnectionChangeSettings(getNativeHandle(), changeSettingsCompleted, + http2ClientConnectionUpdateSettings(getNativeHandle(), updateSettingsCompleted, Http2ConnectionSetting.marshallSettingsForJNI(settings)); } catch (CrtRuntimeException ex) { future.completeExceptionally(ex); @@ -206,7 +204,7 @@ private static native Http2Stream http2ClientConnectionMakeRequest(long connecti HttpRequestBodyStream bodyStream, HttpStreamResponseHandlerNativeAdapter responseHandler) throws CrtRuntimeException; - private static native void http2ClientConnectionChangeSettings(long connectionBinding, + private static native void http2ClientConnectionUpdateSettings(long connectionBinding, AsyncCallback completedCallback, long[] marshalledSettings) throws CrtRuntimeException; private static native void http2ClientConnectionSendPing(long connectionBinding, AsyncCallback completedCallback, diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 1eefbefee..e0cdc7b80 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -692,7 +692,7 @@ static void s_on_settings_completed(struct aws_http_connection *http2_connection s_cleanup_http2_callback_data(callback_data); } -JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionChangeSettings( +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionUpdateSettings( JNIEnv *env, jclass jni_class, jlong jni_connection, @@ -706,12 +706,12 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio if (!native_conn) { aws_jni_throw_null_pointer_exception( - env, "Http2ClientConnection.http2ClientConnectionChangeSettings: Invalid aws_http_connection"); + env, "Http2ClientConnection.http2ClientConnectionUpdateSettings: Invalid aws_http_connection"); return; } if (!java_async_callback) { aws_jni_throw_illegal_argument_exception( - env, "Http2ClientConnection.http2ClientConnectionChangeSettings: Invalid async callback"); + env, "Http2ClientConnection.http2ClientConnectionUpdateSettings: Invalid async callback"); return; } struct aws_allocator *allocator = aws_jni_get_allocator(); @@ -735,7 +735,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnectio if (aws_http2_connection_change_settings( native_conn, settings, settings_len, s_on_settings_completed, callback_data)) { aws_jni_throw_runtime_exception( - env, "Http2ClientConnection.http2ClientConnectionChangeSettings: failed to change settings"); + env, "Http2ClientConnection.http2ClientConnectionUpdateSettings: failed to change settings"); goto done; } success = true; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java index f163c3330..1449980dd 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java @@ -54,7 +54,7 @@ public void testHttp2ConnectionGetVersion() throws Exception { } @Test - public void testHttp2ConnectionChangeSettings() throws Exception { + public void testHttp2ConnectionUpdateSettings() throws Exception { skipIfNetworkUnavailable(); CompletableFuture shutdownComplete = null; @@ -70,7 +70,7 @@ public void testHttp2ConnectionChangeSettings() throws Exception { List settings = new ArrayList(); settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.ENABLE_PUSH, 0)); settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.ENABLE_PUSH, 0)); - conn.changeSettings(settings); + conn.updateSettings(settings).get(5, TimeUnit.SECONDS); } } catch (Exception e) { throw new RuntimeException(e); @@ -84,7 +84,7 @@ public void testHttp2ConnectionChangeSettings() throws Exception { } @Test - public void testHttp2ConnectionChangeSettingsEmpty() throws Exception { + public void testHttp2ConnectionUpdateSettingsEmpty() throws Exception { /* empty settings is allowed to send */ skipIfNetworkUnavailable(); @@ -99,7 +99,7 @@ public void testHttp2ConnectionChangeSettingsEmpty() throws Exception { actuallyConnected = true; Assert.assertTrue(conn.getVersion() == EXPECTED_VERSION); List settings = new ArrayList(); - conn.changeSettings(settings); + conn.updateSettings(settings).get(5, TimeUnit.SECONDS); } } catch (Exception e) { throw new RuntimeException(e); From f19f1c562034ccfeb5b8714f8b12edeaab6f1d9f Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 1 Dec 2021 16:49:10 -0800 Subject: [PATCH 080/142] merge conflict --- .../amazon/awssdk/crt/http/HttpRequestBase.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java index 9585f4a25..8e846258f 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpRequestBase.java @@ -50,12 +50,12 @@ public byte[] marshalForJni() { int size = 0; size += BUFFER_INT_SIZE; /* version */ size += BUFFER_INT_SIZE + method.length(); - size += BUFFER_INT_SIZE + encodedPath.length(); - size += (BUFFER_INT_SIZE * 2) * headers.size(); + byte[] pathBytes = encodedPath.getBytes(UTF8); + size += BUFFER_INT_SIZE + pathBytes.length; for (HttpHeader header : headers) { if (header.getNameBytes().length > 0) { - size += header.getNameBytes().length + header.getValueBytes().length; + size += header.getNameBytes().length + header.getValueBytes().length + (BUFFER_INT_SIZE * 2); } } @@ -63,8 +63,8 @@ public byte[] marshalForJni() { buffer.putInt(version.getValue()); buffer.putInt(method.length()); buffer.put(method.getBytes(UTF8)); - buffer.putInt(encodedPath.length()); - buffer.put(encodedPath.getBytes(UTF8)); + buffer.putInt(pathBytes.length); + buffer.put(pathBytes); for (HttpHeader header : headers) { if (header.getNameBytes().length > 0) { From 24be4c8f7d684901d57d15be92a40345a5c20d1c Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 1 Dec 2021 17:01:04 -0800 Subject: [PATCH 081/142] revert the formatting change --- .../awssdk/crt/http/HttpClientConnection.java | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java index f0ca94b18..2a1a8c677 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnection.java @@ -6,76 +6,72 @@ package software.amazon.awssdk.crt.http; import java.util.concurrent.CompletableFuture; +import java.util.Map; +import java.util.HashMap; import software.amazon.awssdk.crt.CRT; import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; +import software.amazon.awssdk.crt.http.HttpStream; import static software.amazon.awssdk.crt.CRT.awsLastError; /** - * This class wraps aws-c-http to provide the basic HTTP request/response - * functionality via the AWS Common Runtime. + * This class wraps aws-c-http to provide the basic HTTP request/response functionality via the AWS Common Runtime. * - * HttpClientConnection represents a single connection to a HTTP service - * endpoint. + * HttpClientConnection represents a single connection to a HTTP service endpoint. * - * This class is not thread safe and should not be called from different - * threads. + * This class is not thread safe and should not be called from different threads. */ public class HttpClientConnection extends CrtResource { + protected HttpClientConnection(long connectionBinding) { acquireNativeHandle(connectionBinding); } /** - * Schedules an HttpRequest on the Native EventLoop for this - * HttpClientConnection. + * Schedules an HttpRequest on the Native EventLoop for this HttpClientConnection. * - * @param request The Request to make to the Server. - * @param streamHandler The Stream Handler to be called from the Native - * EventLoop + * @param request The Request to make to the Server. + * @param streamHandler The Stream Handler to be called from the Native EventLoop * @throws CrtRuntimeException if stream creation fails - * @return The HttpStream that represents this Request/Response Pair. It can be - * closed at any time during the request/response, but must be closed by - * the user thread making this request when it's done. + * @return The HttpStream that represents this Request/Response Pair. It can be closed at any time during the + * request/response, but must be closed by the user thread making this request when it's done. */ public HttpStream makeRequest(HttpRequestBase request, HttpStreamResponseHandler streamHandler) throws CrtRuntimeException { if (isNull()) { throw new IllegalStateException("HttpClientConnection has been closed, can't make requests on it."); } - HttpStream stream = httpClientConnectionMakeRequest(getNativeHandle(), request.marshalForJni(), - request.getBodyStream(), new HttpStreamResponseHandlerNativeAdapter(streamHandler)); + HttpStream stream = httpClientConnectionMakeRequest(getNativeHandle(), + request.marshalForJni(), + request.getBodyStream(), + new HttpStreamResponseHandlerNativeAdapter(streamHandler)); return stream; } /** - * Determines whether a resource releases its dependencies at the same time the - * native handle is released or if it waits. Resources that wait are responsible - * for calling releaseReferences() manually. + * Determines whether a resource releases its dependencies at the same time the native handle is released or if it waits. + * Resources that wait are responsible for calling releaseReferences() manually. */ @Override - protected boolean canReleaseReferencesImmediately() { - return true; - } + protected boolean canReleaseReferencesImmediately() { return true; } /** - * Releases this HttpClientConnection back into the Connection Pool, and allows - * another Request to acquire this connection. + * Releases this HttpClientConnection back into the Connection Pool, and allows another Request to acquire this connection. */ @Override protected void releaseNativeHandle() { - if (!isNull()) { + if (!isNull()){ httpClientConnectionReleaseManaged(getNativeHandle()); } } /** - * Shuts down the underlying http connection. Even if this function is called, - * you still need to properly close the connection as well in order to release - * the native resources. + * Shuts down the underlying http connection. Even if this function is called, you still need to properly close + * the connection as well in order to release the native resources. */ public void shutdown() { httpClientConnectionShutdown(getNativeHandle()); @@ -87,8 +83,7 @@ public HttpVersion getVersion() { }; /** Called from Native when a new connection is acquired **/ - private static void onConnectionAcquired(CompletableFuture acquireFuture, - long nativeConnectionBinding, int errorCode) { + private static void onConnectionAcquired(CompletableFuture acquireFuture, long nativeConnectionBinding, int errorCode) { if (errorCode != CRT.AWS_CRT_SUCCESS) { acquireFuture.completeExceptionally(new HttpException(errorCode)); return; @@ -106,13 +101,13 @@ private static void onConnectionAcquired(CompletableFuture /******************************************************************************* * Native methods ******************************************************************************/ - private static native HttpStream httpClientConnectionMakeRequest(long connectionBinding, byte[] marshalledRequest, - HttpRequestBodyStream bodyStream, HttpStreamResponseHandlerNativeAdapter responseHandler) - throws CrtRuntimeException; + private static native HttpStream httpClientConnectionMakeRequest(long connectionBinding, + byte[] marshalledRequest, + HttpRequestBodyStream bodyStream, + HttpStreamResponseHandlerNativeAdapter responseHandler) throws CrtRuntimeException; private static native void httpClientConnectionShutdown(long connectionBinding) throws CrtRuntimeException; private static native void httpClientConnectionReleaseManaged(long connectionBinding) throws CrtRuntimeException; - private static native short httpClientConnectionGetVersion(long connectionBinding) throws CrtRuntimeException; } From 7e250f8b5957c6900647edf9f393df9a04ab9935 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 2 Dec 2021 09:53:58 -0800 Subject: [PATCH 082/142] Not sure it's a merge error or something --- src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index 9c4ca900b..eea869ae7 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -221,8 +221,8 @@ public static void main(String args[]) throws Exception { URI uri = new URI(cli.getArgs()[0]); boolean useTls = true; int port = 443; - if (uri.getScheme() == "http") { - useTls = true; + if (uri.getScheme().equals("http")) { + useTls = false; port = 80; } From f4bc91c378812a641721d38120aea3d4c6409eb5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 6 Dec 2021 10:59:03 -0800 Subject: [PATCH 083/142] override http 1.1 function to be un implemented for h2 --- .../amazon/awssdk/crt/http/Http2Stream.java | 11 ++++ .../amazon/awssdk/crt/http/HttpStream.java | 62 +++++++++++-------- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java index bebbf7c2e..5c0023a92 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2Stream.java @@ -7,6 +7,8 @@ import software.amazon.awssdk.crt.CrtRuntimeException; +import java.util.concurrent.CompletableFuture; + public class Http2Stream extends HttpStream { protected Http2Stream(long ptr) { @@ -28,4 +30,13 @@ public void resetStream(final Http2ClientConnection.Http2ErrorCode errorCode) { * @TODO getters for reset stream. Not sure anyone needs it though. */ private static native void http2StreamResetStream(long http_stream, int errorCode); + + @Override + public void writeChunk(final byte[] chunkData, boolean isFinalChunk, final HttpStreamWriteChunkCompletionCallback chunkCompletionCallback) { + throw new RuntimeException("Not implemented"); + } + @Override + public CompletableFuture writeChunk(final byte[] chunkData, boolean isFinalChunk) { + throw new RuntimeException("Not implemented"); + } } diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java index 9e339baf1..ddc208dc6 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java @@ -19,12 +19,6 @@ */ public class HttpStream extends CrtResource { - /** - * Completion interface for writing chunks to an http stream - */ - public interface HttpStreamWriteChunkCompletionCallback { - void onChunkCompleted(int errorCode); - } /* Native code will call this constructor during HttpClientConnection.makeRequest() */ protected HttpStream(long ptr) { @@ -48,6 +42,9 @@ protected void releaseNativeHandle() { } } + /******************************************************************************* + * Shared method + ******************************************************************************/ /** * Opens the Sliding Read/Write Window by the number of bytes passed as an argument for this HttpStream. * @@ -66,6 +63,37 @@ public void incrementWindow(int windowSize) { } } + /** + * Activates the client stream. + */ + public void activate() { + if (!isNull()) { + httpStreamActivate(getNativeHandle(), this); + } + } + + /** + * Retrieves the Http Response Status Code + * @return The Http Response Status Code + */ + public int getResponseStatusCode() { + if (!isNull()) { + return httpStreamGetResponseStatusCode(getNativeHandle()); + } + throw new IllegalStateException("Can't get Status Code on Closed Stream"); + } + + /******************************************************************************* + * HTTP/1.1 Only + ******************************************************************************/ + + /** + * Completion interface for writing chunks to an http stream + */ + public interface HttpStreamWriteChunkCompletionCallback { + void onChunkCompleted(int errorCode); + } + /** * Use only for Http 1.1 Chunked Encoding. At some later point we may adapt this interface for H2, but not yet. * You must call activate() before using this function. @@ -117,25 +145,9 @@ public void onChunkCompleted(int errorCode) { return completionFuture; } - /** - * Activates the client stream. - */ - public void activate() { - if (!isNull()) { - httpStreamActivate(getNativeHandle(), this); - } - } - - /** - * Retrieves the Http Response Status Code - * @return The Http Response Status Code - */ - public int getResponseStatusCode() { - if (!isNull()) { - return httpStreamGetResponseStatusCode(getNativeHandle()); - } - throw new IllegalStateException("Can't get Status Code on Closed Stream"); - } + /******************************************************************************* + * Native methods + ******************************************************************************/ private static native void httpStreamRelease(long http_stream); private static native void httpStreamIncrementWindow(long http_stream, int window_size); From 7f5fd3fbf7adab4257bbeb7d44af3f031a0917eb Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 6 Jan 2022 15:33:15 -0800 Subject: [PATCH 084/142] sm wip --- .../awssdk/crt/http/Http2StreamManager.java | 25 ++ .../crt/http/Http2StreamManagerOptions.java | 220 ++++++++++++++++++ .../crt/http/HttpClientConnectionManager.java | 3 - .../HttpClientConnectionManagerOptions.java | 18 -- 4 files changed, 245 insertions(+), 21 deletions(-) create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java new file mode 100644 index 000000000..4a39a669e --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -0,0 +1,25 @@ +package software.amazon.awssdk.crt.http; + +import java.net.URI; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; + +public class Http2StreamManager extends CrtResource { + private static final String HTTP = "http"; + private static final String HTTPS = "https"; + private static final int DEFAULT_HTTP_PORT = 80; + private static final int DEFAULT_HTTPS_PORT = 443; + public static Http2StreamManager create(Http2StreamManagerOptions options) { + return new Http2StreamManager(options); + } + + private Http2StreamManager(Http2StreamManagerOptions options) { + } + + public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamResponseHandler streamHandler) throws CrtRuntimeException{ + throw UnsupportedOperationException; + } +} diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java new file mode 100644 index 000000000..7d64b484e --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java @@ -0,0 +1,220 @@ +package software.amazon.awssdk.crt.http; + +import java.net.URI; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; + +/** + * Contains all the configuration options for a Http2StreamManager + * instance + */ +public class Http2StreamManagerOptions { + public static final int DEFAULT_MAX_WINDOW_SIZE = Integer.MAX_VALUE; + public static final int DEFAULT_MAX = Integer.MAX_VALUE; + public static final int DEFAULT_MAX_CONNECTIONS = 2; + + private ClientBootstrap clientBootstrap; + private SocketOptions socketOptions; + private TlsContext tlsContext; + private URI uri; + private int port = -1; + private boolean manualWindowManagement = false; + private int windowSize = DEFAULT_MAX_WINDOW_SIZE; + private HttpMonitoringOptions monitoringOptions; + private HttpProxyOptions proxyOptions; + + private int idealConcurrentStreamsPerConnection = DEFAULT_MAX; + private int maxConcurrentStreamsPerConnection = DEFAULT_MAX; + + private int maxConnections = DEFAULT_MAX_CONNECTIONS; + + /** + * Default constructor + */ + public Http2StreamManagerOptions() { + } + + /** + * Sets the client bootstrap instance to use to create the pool's connections + * + * @param clientBootstrap ClientBootstrap to use + * @return this + */ + public Http2StreamManagerOptions withClientBootstrap(ClientBootstrap clientBootstrap) { + this.clientBootstrap = clientBootstrap; + return this; + } + + /** + * Gets the client bootstrap instance to use to create the pool's connections + * + * @return ClientBootstrap used by this connection manager + */ + public ClientBootstrap getClientBootstrap() { + return clientBootstrap; + } + + /** + * Sets the socket options to use for connections in the connection pool + * + * @param socketOptions The socket options to use for all connections in the + * manager + * @return this + */ + public Http2StreamManagerOptions withSocketOptions(SocketOptions socketOptions) { + this.socketOptions = socketOptions; + return this; + } + + /** + * @return the socket options to use for connections in the connection pool + */ + public SocketOptions getSocketOptions() { + return socketOptions; + } + + /** + * Sets the tls context to use for connections in the connection pool + * + * @param tlsContext The TlsContext to use + * @return this + */ + public Http2StreamManagerOptions withTlsContext(TlsContext tlsContext) { + this.tlsContext = tlsContext; + return this; + } + + /** + * @return the tls context used by connections in the connection pool + */ + public TlsContext getTlsContext() { + return tlsContext; + } + + /** + * Sets the IO channel window size to use for connections in the connection pool + * + * @param windowSize The initial window size to use for each connection + * @return this + */ + public Http2StreamManagerOptions withWindowSize(int windowSize) { + this.windowSize = windowSize; + return this; + } + + /** + * @return the IO channel window size to use for connections in the connection + * pool + */ + public int getWindowSize() { + return windowSize; + } + + public Http2StreamManagerOptions withIdealConcurrentStreamsPerConnection(int idealConcurrentStreamsPerConnection) { + this.idealConcurrentStreamsPerConnection = idealConcurrentStreamsPerConnection; + return this; + } + + public int getIdealConcurrentStreamsPerConnection() { + return idealConcurrentStreamsPerConnection; + } + + public Http2StreamManagerOptions withMaxConcurrentStreamsPerConnection(int maxConcurrentStreamsPerConnection) { + this.maxConcurrentStreamsPerConnection = maxConcurrentStreamsPerConnection; + return this; + } + + public int getMaxConcurrentStreamsPerConnection() { + return maxConcurrentStreamsPerConnection; + } + + /** + * Sets the URI to use for connections in the connection pool + * + * @param uri The endpoint URI to connect to + * @return this + */ + public Http2StreamManagerOptions withUri(URI uri) { + this.uri = uri; + return this; + } + + /** + * @return the URI to use for connections in the connection pool + */ + public URI getUri() { + return uri; + } + + /** + * Sets the port to connect to for connections in the connection pool + * + * @param port The port to connect to + * @return this + */ + public Http2StreamManagerOptions withPort(int port) { + this.port = port; + return this; + } + + /** + * @return the port to connect to for connections in the connection pool. + * Returns -1 if none has been explicitly set. + */ + public int getPort() { + return port; + } + + public Http2StreamManagerOptions withMaxConnections(int maxConnections) { + this.maxConnections = maxConnections; + return this; + } + + /** + * @return the maximum number of connections allowed in the connection pool + */ + public int getMaxConnections() { + return maxConnections; + } + + public Http2StreamManagerOptions withProxyOptions(HttpProxyOptions proxyOptions) { + this.proxyOptions = proxyOptions; + return this; + } + + /** + * @return the proxy options for connections in the connection pool + */ + public HttpProxyOptions getProxyOptions() { + return proxyOptions; + } + + public boolean isManualWindowManagement() { + return manualWindowManagement; + } + + public Http2StreamManagerOptions withManualWindowManagement(boolean manualWindowManagement) { + this.manualWindowManagement = manualWindowManagement; + return this; + } + + /** + * Sets the monitoring options for connections in the connection pool + * + * @param monitoringOptions Monitoring options for this connection manager, or + * null to disable monitoring + * @return this + */ + public Http2StreamManagerOptions withMonitoringOptions(HttpMonitoringOptions monitoringOptions) { + this.monitoringOptions = monitoringOptions; + return this; + } + + /** + * @return the monitoring options for connections in the connection pool + */ + public HttpMonitoringOptions getMonitoringOptions() { + return monitoringOptions; + } +} diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java index 60a66a15f..7b4705aaf 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java @@ -64,9 +64,6 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options) int windowSize = options.getWindowSize(); if (windowSize <= 0) { throw new IllegalArgumentException("Window Size must be greater than zero."); } - int bufferSize = options.getBufferSize(); - if (bufferSize <= 0) { throw new IllegalArgumentException("Buffer Size must be greater than zero."); } - int maxConnections = options.getMaxConnections(); if (maxConnections <= 0) { throw new IllegalArgumentException("Max Connections must be greater than zero."); } diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java index 6c52d87b7..4d7e3063e 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java @@ -13,7 +13,6 @@ * Contains all the configuration options for a HttpConnectionPoolManager instance */ public class HttpClientConnectionManagerOptions { - public static final int DEFAULT_MAX_BUFFER_SIZE = 16 * 1024; public static final int DEFAULT_MAX_WINDOW_SIZE = Integer.MAX_VALUE; public static final int DEFAULT_MAX_CONNECTIONS = 2; @@ -21,7 +20,6 @@ public class HttpClientConnectionManagerOptions { private SocketOptions socketOptions; private TlsContext tlsContext; private int windowSize = DEFAULT_MAX_WINDOW_SIZE; - private int bufferSize = DEFAULT_MAX_BUFFER_SIZE; private URI uri; private int port = -1; private int maxConnections = DEFAULT_MAX_CONNECTIONS; @@ -98,22 +96,6 @@ public HttpClientConnectionManagerOptions withWindowSize(int windowSize) { */ public int getWindowSize() { return windowSize; } - /** - * Sets the IO buffer size to use for connections in the connection pool - * @param bufferSize Size of I/O buffer per connection - * @return this - */ - public HttpClientConnectionManagerOptions withBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - return this; - } - - /** - * @return the IO buffer size to use for connections in the connection pool - */ - public int getBufferSize() { return bufferSize; } - - /** * Sets the URI to use for connections in the connection pool * @param uri The endpoint URI to connect to From 50e7f72cc673e57f201519fe55cc2e1f2c483ec9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 13 Jan 2022 15:50:44 -0800 Subject: [PATCH 085/142] bascially it --- .../amazon/awssdk/crt/http/Http2StreamManagerOptions.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java index 7d64b484e..d64e4bdee 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java @@ -35,6 +35,11 @@ public class Http2StreamManagerOptions { public Http2StreamManagerOptions() { } + /** + * TODO: + * finalize with and get functions and documentations. + */ + /** * Sets the client bootstrap instance to use to create the pool's connections * From e776a525fd09da554ac04523a64ebec20de5f042 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 31 Jan 2022 10:01:34 -0800 Subject: [PATCH 086/142] use stream manager branch --- crt/aws-c-http | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crt/aws-c-http b/crt/aws-c-http index 4825acffd..d526aba75 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 4825acffd476ac6fdd5c431ca13e6303b8388832 +Subproject commit d526aba75f84624aa7aa81c335be20b6b0e066ba From 979a253b4f228782468d649b2415767ba584fd54 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 31 Jan 2022 13:52:41 -0800 Subject: [PATCH 087/142] java part finalizing --- .../awssdk/crt/http/Http2StreamManager.java | 251 +++++++++++++++++- .../crt/http/Http2StreamManagerOptions.java | 7 +- 2 files changed, 249 insertions(+), 9 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index 4a39a669e..d7aaa6b51 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -1,25 +1,270 @@ package software.amazon.awssdk.crt.http; -import java.net.URI; import software.amazon.awssdk.crt.CrtResource; import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.SocketOptions; import software.amazon.awssdk.crt.io.TlsContext; +import java.util.concurrent.CompletableFuture; +import java.net.URI; +import java.nio.charset.Charset; + public class Http2StreamManager extends CrtResource { + private static final String HTTP = "http"; private static final String HTTPS = "https"; private static final int DEFAULT_HTTP_PORT = 80; private static final int DEFAULT_HTTPS_PORT = 443; + private final static Charset UTF8 = java.nio.charset.StandardCharsets.UTF_8; + + private final int windowSize; + private final URI uri; + private final int port; + private final int maxConnections; + private final int idealConcurrentStreamsPerConnection; + private final int maxConcurrentStreamsPerConnection; + private final CompletableFuture shutdownComplete = new CompletableFuture<>(); + + /** + * Factory function for Http2StreamManager instances + * + * @param options configuration options + * @return a new instance of an Http2StreamManager + */ public static Http2StreamManager create(Http2StreamManagerOptions options) { return new Http2StreamManager(options); } private Http2StreamManager(Http2StreamManagerOptions options) { + URI uri = options.getUri(); + if (uri == null) { + throw new IllegalArgumentException("URI must not be null"); + } + if (uri.getScheme() == null) { + throw new IllegalArgumentException("URI does not have a Scheme"); + } + if (!HTTP.equals(uri.getScheme()) && !HTTPS.equals(uri.getScheme())) { + throw new IllegalArgumentException("URI has unknown Scheme"); + } + if (uri.getHost() == null) { + throw new IllegalArgumentException("URI does not have a Host name"); + } + + ClientBootstrap clientBootstrap = options.getClientBootstrap(); + if (clientBootstrap == null) { + throw new IllegalArgumentException("ClientBootstrap must not be null"); + } + + SocketOptions socketOptions = options.getSocketOptions(); + if (socketOptions == null) { + throw new IllegalArgumentException("SocketOptions must not be null"); + } + + boolean useTls = HTTPS.equals(uri.getScheme()); + TlsContext tlsContext = options.getTlsContext(); + if (useTls && tlsContext == null) { + throw new IllegalArgumentException("TlsContext must not be null if https is used"); + } + + int windowSize = options.getWindowSize(); + if (windowSize <= 0) { + throw new IllegalArgumentException("Window Size must be greater than zero."); + } + + int maxConnections = options.getMaxConnections(); + if (maxConnections <= 0) { + throw new IllegalArgumentException("Max Connections must be greater than zero."); + } + int maxConcurrentStreamsPerConnection = options.getMaxConcurrentStreamsPerConnection(); + if (maxConcurrentStreamsPerConnection <= 0) { + throw new IllegalArgumentException("Max Concurrent Streams Per Connection must be greater than zero."); + } + int idealConcurrentStreamsPerConnection = options.getIdealConcurrentStreamsPerConnection(); + if (idealConcurrentStreamsPerConnection <= 0 + || idealConcurrentStreamsPerConnection > maxConcurrentStreamsPerConnection) { + throw new IllegalArgumentException( + "Ideal Concurrent Streams Per Connection must be greater than zero and smaller than max."); + } + + int port = options.getPort(); + if (port == -1) { + port = uri.getPort(); + /* Pick a default port based on the scheme if one wasn't set */ + if (port == -1) { + if (HTTP.equals(uri.getScheme())) { + port = DEFAULT_HTTP_PORT; + } + if (HTTPS.equals(uri.getScheme())) { + port = DEFAULT_HTTPS_PORT; + } + } + } + + HttpProxyOptions proxyOptions = options.getProxyOptions(); + + this.windowSize = windowSize; + this.uri = uri; + this.port = port; + this.maxConnections = maxConnections; + this.idealConcurrentStreamsPerConnection = idealConcurrentStreamsPerConnection; + this.maxConcurrentStreamsPerConnection = maxConcurrentStreamsPerConnection; + + int proxyConnectionType = 0; + String proxyHost = null; + int proxyPort = 0; + TlsContext proxyTlsContext = null; + int proxyAuthorizationType = 0; + String proxyAuthorizationUsername = null; + String proxyAuthorizationPassword = null; + + if (proxyOptions != null) { + proxyConnectionType = proxyOptions.getConnectionType().getValue(); + proxyHost = proxyOptions.getHost(); + proxyPort = proxyOptions.getPort(); + proxyTlsContext = proxyOptions.getTlsContext(); + proxyAuthorizationType = proxyOptions.getAuthorizationType().getValue(); + proxyAuthorizationUsername = proxyOptions.getAuthorizationUsername(); + proxyAuthorizationPassword = proxyOptions.getAuthorizationPassword(); + } + + HttpMonitoringOptions monitoringOptions = options.getMonitoringOptions(); + long monitoringThroughputThresholdInBytesPerSecond = 0; + int monitoringFailureIntervalInSeconds = 0; + if (monitoringOptions != null) { + monitoringThroughputThresholdInBytesPerSecond = monitoringOptions.getMinThroughputBytesPerSecond(); + monitoringFailureIntervalInSeconds = monitoringOptions.getAllowableThroughputFailureIntervalSeconds(); + } + + acquireNativeHandle(http2StreamManagerNew(this, + clientBootstrap.getNativeHandle(), + socketOptions.getNativeHandle(), + useTls ? tlsContext.getNativeHandle() : 0, + windowSize, + uri.getHost().getBytes(UTF8), + port, + proxyConnectionType, + proxyHost != null ? proxyHost.getBytes(UTF8) : null, + proxyPort, + proxyTlsContext != null ? proxyTlsContext.getNativeHandle() : 0, + proxyAuthorizationType, + proxyAuthorizationUsername != null ? proxyAuthorizationUsername.getBytes(UTF8) : null, + proxyAuthorizationPassword != null ? proxyAuthorizationPassword.getBytes(UTF8) : null, + options.isManualWindowManagement(), + monitoringThroughputThresholdInBytesPerSecond, + monitoringFailureIntervalInSeconds, + maxConnections, + idealConcurrentStreamsPerConnection, + maxConcurrentStreamsPerConnection)); + + /* + * we don't need to add a reference to socketOptions since it's copied during + * connection manager construction + */ + addReferenceTo(clientBootstrap); + if (useTls) { + addReferenceTo(tlsContext); + } + } + + /** + * Request a Http2Stream from StreamManager. + * + * @param request + * @param streamHandler + * @return A future for a Http2Stream that will be completed when the stream is + * acquired. + * @throws CrtRuntimeException + */ + public CompletableFuture acquireStream(Http2Request request, + HttpStreamResponseHandler streamHandler) + throws CrtRuntimeException { + if (isNull()) { + throw new IllegalStateException( + "Http2StreamManager has been closed, can't acquire new streams"); + } + + CompletableFuture streamRequest = new CompletableFuture<>(); + + http2StreamManagerAcquireStream(this.getNativeHandle(), + request.marshalForJni(), + request.getBodyStream(), + new HttpStreamResponseHandlerNativeAdapter(streamHandler), + streamRequest); + return streamRequest; + } + + /** + * Called from Native when all Streams from this Stream manager have finished + * and underlying resources like connections opened under the hood has been + * cleaned up + * begin releasing Native Resources that Http2StreamManager depends on. + */ + private void onShutdownComplete() { + releaseReferences(); + + this.shutdownComplete.complete(null); } - public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamResponseHandler streamHandler) throws CrtRuntimeException{ - throw UnsupportedOperationException; + /** + * Determines whether a resource releases its dependencies at the same time the + * native handle is released or if it waits. + * Resources that wait are responsible for calling releaseReferences() manually. + */ + @Override + protected boolean canReleaseReferencesImmediately() { + return false; } + + /** + * Closes this Connection Pool and any pending Connection Acquisitions + */ + @Override + protected void releaseNativeHandle() { + if (!isNull()) { + /* + * Release our Native pointer and schedule tasks on the Native Event Loop to + * start sending HTTP/TLS/TCP + * connection shutdown messages to peers for any open Connections. + */ + http2StreamManagerRelease(getNativeHandle()); + } + } + + public CompletableFuture getShutdownCompleteFuture() { + return shutdownComplete; + } + + /******************************************************************************* + * Native methods + ******************************************************************************/ + + private static native long http2StreamManagerNew(Http2StreamManager thisObj, + long client_bootstrap, + long socketOptions, + long tlsContext, + int windowSize, + byte[] endpoint, + int port, + int proxyConnectionType, + byte[] proxyHost, + int proxyPort, + long proxyTlsContext, + int proxyAuthorizationType, + byte[] proxyAuthorizationUsername, + byte[] proxyAuthorizationPassword, + boolean isManualWindowManagement, + long monitoringThroughputThresholdInBytesPerSecond, + int monitoringFailureIntervalInSeconds, + int maxConns, + int ideal_concurrent_streams_per_connection, + int max_concurrent_streams_per_connection) throws CrtRuntimeException; + + private static native void http2StreamManagerRelease(long stream_manager) throws CrtRuntimeException; + + private static native void http2StreamManagerAcquireStream(long stream_manager, + byte[] marshalledRequest, + HttpRequestBodyStream bodyStream, + HttpStreamResponseHandlerNativeAdapter responseHandler, + CompletableFuture acquireFuture) throws CrtRuntimeException; } diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java index d64e4bdee..66eb4b492 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java @@ -24,7 +24,7 @@ public class Http2StreamManagerOptions { private HttpMonitoringOptions monitoringOptions; private HttpProxyOptions proxyOptions; - private int idealConcurrentStreamsPerConnection = DEFAULT_MAX; + private int idealConcurrentStreamsPerConnection = 100; private int maxConcurrentStreamsPerConnection = DEFAULT_MAX; private int maxConnections = DEFAULT_MAX_CONNECTIONS; @@ -35,11 +35,6 @@ public class Http2StreamManagerOptions { public Http2StreamManagerOptions() { } - /** - * TODO: - * finalize with and get functions and documentations. - */ - /** * Sets the client bootstrap instance to use to create the pool's connections * From 9df51fda280d8bd4a592091d57068dd39161c82f Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 31 Jan 2022 13:57:25 -0800 Subject: [PATCH 088/142] use the tag --- crt/aws-c-http | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crt/aws-c-http b/crt/aws-c-http index 4825acffd..0426a065b 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 4825acffd476ac6fdd5c431ca13e6303b8388832 +Subproject commit 0426a065b77e99ce8dbbe2e89f6e9fae35840728 From b406ab4781fbfaebaedefad87dbf55905cdd6251 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 31 Jan 2022 14:16:39 -0800 Subject: [PATCH 089/142] get the stream public --- src/native/http2_stream_manager.c | 24 +++++++++++ src/native/http_request_response.c | 68 +++++++++++------------------- src/native/http_request_response.h | 43 +++++++++++++++++++ 3 files changed, 92 insertions(+), 43 deletions(-) create mode 100644 src/native/http2_stream_manager.c create mode 100644 src/native/http_request_response.h diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c new file mode 100644 index 000000000..713bb39f7 --- /dev/null +++ b/src/native/http2_stream_manager.c @@ -0,0 +1,24 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include "crt.h" +#include "java_class_ids.h" + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index e0cdc7b80..4e37ecca6 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -7,6 +7,7 @@ #include "crt.h" #include "http_connection_manager.h" +#include "http_request_response.h" #include "http_request_utils.h" #include "java_class_ids.h" @@ -34,7 +35,7 @@ # endif #endif -static jobject s_java_http_stream_from_native_new(JNIEnv *env, void *opaque, enum aws_http_version version) { +jobject aws_java_http_stream_from_native_new(JNIEnv *env, void *opaque, int version) { jlong jni_native_ptr = (jlong)opaque; AWS_ASSERT(jni_native_ptr); jobject stream = NULL; @@ -57,42 +58,24 @@ static jobject s_java_http_stream_from_native_new(JNIEnv *env, void *opaque, enu return stream; } -static void s_java_http_stream_from_native_delete(JNIEnv *env, jobject jHttpStream) { +void aws_java_http_stream_from_native_delete(JNIEnv *env, jobject jHttpStream) { /* Delete our reference to the HttpStream Object from the JVM. */ (*env)->DeleteGlobalRef(env, jHttpStream); } /******************************************************************************* - * http_stream_callback_data - carries around data needed by the various http request + * http_stream_binding - carries around data needed by the various http request * callbacks. ******************************************************************************/ -struct http_stream_callback_data { - JavaVM *jvm; - - // TEMP: Until Java API changes to match "H1B" native HTTP API, - // create aws_http_message and aws_input_stream under the hood. - struct aws_http_message *native_request; - - jobject java_http_response_stream_handler; - jobject java_http_stream; - struct aws_http_stream *native_stream; - struct aws_byte_buf headers_buf; - int response_status; - - /* - * Unactivated streams must have their callback data destroyed at release time - */ - struct aws_atomic_var activated; -}; -static void http_stream_callback_destroy(JNIEnv *env, struct http_stream_callback_data *callback) { +void aws_http_stream_binding_destroy(JNIEnv *env, struct http_stream_binding *callback) { if (callback == NULL) { return; } if (callback->java_http_stream) { - s_java_http_stream_from_native_delete(env, callback->java_http_stream); + aws_java_http_stream_from_native_delete(env, callback->java_http_stream); } if (callback->java_http_response_stream_handler != NULL) { @@ -113,10 +96,10 @@ static void http_stream_callback_destroy(JNIEnv *env, struct http_stream_callbac } // If error occurs, A Java exception is thrown and NULL is returned. -static struct http_stream_callback_data *http_stream_callback_alloc(JNIEnv *env, jobject java_callback_handler) { +struct http_stream_binding *aws_http_stream_binding_alloc(JNIEnv *env, jobject java_callback_handler) { struct aws_allocator *allocator = aws_jni_get_allocator(); - struct http_stream_callback_data *callback = aws_mem_calloc(allocator, 1, sizeof(struct http_stream_callback_data)); + struct http_stream_binding *callback = aws_mem_calloc(allocator, 1, sizeof(struct http_stream_binding)); AWS_FATAL_ASSERT(callback); // GetJavaVM() reference doesn't need a NewGlobalRef() call since it's global by default @@ -141,7 +124,7 @@ static int s_on_incoming_headers_fn( void *user_data) { (void)block_type; - struct http_stream_callback_data *callback = (struct http_stream_callback_data *)user_data; + struct http_stream_binding *callback = (struct http_stream_binding *)user_data; int resp_status = -1; int err_code = aws_http_stream_get_incoming_response_status(stream, &resp_status); if (err_code != AWS_OP_SUCCESS) { @@ -166,7 +149,7 @@ static int s_on_incoming_header_block_done_fn( void *user_data) { (void)stream; - struct http_stream_callback_data *callback = (struct http_stream_callback_data *)user_data; + struct http_stream_binding *callback = (struct http_stream_binding *)user_data; JNIEnv *env = aws_jni_get_thread_env(callback->jvm); jint jni_block_type = block_type; @@ -207,7 +190,7 @@ static int s_on_incoming_header_block_done_fn( } static int s_on_incoming_body_fn(struct aws_http_stream *stream, const struct aws_byte_cursor *data, void *user_data) { - struct http_stream_callback_data *callback = (struct http_stream_callback_data *)user_data; + struct http_stream_binding *callback = (struct http_stream_binding *)user_data; size_t total_window_increment = 0; @@ -244,7 +227,7 @@ static int s_on_incoming_body_fn(struct aws_http_stream *stream, const struct aw static void s_on_stream_complete_fn(struct aws_http_stream *stream, int error_code, void *user_data) { - struct http_stream_callback_data *callback = (struct http_stream_callback_data *)user_data; + struct http_stream_binding *callback = (struct http_stream_binding *)user_data; JNIEnv *env = aws_jni_get_thread_env(callback->jvm); /* Don't invoke Java callbacks if Java HttpStream failed to completely setup */ @@ -261,7 +244,7 @@ static void s_on_stream_complete_fn(struct aws_http_stream *stream, int error_co aws_http_connection_close(aws_http_stream_get_connection(stream)); } - http_stream_callback_destroy(env, callback); + aws_http_stream_binding_destroy(env, callback); } jobjectArray aws_java_http_headers_from_native(JNIEnv *env, struct aws_http_headers *headers) { @@ -313,8 +296,7 @@ static jobject s_make_request_general( return (jobject)NULL; } - struct http_stream_callback_data *callback_data = - http_stream_callback_alloc(env, jni_http_response_callback_handler); + struct http_stream_binding *callback_data = aws_http_stream_binding_alloc(env, jni_http_response_callback_handler); if (!callback_data) { /* Exception already thrown */ return (jobject)NULL; @@ -324,7 +306,7 @@ static jobject s_make_request_general( aws_http_request_new_from_java_http_request(env, marshalled_request, jni_http_request_body_stream); if (callback_data->native_request == NULL) { /* Exception already thrown */ - http_stream_callback_destroy(env, callback_data); + aws_http_stream_binding_destroy(env, callback_data); return (jobject)NULL; } @@ -349,7 +331,7 @@ static jobject s_make_request_general( (void *)native_conn, (void *)callback_data->native_stream); - jHttpStream = s_java_http_stream_from_native_new(env, callback_data, version); + jHttpStream = aws_java_http_stream_from_native_new(env, callback_data, version); } /* Check for errors that might have occurred while holding the lock. */ @@ -357,7 +339,7 @@ static jobject s_make_request_general( /* Failed to create native aws_http_stream. Clean up callback_data. */ AWS_LOGF_ERROR(AWS_LS_HTTP_CONNECTION, "Stream Request Failed. conn: %p", (void *)native_conn); aws_jni_throw_runtime_exception(env, "HttpClientConnection.MakeRequest: Unable to Execute Request"); - http_stream_callback_destroy(env, callback_data); + aws_http_stream_binding_destroy(env, callback_data); return NULL; } else if (!jHttpStream) { /* Failed to create java HttpStream, but did create native aws_http_stream. @@ -407,7 +389,7 @@ JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnec } struct http_stream_chunked_callback_data { - struct http_stream_callback_data *stream_cb_data; + struct http_stream_binding *stream_cb_data; struct aws_byte_buf chunk_data; struct aws_input_stream *chunk_stream; jobject completion_callback; @@ -447,7 +429,7 @@ JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_http_HttpStream_httpStrea jobject completion_callback) { (void)jni_class; - struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct http_stream_binding *cb_data = (struct http_stream_binding *)jni_cb_data; struct aws_http_stream *stream = cb_data->native_stream; struct http_stream_chunked_callback_data *chunked_callback_data = @@ -495,7 +477,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_HttpStream_httpStrea jobject j_http_stream) { (void)jni_class; - struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct http_stream_binding *cb_data = (struct http_stream_binding *)jni_cb_data; struct aws_http_stream *stream = cb_data->native_stream; if (stream == NULL) { @@ -524,7 +506,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_HttpStream_httpStrea (void)jni_class; - struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct http_stream_binding *cb_data = (struct http_stream_binding *)jni_cb_data; struct aws_http_stream *stream = cb_data->native_stream; if (stream == NULL) { @@ -536,7 +518,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_HttpStream_httpStrea size_t not_activated = 0; if (aws_atomic_compare_exchange_int(&cb_data->activated, ¬_activated, 1)) { - http_stream_callback_destroy(env, cb_data); + aws_http_stream_binding_destroy(env, cb_data); } } @@ -547,7 +529,7 @@ JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_http_HttpStream_httpStrea (void)jni_class; - struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct http_stream_binding *cb_data = (struct http_stream_binding *)jni_cb_data; struct aws_http_stream *stream = cb_data->native_stream; if (stream == NULL) { @@ -574,7 +556,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_HttpStream_httpStrea (void)jni_class; - struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct http_stream_binding *cb_data = (struct http_stream_binding *)jni_cb_data; struct aws_http_stream *stream = cb_data->native_stream; if (stream == NULL) { @@ -600,7 +582,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2Stream_http2Str (void)jni_class; - struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct http_stream_binding *cb_data = (struct http_stream_binding *)jni_cb_data; struct aws_http_stream *stream = cb_data->native_stream; if (stream == NULL) { diff --git a/src/native/http_request_response.h b/src/native/http_request_response.h new file mode 100644 index 000000000..3a2c696a7 --- /dev/null +++ b/src/native/http_request_response.h @@ -0,0 +1,43 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#ifndef AWS_JNI_CRT_HTTP_REQUEST_RESPONSE_H +#define AWS_JNI_CRT_HTTP_REQUEST_RESPONSE_H + +#include + +struct aws_http_message; +struct aws_http_stream; +struct aws_byte_buf; +struct aws_atomic_var; + +struct http_stream_binding { + JavaVM *jvm; + + // TEMP: Until Java API changes to match "H1B" native HTTP API, + // create aws_http_message and aws_input_stream under the hood. + struct aws_http_message *native_request; + + jobject java_http_response_stream_handler; + jobject java_http_stream; + struct aws_http_stream *native_stream; + struct aws_byte_buf headers_buf; + int response_status; + + /* + * Unactivated streams must have their callback data destroyed at release time + */ + struct aws_atomic_var activated; +}; + +jobject aws_java_http_stream_from_native_new(JNIEnv *env, void *opaque, int version); +void aws_java_http_stream_from_native_delete(JNIEnv *env, jobject jHttpStream); + +void aws_http_stream_binding_destroy(JNIEnv *env, struct http_stream_binding *callback); + +// If error occurs, A Java exception is thrown and NULL is returned. +struct http_stream_binding *aws_http_stream_binding_alloc(JNIEnv *env, jobject java_callback_handler); + +#endif /* AWS_JNI_CRT_HTTP_REQUEST_RESPONSE_H */ From 2daf6fe1f80ddcd61827f6879ef1197fd296c895 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 1 Feb 2022 12:03:53 -0800 Subject: [PATCH 090/142] binding WIP --- .../awssdk/crt/http/Http2StreamManager.java | 32 +- src/native/http2_stream_manager.c | 293 +++++++++++++++++- src/native/http_request_response.c | 19 +- src/native/http_request_response.h | 17 + 4 files changed, 338 insertions(+), 23 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index d7aaa6b51..05d58cd47 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -4,6 +4,7 @@ import software.amazon.awssdk.crt.CrtRuntimeException; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.AsyncCallback; import software.amazon.awssdk.crt.io.TlsContext; import java.util.concurrent.CompletableFuture; @@ -177,21 +178,24 @@ private Http2StreamManager(Http2StreamManagerOptions options) { * @throws CrtRuntimeException */ public CompletableFuture acquireStream(Http2Request request, - HttpStreamResponseHandler streamHandler) - throws CrtRuntimeException { + HttpStreamResponseHandler streamHandler) { + CompletableFuture completionFuture = new CompletableFuture<>(); + AsyncCallback acquireStreamCompleted = AsyncCallback.wrapFuture(completionFuture, null); if (isNull()) { - throw new IllegalStateException( - "Http2StreamManager has been closed, can't acquire new streams"); + completionFuture.completeExceptionally(new IllegalStateException( + "Http2StreamManager has been closed, can't acquire new streams")); + return completionFuture; } - - CompletableFuture streamRequest = new CompletableFuture<>(); - - http2StreamManagerAcquireStream(this.getNativeHandle(), - request.marshalForJni(), - request.getBodyStream(), - new HttpStreamResponseHandlerNativeAdapter(streamHandler), - streamRequest); - return streamRequest; + try { + http2StreamManagerAcquireStream(this.getNativeHandle(), + request.marshalForJni(), + request.getBodyStream(), + new HttpStreamResponseHandlerNativeAdapter(streamHandler), + acquireStreamCompleted); + } catch (CrtRuntimeException ex) { + completionFuture.completeExceptionally(ex); + } + return completionFuture; } /** @@ -266,5 +270,5 @@ private static native void http2StreamManagerAcquireStream(long stream_manager, byte[] marshalledRequest, HttpRequestBodyStream bodyStream, HttpStreamResponseHandlerNativeAdapter responseHandler, - CompletableFuture acquireFuture) throws CrtRuntimeException; + AsyncCallback completedCallback) throws CrtRuntimeException; } diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c index 713bb39f7..a84d339e7 100644 --- a/src/native/http2_stream_manager.c +++ b/src/native/http2_stream_manager.c @@ -4,6 +4,7 @@ */ #include "crt.h" +#include "http_request_response.h" #include "java_class_ids.h" #include @@ -19,6 +20,296 @@ #include #include -#include #include +#include #include + +/* on 32-bit platforms, casting pointers to longs throws a warning we don't need */ +#if UINTPTR_MAX == 0xffffffff +# if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4305) /* 'type cast': truncation from 'jlong' to 'jni_tls_ctx_options *' */ +# else +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpointer-to-int-cast" +# pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +# endif +#endif + +/* + * Stream manager binding, persists across the lifetime of the native object. + */ +struct aws_http2_stream_manager_binding { + JavaVM *jvm; + jweak java_http2_stream_manager; + struct aws_http2_stream_manager *stream_manager; +}; + +JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_http2StreamManagerNew( + JNIEnv *env, + jclass jni_class, + jobject stream_manager_jobject, + jlong jni_client_bootstrap, + jlong jni_socket_options, + jlong jni_tls_ctx, + jint jni_window_size, + jbyteArray jni_endpoint, + jint jni_port, + jint jni_proxy_connection_type, + jbyteArray jni_proxy_host, + jint jni_proxy_port, + jlong jni_proxy_tls_context, + jint jni_proxy_authorization_type, + jbyteArray jni_proxy_authorization_username, + jbyteArray jni_proxy_authorization_password, + jboolean jni_manual_window_management, + jlong jni_monitoring_throughput_threshold_in_bytes_per_second, + jint jni_monitoring_failure_interval_in_seconds, + jint jni_max_conns, + jint ideal_concurrent_streams_per_connection, + jint max_concurrent_streams_per_connection) { + + (void)jni_class; + (void)jni_expected_protocol_version; + + struct aws_client_bootstrap *client_bootstrap = (struct aws_client_bootstrap *)jni_client_bootstrap; + struct aws_socket_options *socket_options = (struct aws_socket_options *)jni_socket_options; + struct aws_tls_ctx *tls_ctx = (struct aws_tls_ctx *)jni_tls_ctx; + struct http_connection_manager_binding *binding = NULL; + + if (!client_bootstrap) { + aws_jni_throw_runtime_exception(env, "ClientBootstrap can't be null"); + return (jlong)NULL; + } + + if (!socket_options) { + aws_jni_throw_runtime_exception(env, "SocketOptions can't be null"); + return (jlong)NULL; + } + + struct aws_allocator *allocator = aws_jni_get_allocator(); + struct aws_byte_cursor endpoint = aws_jni_byte_cursor_from_jbyteArray_acquire(env, jni_endpoint); + + if (jni_port <= 0 || 65535 < jni_port) { + aws_jni_throw_runtime_exception(env, "Port must be between 1 and 65535"); + goto cleanup; + } + + if (jni_window_size <= 0) { + aws_jni_throw_runtime_exception(env, "Window Size must be > 0"); + goto cleanup; + } + + if (jni_max_conns <= 0) { + aws_jni_throw_runtime_exception(env, "Max Connections must be > 0"); + goto cleanup; + } + + uint16_t port = (uint16_t)jni_port; + + int use_tls = (jni_tls_ctx != 0); + + struct aws_tls_connection_options tls_conn_options; + AWS_ZERO_STRUCT(tls_conn_options); + + if (use_tls) { + aws_tls_connection_options_init_from_ctx(&tls_conn_options, tls_ctx); + aws_tls_connection_options_set_server_name(&tls_conn_options, allocator, &endpoint); + } + + binding = aws_mem_calloc(allocator, 1, sizeof(struct http_connection_manager_binding)); + AWS_FATAL_ASSERT(binding); + binding->java_http_conn_manager = (*env)->NewWeakGlobalRef(env, conn_manager_jobject); + + jint jvmresult = (*env)->GetJavaVM(env, &binding->jvm); + (void)jvmresult; + AWS_FATAL_ASSERT(jvmresult == 0); + + struct aws_http_connection_manager_options manager_options; + AWS_ZERO_STRUCT(manager_options); + + manager_options.bootstrap = client_bootstrap; + manager_options.initial_window_size = (size_t)jni_window_size; + manager_options.socket_options = socket_options; + manager_options.tls_connection_options = NULL; + manager_options.host = endpoint; + manager_options.port = port; + manager_options.max_connections = (size_t)jni_max_conns; + manager_options.shutdown_complete_callback = &s_on_http_conn_manager_shutdown_complete_callback; + manager_options.shutdown_complete_user_data = binding; + manager_options.monitoring_options = NULL; + /* TODO: this variable needs to be renamed in aws-c-http. Come back and change it next revision. */ + manager_options.enable_read_back_pressure = jni_manual_window_management; + manager_options.max_connection_idle_in_milliseconds = jni_max_connection_idle_in_milliseconds; + + if (use_tls) { + manager_options.tls_connection_options = &tls_conn_options; + } + + struct aws_http_connection_monitoring_options monitoring_options; + AWS_ZERO_STRUCT(monitoring_options); + if (jni_monitoring_throughput_threshold_in_bytes_per_second >= 0 && + jni_monitoring_failure_interval_in_seconds >= 2) { + monitoring_options.minimum_throughput_bytes_per_second = + jni_monitoring_throughput_threshold_in_bytes_per_second; + monitoring_options.allowable_throughput_failure_interval_seconds = jni_monitoring_failure_interval_in_seconds; + + manager_options.monitoring_options = &monitoring_options; + } + + struct aws_http_proxy_options proxy_options; + AWS_ZERO_STRUCT(proxy_options); + + struct aws_tls_connection_options proxy_tls_conn_options; + AWS_ZERO_STRUCT(proxy_tls_conn_options); + + aws_http_proxy_options_jni_init( + env, + &proxy_options, + jni_proxy_connection_type, + &proxy_tls_conn_options, + jni_proxy_host, + (uint16_t)jni_proxy_port, + jni_proxy_authorization_username, + jni_proxy_authorization_password, + jni_proxy_authorization_type, + (struct aws_tls_ctx *)jni_proxy_tls_context); + + if (jni_proxy_host != NULL) { + manager_options.proxy_options = &proxy_options; + } + + binding->manager = aws_http_connection_manager_new(allocator, &manager_options); + if (binding->manager == NULL) { + aws_jni_throw_runtime_exception( + env, "Failed to create connection manager: %s", aws_error_str(aws_last_error())); + } + + aws_http_proxy_options_jni_clean_up( + env, &proxy_options, jni_proxy_host, jni_proxy_authorization_username, jni_proxy_authorization_password); + + if (use_tls) { + aws_tls_connection_options_clean_up(&tls_conn_options); + } + +cleanup: + aws_jni_byte_cursor_from_jbyteArray_release(env, jni_endpoint, endpoint); + + if (binding->manager == NULL) { + s_destroy_manager_binding(binding); + binding = NULL; + } + + return (jlong)binding; +} + +/* + * Stream manager binding, persists across the lifetime of the native object. + */ +struct aws_sm_acquire_stream_callback_data { + JavaVM *jvm; + struct http_stream_binding *stream_binding; + jobject java_async_callback; +}; + +static void s_cleanup_sm_acquire_stream_callback_data(struct aws_sm_acquire_stream_callback_data *callback_data) { + + JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); + + if (callback_data->async_callback) { + (*env)->DeleteGlobalRef(env, callback_data->async_callback); + } + aws_mem_release(aws_jni_get_allocator(), callback_data); +} + +static struct aws_sm_acquire_stream_callback_data *s_new_sm_acquire_stream_callback_data( + JNIEnv *env, + struct aws_allocator *allocator, + struct http_stream_binding *stream_binding, + jobject async_callback) { + struct aws_sm_acquire_stream_callback_data *callback_data = + aws_mem_calloc(allocator, 1, sizeof(struct aws_sm_acquire_stream_callback_data)); + + jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); + AWS_FATAL_ASSERT(jvmresult == 0); + callback_data->async_callback = async_callback ? (*env)->NewGlobalRef(env, async_callback) : NULL; + AWS_FATAL_ASSERT(callback_data->async_callback != NULL); + callback_data->stream_binding = stream_binding; + + return callback_data; +} + +static void s_on_stream_acquired((struct aws_http_stream *stream, int error_code, void *user_data){ + struct aws_sm_acquire_stream_callback_data *callback_data = user_data; + JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); + if (error_code) { + jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); + (*env)->CallVoidMethod(env, callback_data->async_callback, async_callback_properties.on_failure, crt_exception); + (*env)->DeleteLocalRef(env, crt_exception); + aws_http_stream_binding_destroy(env, callback_data->stream_binding); + } else { + /* TODO: finalize */ + callback_data->stream_binding->native_stream = stream; + jobject java_round_trip_time_ns = (*env)->NewObject( + env, boxed_long_properties.long_class, boxed_long_properties.constructor, (jlong)round_trip_time_ns); + (*env)->CallVoidMethod( + env, + callback_data->async_callback, + async_callback_properties.on_success_with_object, + java_round_trip_time_ns); + (*env)->DeleteLocalRef(env, java_round_trip_time_ns); + } + AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); + s_cleanup_http2_callback_data(callback_data); +} + +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_http2StreamManagerAcquireStream( + JNIEnv *env, + jclass jni_class, + jlong jni_stream_manager, + jbyteArray marshalled_request, + jobject jni_http_request_body_stream, + jobject jni_http_response_callback_handler, + jobject java_async_callback) { + (void)jni_class; + struct aws_http2_stream_manager_binding *sm_binding = (struct aws_http2_stream_manager_binding *)jni_stream_manager; + + /* TODO: check for null */ + struct http_stream_binding *stream_binding = aws_http_stream_binding_alloc(env, jni_http_response_callback_handler); + if (!stream_binding) { + /* Exception already thrown */ + return; + } + + stream_binding->native_request = + aws_http_request_new_from_java_http_request(env, marshalled_request, jni_http_request_body_stream); + if (stream_binding->native_request == NULL) { + /* Exception already thrown */ + aws_http_stream_binding_destroy(env, stream_binding); + return; + } + + struct aws_http_make_request_options request_options = { + .self_size = sizeof(request_options), + .request = stream_binding->native_request, + /* Set Callbacks */ + .on_response_headers = aws_java_http_stream_on_incoming_headers_fn, + .on_response_header_block_done = aws_java_http_stream_on_incoming_header_block_done_fn, + .on_response_body = aws_java_http_stream_on_incoming_body_fn, + .on_complete = aws_java_http_stream_on_stream_complete_fn, + .user_data = stream_binding, + }; + + struct aws_allocator *allocator = aws_jni_get_allocator(); + struct aws_sm_acquire_stream_callback_data *callback_data = + s_new_sm_acquire_stream_callback_data(env, allocator, java_async_callback); + + struct aws_http2_stream_manager_acquire_stream_options acquire_options = + { + .options = &request_options, + .callback = s_on_stream_acquired, + .user_data = callback_data, + } + + aws_http2_stream_manager_acquire_stream(sm_binding->stream_manager, acquire_options); +} \ No newline at end of file diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 4e37ecca6..a26c58b96 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -116,7 +116,7 @@ struct http_stream_binding *aws_http_stream_binding_alloc(JNIEnv *env, jobject j return callback; } -static int s_on_incoming_headers_fn( +int aws_java_http_stream_on_incoming_headers_fn( struct aws_http_stream *stream, enum aws_http_header_block block_type, const struct aws_http_header *header_array, @@ -143,7 +143,7 @@ static int s_on_incoming_headers_fn( return AWS_OP_SUCCESS; } -static int s_on_incoming_header_block_done_fn( +int aws_java_http_stream_on_incoming_header_block_done_fn( struct aws_http_stream *stream, enum aws_http_header_block block_type, void *user_data) { @@ -189,7 +189,10 @@ static int s_on_incoming_header_block_done_fn( return AWS_OP_SUCCESS; } -static int s_on_incoming_body_fn(struct aws_http_stream *stream, const struct aws_byte_cursor *data, void *user_data) { +int aws_java_http_stream_on_incoming_body_fn( + struct aws_http_stream *stream, + const struct aws_byte_cursor *data, + void *user_data) { struct http_stream_binding *callback = (struct http_stream_binding *)user_data; size_t total_window_increment = 0; @@ -225,7 +228,7 @@ static int s_on_incoming_body_fn(struct aws_http_stream *stream, const struct aw return AWS_OP_SUCCESS; } -static void s_on_stream_complete_fn(struct aws_http_stream *stream, int error_code, void *user_data) { +void aws_java_http_stream_on_stream_complete_fn(struct aws_http_stream *stream, int error_code, void *user_data) { struct http_stream_binding *callback = (struct http_stream_binding *)user_data; JNIEnv *env = aws_jni_get_thread_env(callback->jvm); @@ -314,10 +317,10 @@ static jobject s_make_request_general( .self_size = sizeof(request_options), .request = callback_data->native_request, /* Set Callbacks */ - .on_response_headers = s_on_incoming_headers_fn, - .on_response_header_block_done = s_on_incoming_header_block_done_fn, - .on_response_body = s_on_incoming_body_fn, - .on_complete = s_on_stream_complete_fn, + .on_response_headers = aws_java_http_stream_on_incoming_headers_fn, + .on_response_header_block_done = aws_java_http_stream_on_incoming_header_block_done_fn, + .on_response_body = aws_java_http_stream_on_incoming_body_fn, + .on_complete = aws_java_http_stream_on_stream_complete_fn, .user_data = callback_data, }; diff --git a/src/native/http_request_response.h b/src/native/http_request_response.h index 3a2c696a7..f68cb6bd4 100644 --- a/src/native/http_request_response.h +++ b/src/native/http_request_response.h @@ -40,4 +40,21 @@ void aws_http_stream_binding_destroy(JNIEnv *env, struct http_stream_binding *ca // If error occurs, A Java exception is thrown and NULL is returned. struct http_stream_binding *aws_http_stream_binding_alloc(JNIEnv *env, jobject java_callback_handler); +/* Default callbacks using binding */ +int aws_java_http_stream_on_incoming_headers_fn( + struct aws_http_stream *stream, + enum aws_http_header_block block_type, + const struct aws_http_header *header_array, + size_t num_headers, + void *user_data); +int aws_java_http_stream_on_incoming_header_block_done_fn( + struct aws_http_stream *stream, + enum aws_http_header_block block_type, + void *user_data); +int aws_java_http_stream_on_incoming_body_fn( + struct aws_http_stream *stream, + const struct aws_byte_cursor *data, + void *user_data); +void aws_java_http_stream_on_stream_complete_fn(struct aws_http_stream *stream, int error_code, void *user_data); + #endif /* AWS_JNI_CRT_HTTP_REQUEST_RESPONSE_H */ From 89369259b14394a729771262f7a13f43363cc4bb Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 2 Feb 2022 17:39:04 -0800 Subject: [PATCH 091/142] binding WIP --- src/native/http2_stream_manager.c | 109 +++++++++++++++++++---------- src/native/http_request_response.h | 1 + src/native/java_class_ids.c | 11 +++ src/native/java_class_ids.h | 6 ++ 4 files changed, 89 insertions(+), 38 deletions(-) diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c index a84d339e7..49b8ffcb0 100644 --- a/src/native/http2_stream_manager.c +++ b/src/native/http2_stream_manager.c @@ -4,7 +4,9 @@ */ #include "crt.h" +#include "http_connection_manager.h" #include "http_request_response.h" +#include "http_request_utils.h" #include "java_class_ids.h" #include @@ -45,6 +47,37 @@ struct aws_http2_stream_manager_binding { struct aws_http2_stream_manager *stream_manager; }; +static void s_destroy_manager_binding(struct aws_http2_stream_manager_binding *binding) { + if (binding == NULL) { + return; + } + + JNIEnv *env = aws_jni_get_thread_env(binding->jvm); + if (binding->java_http2_stream_manager != NULL) { + (*env)->DeleteWeakGlobalRef(env, binding->java_http2_stream_manager); + } + + aws_mem_release(aws_jni_get_allocator(), binding); +} + +static void s_on_http_conn_manager_shutdown_complete_callback(void *user_data) { + + struct aws_http2_stream_manager_binding *binding = (struct aws_http2_stream_manager_binding *)user_data; + JNIEnv *env = aws_jni_get_thread_env(binding->jvm); + + AWS_LOGF_DEBUG(AWS_LS_HTTP_STREAM_MANAGER, "Stream Manager Shutdown Complete"); + jobject java_http2_stream_manager = (*env)->NewLocalRef(env, binding->java_http2_stream_manager); + if (java_http2_stream_manager != NULL) { + (*env)->CallVoidMethod(env, java_http2_stream_manager, http2_stream_manager_properties.onShutdownComplete); + + AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); + (*env)->DeleteLocalRef(env, java_http2_stream_manager); + } + + // We're done with this wrapper, free it. + s_destroy_manager_binding(binding); +} + JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_http2StreamManagerNew( JNIEnv *env, jclass jni_class, @@ -66,16 +99,15 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ jlong jni_monitoring_throughput_threshold_in_bytes_per_second, jint jni_monitoring_failure_interval_in_seconds, jint jni_max_conns, - jint ideal_concurrent_streams_per_connection, - jint max_concurrent_streams_per_connection) { + jint jni_ideal_concurrent_streams_per_connection, + jint jni_max_concurrent_streams_per_connection) { (void)jni_class; - (void)jni_expected_protocol_version; struct aws_client_bootstrap *client_bootstrap = (struct aws_client_bootstrap *)jni_client_bootstrap; struct aws_socket_options *socket_options = (struct aws_socket_options *)jni_socket_options; struct aws_tls_ctx *tls_ctx = (struct aws_tls_ctx *)jni_tls_ctx; - struct http_connection_manager_binding *binding = NULL; + struct aws_http2_stream_manager_binding *binding = NULL; if (!client_bootstrap) { aws_jni_throw_runtime_exception(env, "ClientBootstrap can't be null"); @@ -117,15 +149,15 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ aws_tls_connection_options_set_server_name(&tls_conn_options, allocator, &endpoint); } - binding = aws_mem_calloc(allocator, 1, sizeof(struct http_connection_manager_binding)); + binding = aws_mem_calloc(allocator, 1, sizeof(struct aws_http2_stream_manager_binding)); AWS_FATAL_ASSERT(binding); - binding->java_http_conn_manager = (*env)->NewWeakGlobalRef(env, conn_manager_jobject); + binding->java_http2_stream_manager = (*env)->NewWeakGlobalRef(env, stream_manager_jobject); jint jvmresult = (*env)->GetJavaVM(env, &binding->jvm); (void)jvmresult; AWS_FATAL_ASSERT(jvmresult == 0); - struct aws_http_connection_manager_options manager_options; + struct aws_http2_stream_manager_options manager_options; AWS_ZERO_STRUCT(manager_options); manager_options.bootstrap = client_bootstrap; @@ -134,13 +166,15 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ manager_options.tls_connection_options = NULL; manager_options.host = endpoint; manager_options.port = port; - manager_options.max_connections = (size_t)jni_max_conns; manager_options.shutdown_complete_callback = &s_on_http_conn_manager_shutdown_complete_callback; manager_options.shutdown_complete_user_data = binding; manager_options.monitoring_options = NULL; /* TODO: this variable needs to be renamed in aws-c-http. Come back and change it next revision. */ manager_options.enable_read_back_pressure = jni_manual_window_management; - manager_options.max_connection_idle_in_milliseconds = jni_max_connection_idle_in_milliseconds; + + manager_options.max_connections = (size_t)jni_max_conns; + manager_options.ideal_concurrent_streams_per_connection = (size_t)jni_ideal_concurrent_streams_per_connection; + manager_options.max_concurrent_streams_per_connection = (size_t)jni_max_concurrent_streams_per_connection; if (use_tls) { manager_options.tls_connection_options = &tls_conn_options; @@ -179,10 +213,9 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ manager_options.proxy_options = &proxy_options; } - binding->manager = aws_http_connection_manager_new(allocator, &manager_options); - if (binding->manager == NULL) { - aws_jni_throw_runtime_exception( - env, "Failed to create connection manager: %s", aws_error_str(aws_last_error())); + binding->stream_manager = aws_http2_stream_manager_new(allocator, &manager_options); + if (binding->stream_manager == NULL) { + aws_jni_throw_runtime_exception(env, "Failed to create stream manager: %s", aws_error_str(aws_last_error())); } aws_http_proxy_options_jni_clean_up( @@ -195,7 +228,7 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ cleanup: aws_jni_byte_cursor_from_jbyteArray_release(env, jni_endpoint, endpoint); - if (binding->manager == NULL) { + if (binding->stream_manager == NULL) { s_destroy_manager_binding(binding); binding = NULL; } @@ -216,8 +249,8 @@ static void s_cleanup_sm_acquire_stream_callback_data(struct aws_sm_acquire_stre JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); - if (callback_data->async_callback) { - (*env)->DeleteGlobalRef(env, callback_data->async_callback); + if (callback_data->java_async_callback) { + (*env)->DeleteGlobalRef(env, callback_data->java_async_callback); } aws_mem_release(aws_jni_get_allocator(), callback_data); } @@ -232,35 +265,36 @@ static struct aws_sm_acquire_stream_callback_data *s_new_sm_acquire_stream_callb jint jvmresult = (*env)->GetJavaVM(env, &callback_data->jvm); AWS_FATAL_ASSERT(jvmresult == 0); - callback_data->async_callback = async_callback ? (*env)->NewGlobalRef(env, async_callback) : NULL; - AWS_FATAL_ASSERT(callback_data->async_callback != NULL); + callback_data->java_async_callback = async_callback ? (*env)->NewGlobalRef(env, async_callback) : NULL; + AWS_FATAL_ASSERT(callback_data->java_async_callback != NULL); callback_data->stream_binding = stream_binding; return callback_data; } -static void s_on_stream_acquired((struct aws_http_stream *stream, int error_code, void *user_data){ +static void s_on_stream_acquired(struct aws_http_stream *stream, int error_code, void *user_data) { struct aws_sm_acquire_stream_callback_data *callback_data = user_data; JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); if (error_code) { jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); - (*env)->CallVoidMethod(env, callback_data->async_callback, async_callback_properties.on_failure, crt_exception); + (*env)->CallVoidMethod( + env, callback_data->java_async_callback, async_callback_properties.on_failure, crt_exception); (*env)->DeleteLocalRef(env, crt_exception); aws_http_stream_binding_destroy(env, callback_data->stream_binding); } else { /* TODO: finalize */ callback_data->stream_binding->native_stream = stream; - jobject java_round_trip_time_ns = (*env)->NewObject( - env, boxed_long_properties.long_class, boxed_long_properties.constructor, (jlong)round_trip_time_ns); - (*env)->CallVoidMethod( - env, - callback_data->async_callback, - async_callback_properties.on_success_with_object, - java_round_trip_time_ns); - (*env)->DeleteLocalRef(env, java_round_trip_time_ns); + // jobject java_round_trip_time_ns = (*env)->NewObject( + // env, boxed_long_properties.long_class, boxed_long_properties.constructor, (jlong)round_trip_time_ns); + // (*env)->CallVoidMethod( + // env, + // callback_data->java_async_callback, + // async_callback_properties.on_success_with_object, + // java_round_trip_time_ns); + // (*env)->DeleteLocalRef(env, java_round_trip_time_ns); } AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); - s_cleanup_http2_callback_data(callback_data); + s_cleanup_sm_acquire_stream_callback_data(callback_data); } JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_http2StreamManagerAcquireStream( @@ -302,14 +336,13 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_h struct aws_allocator *allocator = aws_jni_get_allocator(); struct aws_sm_acquire_stream_callback_data *callback_data = - s_new_sm_acquire_stream_callback_data(env, allocator, java_async_callback); + s_new_sm_acquire_stream_callback_data(env, allocator, stream_binding, java_async_callback); - struct aws_http2_stream_manager_acquire_stream_options acquire_options = - { - .options = &request_options, - .callback = s_on_stream_acquired, - .user_data = callback_data, - } + struct aws_http2_stream_manager_acquire_stream_options acquire_options = { + .options = &request_options, + .callback = s_on_stream_acquired, + .user_data = callback_data, + }; - aws_http2_stream_manager_acquire_stream(sm_binding->stream_manager, acquire_options); -} \ No newline at end of file + aws_http2_stream_manager_acquire_stream(sm_binding->stream_manager, &acquire_options); +} diff --git a/src/native/http_request_response.h b/src/native/http_request_response.h index f68cb6bd4..d707b2b73 100644 --- a/src/native/http_request_response.h +++ b/src/native/http_request_response.h @@ -6,6 +6,7 @@ #ifndef AWS_JNI_CRT_HTTP_REQUEST_RESPONSE_H #define AWS_JNI_CRT_HTTP_REQUEST_RESPONSE_H +#include #include struct aws_http_message; diff --git a/src/native/java_class_ids.c b/src/native/java_class_ids.c index 3481d4949..f55e1b62d 100644 --- a/src/native/java_class_ids.c +++ b/src/native/java_class_ids.c @@ -366,6 +366,16 @@ static void s_cache_http_client_connection_manager(JNIEnv *env) { AWS_FATAL_ASSERT(http_client_connection_manager_properties.onShutdownComplete); } +struct java_http2_stream_manager_properties http2_stream_manager_properties; + +static void s_cache_http2_stream_manager(JNIEnv *env) { + jclass cls = (*env)->FindClass(env, "software/amazon/awssdk/crt/http/Http2StreamManager"); + AWS_FATAL_ASSERT(cls); + + http2_stream_manager_properties.onShutdownComplete = (*env)->GetMethodID(env, cls, "onShutdownComplete", "()V"); + AWS_FATAL_ASSERT(http2_stream_manager_properties.onShutdownComplete); +} + struct java_http_client_connection_properties http_client_connection_properties; static void s_cache_http_client_connection(JNIEnv *env) { @@ -811,6 +821,7 @@ void cache_java_class_ids(JNIEnv *env) { s_cache_client_bootstrap(env); s_cache_tls_context_pkcs11_options(env); s_cache_http_client_connection_manager(env); + s_cache_http2_stream_manager(env); s_cache_http_client_connection(env); s_cache_http_stream(env); s_cache_http2_stream(env); diff --git a/src/native/java_class_ids.h b/src/native/java_class_ids.h index 7e0560a4b..a3b4a1957 100644 --- a/src/native/java_class_ids.h +++ b/src/native/java_class_ids.h @@ -164,6 +164,12 @@ struct java_http_client_connection_manager_properties { }; extern struct java_http_client_connection_manager_properties http_client_connection_manager_properties; +/* Http2StreamManager */ +struct java_http2_stream_manager_properties { + jmethodID onShutdownComplete; +}; +extern struct java_http2_stream_manager_properties http2_stream_manager_properties; + /* HttpClientConnection */ struct java_http_client_connection_properties { jclass http_client_connection_class; From 8849aa35d7c3be2035d3b19398a6252b1f1c2586 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 3 Feb 2022 17:40:10 -0800 Subject: [PATCH 092/142] add tests and finalize binding --- src/native/http2_stream_manager.c | 67 +++++- .../crt/test/Http2StreamManagerTest.java | 203 ++++++++++++++++++ 2 files changed, 259 insertions(+), 11 deletions(-) create mode 100644 src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c index 49b8ffcb0..7d7de565b 100644 --- a/src/native/http2_stream_manager.c +++ b/src/native/http2_stream_manager.c @@ -65,7 +65,7 @@ static void s_on_http_conn_manager_shutdown_complete_callback(void *user_data) { struct aws_http2_stream_manager_binding *binding = (struct aws_http2_stream_manager_binding *)user_data; JNIEnv *env = aws_jni_get_thread_env(binding->jvm); - AWS_LOGF_DEBUG(AWS_LS_HTTP_STREAM_MANAGER, "Stream Manager Shutdown Complete"); + AWS_LOGF_DEBUG(AWS_LS_HTTP_STREAM_MANAGER, "Java Stream Manager Shutdown Complete"); jobject java_http2_stream_manager = (*env)->NewLocalRef(env, binding->java_http2_stream_manager); if (java_http2_stream_manager != NULL) { (*env)->CallVoidMethod(env, java_http2_stream_manager, http2_stream_manager_properties.onShutdownComplete); @@ -147,6 +147,8 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ if (use_tls) { aws_tls_connection_options_init_from_ctx(&tls_conn_options, tls_ctx); aws_tls_connection_options_set_server_name(&tls_conn_options, allocator, &endpoint); + /* Make sure we use h2 alpn list to create HTTP/2 connection. */ + aws_tls_connection_options_set_alpn_list(&tls_conn_options, allocator, "h2"); } binding = aws_mem_calloc(allocator, 1, sizeof(struct aws_http2_stream_manager_binding)); @@ -282,16 +284,26 @@ static void s_on_stream_acquired(struct aws_http_stream *stream, int error_code, (*env)->DeleteLocalRef(env, crt_exception); aws_http_stream_binding_destroy(env, callback_data->stream_binding); } else { - /* TODO: finalize */ callback_data->stream_binding->native_stream = stream; - // jobject java_round_trip_time_ns = (*env)->NewObject( - // env, boxed_long_properties.long_class, boxed_long_properties.constructor, (jlong)round_trip_time_ns); - // (*env)->CallVoidMethod( - // env, - // callback_data->java_async_callback, - // async_callback_properties.on_success_with_object, - // java_round_trip_time_ns); - // (*env)->DeleteLocalRef(env, java_round_trip_time_ns); + jobject j_http_stream = + aws_java_http_stream_from_native_new(env, callback_data->stream_binding, AWS_HTTP_VERSION_2); + if (!j_http_stream) { + jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, aws_last_error()); + (*env)->CallVoidMethod( + env, callback_data->java_async_callback, async_callback_properties.on_failure, crt_exception); + (*env)->DeleteLocalRef(env, crt_exception); + aws_http_stream_binding_destroy(env, callback_data->stream_binding); + } else { + /* Stream is activated once we acquired from the Stream Manager */ + aws_atomic_store_int(&callback_data->stream_binding->activated, 1); + callback_data->stream_binding->java_http_stream = (*env)->NewGlobalRef(env, j_http_stream); + (*env)->CallVoidMethod( + env, + callback_data->java_async_callback, + async_callback_properties.on_success_with_object, + callback_data->stream_binding->java_http_stream); + (*env)->DeleteLocalRef(env, j_http_stream); + } } AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); s_cleanup_sm_acquire_stream_callback_data(callback_data); @@ -307,8 +319,23 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_h jobject java_async_callback) { (void)jni_class; struct aws_http2_stream_manager_binding *sm_binding = (struct aws_http2_stream_manager_binding *)jni_stream_manager; + struct aws_http2_stream_manager *stream_manager = sm_binding->stream_manager; + + if (!stream_manager) { + aws_jni_throw_runtime_exception(env, "Stream Manager can't be null"); + return; + } + + if (!jni_http_response_callback_handler) { + aws_jni_throw_illegal_argument_exception( + env, "Http2StreamManager.acquireStream: Invalid jni_http_response_callback_handler"); + return; + } + if (!java_async_callback) { + aws_jni_throw_illegal_argument_exception(env, "Http2StreamManager.acquireStream: Invalid async callback"); + return; + } - /* TODO: check for null */ struct http_stream_binding *stream_binding = aws_http_stream_binding_alloc(env, jni_http_response_callback_handler); if (!stream_binding) { /* Exception already thrown */ @@ -346,3 +373,21 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_h aws_http2_stream_manager_acquire_stream(sm_binding->stream_manager, &acquire_options); } + +JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_http2StreamManagerRelease( + JNIEnv *env, + jclass jni_class, + jlong jni_stream_manager) { + (void)jni_class; + + struct aws_http2_stream_manager_binding *sm_binding = (struct aws_http2_stream_manager_binding *)jni_stream_manager; + struct aws_http2_stream_manager *stream_manager = sm_binding->stream_manager; + + if (!stream_manager) { + aws_jni_throw_runtime_exception(env, "Stream Manager can't be null"); + return; + } + + AWS_LOGF_DEBUG(AWS_LS_HTTP_CONNECTION, "Releasing StreamManager: id: %p", (void *)stream_manager); + aws_http2_stream_manager_release(stream_manager); +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java new file mode 100644 index 000000000..0c20d12d9 --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java @@ -0,0 +1,203 @@ +package software.amazon.awssdk.crt.test; + +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.Http2StreamManager; +import software.amazon.awssdk.crt.http.Http2Request; +import software.amazon.awssdk.crt.http.Http2Stream; +import software.amazon.awssdk.crt.http.Http2StreamManagerOptions; +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpProxyOptions; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; +import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.Log; + +public class Http2StreamManagerTest extends HttpClientTestFixture { + private final static Charset UTF8 = StandardCharsets.UTF_8; + private final static int NUM_THREADS = 10; + private final static int NUM_CONNECTIONS = 20; + private final static int NUM_REQUESTS = 60; + private final static int NUM_ITERATIONS = 10; + private final static int GROWTH_PER_THREAD = 0; // expected VM footprint growth per thread + private final static int EXPECTED_HTTP_STATUS = 200; + private final static String endpoint = "https://d1cz66xoahf9cl.cloudfront.net/"; // Use cloudfront for HTTP/2 + private final static String path = "/random_32_byte.data"; + private final String EMPTY_BODY = ""; + + private Http2StreamManager createStreamManager(URI uri, int numConnections) { + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContext tlsContext = createHttpClientTlsContext()) { + + Http2StreamManagerOptions options = new Http2StreamManagerOptions(); + options.withClientBootstrap(bootstrap) + .withSocketOptions(sockOpts) + .withTlsContext(tlsContext) + .withUri(uri) + .withMaxConnections(numConnections); + + return Http2StreamManager.create(options); + } + } + + private Http2Request createHttp2Request(String method, String endpoint, String path, String requestBody) + throws Exception { + URI uri = new URI(endpoint); + HttpHeader[] requestHeaders = new HttpHeader[] { + new HttpHeader(":method", method), + new HttpHeader(":path", path), + new HttpHeader(":scheme", uri.getScheme()), + new HttpHeader(":authority", uri.getHost()), + new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)) + }; + Http2Request request = new Http2Request(requestHeaders, null); + + return request; + } + + private void testParallelStreams(Http2StreamManager streamManager, Http2Request request, int numThreads, + int numRequests) { + final AtomicInteger numRequestsMade = new AtomicInteger(0); + final AtomicInteger numStreamsFailures = new AtomicInteger(0); + final ConcurrentHashMap reqIdToStatus = new ConcurrentHashMap<>(); + final AtomicInteger numErrorCode = new AtomicInteger(0); + + final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads); + List> requestCompleteFutures = new ArrayList<>(); + + for (int i = 0; i < numRequests; i++) { + + Log.log(Log.LogLevel.Trace, Log.LogSubject.HttpConnectionManager, String.format("Starting request %d", i)); + CompletableFuture requestCompleteFuture = new CompletableFuture(); + requestCompleteFutures.add(requestCompleteFuture); + + threadPool.execute(() -> { + // Request a connection from the connection pool + int requestId = numRequestsMade.incrementAndGet(); + streamManager.acquireStream(request, new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + reqIdToStatus.put(requestId, responseStatusCode); + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + if (errorCode != CRT.AWS_CRT_SUCCESS) { + numErrorCode.incrementAndGet(); + } + stream.close(); + requestCompleteFuture.complete(null); + } + }).whenComplete((stream, throwable) -> { + if (throwable != null) { + numStreamsFailures.incrementAndGet(); + requestCompleteFuture.completeExceptionally(throwable); + } + }); + }); + } + + // Wait for all Requests to complete + for (CompletableFuture f : requestCompleteFutures) { + f.join(); + } + + final int requiredSuccesses = (int) Math.floor(numRequests * 0.95); + final int allowedFailures = numRequests - requiredSuccesses; + + // Verify we got some Http Status Code for each Request + Assert.assertTrue(reqIdToStatus.size() >= requiredSuccesses); + // Verify that the failure counts aren't too high ????? + Assert.assertTrue(numErrorCode.get() <= allowedFailures); + Assert.assertTrue(numStreamsFailures.get() <= allowedFailures); + } + + public void testParallelRequests(int numThreads, int numRequests) throws Exception { + skipIfNetworkUnavailable(); + + URI uri = new URI(endpoint); + + try (Http2StreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS)) { + Http2Request request = createHttp2Request("GET", endpoint, path, EMPTY_BODY); + testParallelStreams(streamManager, request, 1, numRequests); + } + + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + public void testParallelRequestsWithLeakCheck(int numThreads, int numRequests) throws Exception { + skipIfNetworkUnavailable(); + Callable fn = () -> { + testParallelRequests(numThreads, numRequests); + Thread.sleep(2000); // wait for async shutdowns to complete + return null; + }; + + // Dalvik is SUPER STOCHASTIC about when it frees JVM memory, it has no + // observable correlation + // to when System.gc() is called. Therefore, we cannot reliably sample it, so we + // don't bother. + // If we have a leak, we should have it on all platforms, and we'll catch it + // elsewhere. + if (CRT.getOSIdentifier() != "android") { + int fixedGrowth = CrtMemoryLeakDetector.expectedFixedGrowth(); + fixedGrowth += (numThreads * GROWTH_PER_THREAD); + // On Mac, JVM seems to expand by about 4K no matter how careful we are. With + // the workload + // we're running, 8K worth of growth (an additional 4K for an increased healthy + // margin) + // in the JVM only is acceptable. + fixedGrowth = Math.max(fixedGrowth, 8192); + CrtMemoryLeakDetector.leakCheck(NUM_ITERATIONS, fixedGrowth, fn); + } + } + + @Test + public void testSanitizer() throws Exception { + Log.initLoggingToStderr(Log.LogLevel.Trace); + URI uri = new URI(endpoint); + try (Http2StreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS)) { + } + + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testSerialRequests() throws Exception { + Log.initLoggingToStderr(Log.LogLevel.Trace); + testParallelRequestsWithLeakCheck(1, NUM_REQUESTS / NUM_THREADS); + } + + @Test + public void testMaxParallelRequests() throws Exception { + testParallelRequestsWithLeakCheck(NUM_THREADS, NUM_REQUESTS); + } +} From b33ba8db404c6864c71d39f61ad241dd1d7df479 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 3 Feb 2022 20:15:27 -0800 Subject: [PATCH 093/142] git rid of the logs --- .../software/amazon/awssdk/crt/test/Http2StreamManagerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java index 0c20d12d9..921a34901 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java @@ -181,7 +181,6 @@ public void testParallelRequestsWithLeakCheck(int numThreads, int numRequests) t @Test public void testSanitizer() throws Exception { - Log.initLoggingToStderr(Log.LogLevel.Trace); URI uri = new URI(endpoint); try (Http2StreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS)) { } @@ -192,7 +191,6 @@ public void testSanitizer() throws Exception { @Test public void testSerialRequests() throws Exception { - Log.initLoggingToStderr(Log.LogLevel.Trace); testParallelRequestsWithLeakCheck(1, NUM_REQUESTS / NUM_THREADS); } From 279f5abbd63171336ddd84e72b16d8c7c2cf0e87 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 3 Feb 2022 22:03:54 -0800 Subject: [PATCH 094/142] use the version we got instead --- src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index eea869ae7..6d7fa7025 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -333,7 +333,7 @@ public void onResponseComplete(HttpStream stream, int errorCode) { stream.close(); } }; - HttpRequestBase request = buildHttpRequest(cli, requiredVersion, uri); + HttpRequestBase request = buildHttpRequest(cli, conn.getVersion(), uri); HttpStream stream = conn.makeRequest(request, streamHandler); stream.activate(); From 15a6e3586182d28d204f08b74ec853abcf6e3914 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 9 Feb 2022 10:31:24 -0800 Subject: [PATCH 095/142] update submodules --- crt/aws-c-auth | 2 +- crt/aws-c-http | 2 +- crt/aws-c-io | 2 +- crt/aws-c-sdkutils | 2 +- crt/s2n | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crt/aws-c-auth b/crt/aws-c-auth index f29ffca89..e1b95cca6 160000 --- a/crt/aws-c-auth +++ b/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit f29ffca89dc0b6a1d722ae9bacb230ec9dc765f4 +Subproject commit e1b95cca6f2248c28b66ddb40bcccd35a59cb8b5 diff --git a/crt/aws-c-http b/crt/aws-c-http index 0426a065b..ad7d98cad 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 0426a065b77e99ce8dbbe2e89f6e9fae35840728 +Subproject commit ad7d98cadbc8c9a26a10232ab82c0adb0846d163 diff --git a/crt/aws-c-io b/crt/aws-c-io index e538d31d4..fb9d38bf7 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit e538d31d458e25918eed892a998c697386f7800e +Subproject commit fb9d38bf7873c649a9e1513c2f6675072ac06950 diff --git a/crt/aws-c-sdkutils b/crt/aws-c-sdkutils index 399f5253c..e3c23f4ac 160000 --- a/crt/aws-c-sdkutils +++ b/crt/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit 399f5253c42c673091dca131bcf51f20f3de194c +Subproject commit e3c23f4aca31d9e66df25827645f72cbcbfb657a diff --git a/crt/s2n b/crt/s2n index 785596b96..b956eab5f 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 785596b9694d7e12274afea9f42b8216a07102da +Subproject commit b956eab5f1676a961b16cb060d41c3844e118af2 From 2fa36cce448efe9c92a795b7eadf060d91a0c782 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 3 Feb 2022 22:03:54 -0800 Subject: [PATCH 096/142] use the version we got instead --- src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index eea869ae7..6d7fa7025 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -333,7 +333,7 @@ public void onResponseComplete(HttpStream stream, int errorCode) { stream.close(); } }; - HttpRequestBase request = buildHttpRequest(cli, requiredVersion, uri); + HttpRequestBase request = buildHttpRequest(cli, conn.getVersion(), uri); HttpStream stream = conn.makeRequest(request, streamHandler); stream.activate(); From 6b606c85f15455616dc919f04ba6bedede83d3d1 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 18 Feb 2022 12:02:36 -0800 Subject: [PATCH 097/142] comment addressed --- .../awssdk/crt/http/Http2StreamManager.java | 18 ++++- ...ons.java => HttpStreamManagerOptions.java} | 68 +++++++++++++++---- src/native/http2_stream_manager.c | 2 - .../crt/test/Http2StreamManagerTest.java | 16 +++-- 4 files changed, 81 insertions(+), 23 deletions(-) rename src/main/java/software/amazon/awssdk/crt/http/{Http2StreamManagerOptions.java => HttpStreamManagerOptions.java} (66%) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index 05d58cd47..ba9f8486d 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -11,6 +11,10 @@ import java.net.URI; import java.nio.charset.Charset; +/** + * Manages a Pool of HTTP/2 Streams. Creates and manages HTTP/2 connections + * under the hood. + */ public class Http2StreamManager extends CrtResource { private static final String HTTP = "http"; @@ -33,11 +37,11 @@ public class Http2StreamManager extends CrtResource { * @param options configuration options * @return a new instance of an Http2StreamManager */ - public static Http2StreamManager create(Http2StreamManagerOptions options) { + public static Http2StreamManager create(HttpStreamManagerOptions options) { return new Http2StreamManager(options); } - private Http2StreamManager(Http2StreamManagerOptions options) { + private Http2StreamManager(HttpStreamManagerOptions options) { URI uri = options.getUri(); if (uri == null) { throw new IllegalArgumentException("URI must not be null"); @@ -179,6 +183,16 @@ private Http2StreamManager(Http2StreamManagerOptions options) { */ public CompletableFuture acquireStream(Http2Request request, HttpStreamResponseHandler streamHandler) { + return this.acquireStream((HttpRequestBase) request, streamHandler); + } + + public CompletableFuture acquireStream(HttpRequest request, + HttpStreamResponseHandler streamHandler) { + return this.acquireStream((HttpRequestBase) request, streamHandler); + } + + private CompletableFuture acquireStream(HttpRequestBase request, + HttpStreamResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); AsyncCallback acquireStreamCompleted = AsyncCallback.wrapFuture(completionFuture, null); if (isNull()) { diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java similarity index 66% rename from src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java rename to src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java index 66eb4b492..be4477d33 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java @@ -9,7 +9,7 @@ * Contains all the configuration options for a Http2StreamManager * instance */ -public class Http2StreamManagerOptions { +public class HttpStreamManagerOptions { public static final int DEFAULT_MAX_WINDOW_SIZE = Integer.MAX_VALUE; public static final int DEFAULT_MAX = Integer.MAX_VALUE; public static final int DEFAULT_MAX_CONNECTIONS = 2; @@ -32,7 +32,7 @@ public class Http2StreamManagerOptions { /** * Default constructor */ - public Http2StreamManagerOptions() { + public HttpStreamManagerOptions() { } /** @@ -41,7 +41,7 @@ public Http2StreamManagerOptions() { * @param clientBootstrap ClientBootstrap to use * @return this */ - public Http2StreamManagerOptions withClientBootstrap(ClientBootstrap clientBootstrap) { + public HttpStreamManagerOptions withClientBootstrap(ClientBootstrap clientBootstrap) { this.clientBootstrap = clientBootstrap; return this; } @@ -62,7 +62,7 @@ public ClientBootstrap getClientBootstrap() { * manager * @return this */ - public Http2StreamManagerOptions withSocketOptions(SocketOptions socketOptions) { + public HttpStreamManagerOptions withSocketOptions(SocketOptions socketOptions) { this.socketOptions = socketOptions; return this; } @@ -80,7 +80,7 @@ public SocketOptions getSocketOptions() { * @param tlsContext The TlsContext to use * @return this */ - public Http2StreamManagerOptions withTlsContext(TlsContext tlsContext) { + public HttpStreamManagerOptions withTlsContext(TlsContext tlsContext) { this.tlsContext = tlsContext; return this; } @@ -98,7 +98,7 @@ public TlsContext getTlsContext() { * @param windowSize The initial window size to use for each connection * @return this */ - public Http2StreamManagerOptions withWindowSize(int windowSize) { + public HttpStreamManagerOptions withWindowSize(int windowSize) { this.windowSize = windowSize; return this; } @@ -111,20 +111,52 @@ public int getWindowSize() { return windowSize; } - public Http2StreamManagerOptions withIdealConcurrentStreamsPerConnection(int idealConcurrentStreamsPerConnection) { + /** + * For HTTP/2 stream manager only. + * + * The ideal number of concurrent streams for a connection. Stream manager will + * try to create a new connection if one connection reaches this number. But, if + * the max connections reaches, manager will reuse connections to create the + * acquired steams as much as possible. + * + * @param idealConcurrentStreamsPerConnection The ideal number of concurrent + * streams for a connection + * @return this + */ + public HttpStreamManagerOptions withIdealConcurrentStreamsPerConnection(int idealConcurrentStreamsPerConnection) { this.idealConcurrentStreamsPerConnection = idealConcurrentStreamsPerConnection; return this; } + /** + * @return The ideal number of concurrent streams for a connection used for + * manager + */ public int getIdealConcurrentStreamsPerConnection() { return idealConcurrentStreamsPerConnection; } - public Http2StreamManagerOptions withMaxConcurrentStreamsPerConnection(int maxConcurrentStreamsPerConnection) { + /** + * Default is no limit, which will use the limit from the server. 0 will be + * considered as using the default value. + * The real number of concurrent streams per connection will be controlled by + * the minimal value of the setting from other end and the value here. + * + * @param maxConcurrentStreamsPerConnection The max number of concurrent + * streams for a connection + * @return + */ + public HttpStreamManagerOptions withMaxConcurrentStreamsPerConnection(int maxConcurrentStreamsPerConnection) { this.maxConcurrentStreamsPerConnection = maxConcurrentStreamsPerConnection; return this; } + /** + * @return The max number of concurrent streams for a connection set for + * manager. + * It could be different than the real limits, which is the minimal set + * for manager and the settings from the other side. + */ public int getMaxConcurrentStreamsPerConnection() { return maxConcurrentStreamsPerConnection; } @@ -135,7 +167,7 @@ public int getMaxConcurrentStreamsPerConnection() { * @param uri The endpoint URI to connect to * @return this */ - public Http2StreamManagerOptions withUri(URI uri) { + public HttpStreamManagerOptions withUri(URI uri) { this.uri = uri; return this; } @@ -153,7 +185,7 @@ public URI getUri() { * @param port The port to connect to * @return this */ - public Http2StreamManagerOptions withPort(int port) { + public HttpStreamManagerOptions withPort(int port) { this.port = port; return this; } @@ -166,7 +198,15 @@ public int getPort() { return port; } - public Http2StreamManagerOptions withMaxConnections(int maxConnections) { + /** + * The max number of connections will be open at same time. If all the + * connections are full, manager will wait until available to vender more + * streams + * + * @param maxConnections + * @return + */ + public HttpStreamManagerOptions withMaxConnections(int maxConnections) { this.maxConnections = maxConnections; return this; } @@ -178,7 +218,7 @@ public int getMaxConnections() { return maxConnections; } - public Http2StreamManagerOptions withProxyOptions(HttpProxyOptions proxyOptions) { + public HttpStreamManagerOptions withProxyOptions(HttpProxyOptions proxyOptions) { this.proxyOptions = proxyOptions; return this; } @@ -194,7 +234,7 @@ public boolean isManualWindowManagement() { return manualWindowManagement; } - public Http2StreamManagerOptions withManualWindowManagement(boolean manualWindowManagement) { + public HttpStreamManagerOptions withManualWindowManagement(boolean manualWindowManagement) { this.manualWindowManagement = manualWindowManagement; return this; } @@ -206,7 +246,7 @@ public Http2StreamManagerOptions withManualWindowManagement(boolean manualWindow * null to disable monitoring * @return this */ - public Http2StreamManagerOptions withMonitoringOptions(HttpMonitoringOptions monitoringOptions) { + public HttpStreamManagerOptions withMonitoringOptions(HttpMonitoringOptions monitoringOptions) { this.monitoringOptions = monitoringOptions; return this; } diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c index 7d7de565b..596ab3e52 100644 --- a/src/native/http2_stream_manager.c +++ b/src/native/http2_stream_manager.c @@ -147,8 +147,6 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ if (use_tls) { aws_tls_connection_options_init_from_ctx(&tls_conn_options, tls_ctx); aws_tls_connection_options_set_server_name(&tls_conn_options, allocator, &endpoint); - /* Make sure we use h2 alpn list to create HTTP/2 connection. */ - aws_tls_connection_options_set_alpn_list(&tls_conn_options, allocator, "h2"); } binding = aws_mem_calloc(allocator, 1, sizeof(struct aws_http2_stream_manager_binding)); diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java index 921a34901..ae8050aaa 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java @@ -19,7 +19,7 @@ import software.amazon.awssdk.crt.http.Http2StreamManager; import software.amazon.awssdk.crt.http.Http2Request; import software.amazon.awssdk.crt.http.Http2Stream; -import software.amazon.awssdk.crt.http.Http2StreamManagerOptions; +import software.amazon.awssdk.crt.http.HttpStreamManagerOptions; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; @@ -33,6 +33,7 @@ import software.amazon.awssdk.crt.io.HostResolver; import software.amazon.awssdk.crt.io.SocketOptions; import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.io.TlsContextOptions; import software.amazon.awssdk.crt.Log; public class Http2StreamManagerTest extends HttpClientTestFixture { @@ -48,13 +49,14 @@ public class Http2StreamManagerTest extends HttpClientTestFixture { private final String EMPTY_BODY = ""; private Http2StreamManager createStreamManager(URI uri, int numConnections) { + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); HostResolver resolver = new HostResolver(eventLoopGroup); ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); SocketOptions sockOpts = new SocketOptions(); - TlsContext tlsContext = createHttpClientTlsContext()) { - - Http2StreamManagerOptions options = new Http2StreamManagerOptions(); + TlsContextOptions tlsOpts = TlsContextOptions.createDefaultClient().withAlpnList("h2"); + TlsContext tlsContext = createHttpClientTlsContext(tlsOpts)) { + HttpStreamManagerOptions options = new HttpStreamManagerOptions(); options.withClientBootstrap(bootstrap) .withSocketOptions(sockOpts) .withTlsContext(tlsContext) @@ -108,7 +110,11 @@ public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blo @Override public void onResponseComplete(HttpStream stream, int errorCode) { - if (errorCode != CRT.AWS_CRT_SUCCESS) { + if (errorCode != CRT.AWS_CRT_SUCCESS + || stream.getResponseStatusCode() != EXPECTED_HTTP_STATUS) { + Log.log(Log.LogLevel.Error, Log.LogSubject.HttpConnectionManager, + String.format("Response completed with error: error_code=%s, response status=%d", + CRT.awsErrorName(errorCode), stream.getResponseStatusCode())); numErrorCode.incrementAndGet(); } stream.close(); From c65ff375d169bac4581a773ec67c3120d746995a Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 09:16:04 -0800 Subject: [PATCH 098/142] unified stream manager --- .../awssdk/crt/http/Http2StreamManager.java | 2 +- .../awssdk/crt/http/HttpStreamManager.java | 180 ++++++++++++ .../crt/test/HttpStreamManagerTest.java | 260 ++++++++++++++++++ 3 files changed, 441 insertions(+), 1 deletion(-) create mode 100644 src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java create mode 100644 src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index ba9f8486d..fe07bfffa 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -191,7 +191,7 @@ public CompletableFuture acquireStream(HttpRequest request, return this.acquireStream((HttpRequestBase) request, streamHandler); } - private CompletableFuture acquireStream(HttpRequestBase request, + public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); AsyncCallback acquireStreamCompleted = AsyncCallback.wrapFuture(completionFuture, null); diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java new file mode 100644 index 000000000..33905333e --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -0,0 +1,180 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package software.amazon.awssdk.crt.http; + +import software.amazon.awssdk.crt.CrtRuntimeException; + +import java.util.concurrent.CompletableFuture; + +/** + * Manages a pool for either HTTP/1.1 or HTTP/2 connection. + * For HTTP/1.1, it will grab a connection from HttpClientConnectionManager to + * make request on it, and will return it back until the request finishes. + * For HTTP/2, it will use Http2StreamManager under the hood. + */ +public class HttpStreamManager implements AutoCloseable { + + private HttpClientConnectionManager connectionManager = null; + private Http2StreamManager h2StreamManager = null; + private Exception initException = null; + + /** + * Factory function for HttpStreamManager instances + * + * @param options configuration options + * @return a new instance of an HttpStreamManager + */ + public static HttpStreamManager create(HttpStreamManagerOptions options) { + return new HttpStreamManager(options); + } + + private HttpClientConnectionManagerOptions getConnManagerOptFromStreamManagerOpt(HttpStreamManagerOptions options) { + + HttpClientConnectionManagerOptions connManagerOptions = new HttpClientConnectionManagerOptions(); + connManagerOptions.withClientBootstrap(options.getClientBootstrap()) + .withSocketOptions(options.getSocketOptions()) + .withTlsContext(options.getTlsContext()) + .withWindowSize(options.getWindowSize()) + .withUri(options.getUri()) + .withMaxConnections(options.getMaxConnections()) + .withPort(options.getPort()) + .withProxyOptions(options.getProxyOptions()) + .withManualWindowManagement(options.isManualWindowManagement()) + .withMonitoringOptions(options.getMonitoringOptions()); + return connManagerOptions; + } + + private HttpStreamManager(HttpStreamManagerOptions options) { + HttpClientConnectionManagerOptions connManagerOptions = getConnManagerOptFromStreamManagerOpt(options); + this.connectionManager = HttpClientConnectionManager.create(connManagerOptions); + try (HttpClientConnection conn = this.connectionManager.acquireConnection().get()) { + if (conn.getVersion() == HttpVersion.HTTP_2) { + /* + * Create Http2StreamManager and clean up connection manager later as we don't + * need it anymore + */ + this.h2StreamManager = Http2StreamManager.create(options); + this.connectionManager.releaseConnection(conn); + this.connectionManager.close(); + this.connectionManager = null; + } else { + this.connectionManager.releaseConnection(conn); + } + } catch (Exception ex) { + this.initException = ex; + } + } + + /** + * Get the protocol version the manager runs on. + */ + public HttpVersion getHttpVersion() { + if (this.h2StreamManager != null) { + return HttpVersion.HTTP_2; + } + return HttpVersion.HTTP_1_1; + } + + /** + * Request a HttpStream from StreamManager. If the streamManager is made with + * HTTP/2 connection under the hood, it will be Http2Stream. + * + * @param request + * @param streamHandler + * @return A future for a Http2Stream that will be completed when the stream is + * acquired. + * @throws CrtRuntimeException + */ + public CompletableFuture acquireStream(Http2Request request, + HttpStreamResponseHandler streamHandler) { + return this.acquireStream((HttpRequestBase) request, streamHandler); + } + + public CompletableFuture acquireStream(HttpRequest request, + HttpStreamResponseHandler streamHandler) { + return this.acquireStream((HttpRequestBase) request, streamHandler); + } + + private CompletableFuture acquireStream(HttpRequestBase request, + HttpStreamResponseHandler streamHandler) { + CompletableFuture completionFuture = new CompletableFuture<>(); + + if (this.initException != null) { + completionFuture.completeExceptionally(this.initException); + return completionFuture; + } + if (this.h2StreamManager != null) { + this.h2StreamManager.acquireStream(request, streamHandler).whenComplete((stream, throwable) -> { + if (throwable != null) { + completionFuture.completeExceptionally(throwable); + } else { + completionFuture.complete((HttpStream) stream); + } + }); + return completionFuture; + } + HttpClientConnectionManager connManager = this.connectionManager; + this.connectionManager.acquireConnection().whenComplete((conn, throwable) -> { + if (throwable != null) { + completionFuture.completeExceptionally(throwable); + } else { + try { + HttpStream stream = conn.makeRequest(request, new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + streamHandler.onResponseHeaders(stream, responseStatusCode, blockType, nextHeaders); + } + + @Override + public void onResponseHeadersDone(HttpStream stream, int blockType) { + streamHandler.onResponseHeadersDone(stream, blockType); + } + + @Override + public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + return streamHandler.onResponseBody(stream, bodyBytesIn); + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + streamHandler.onResponseComplete(stream, errorCode); + /* Release the connection back */ + connManager.releaseConnection(conn); + } + }); + completionFuture.complete(stream); + /* Active the stream for user */ + try { + stream.activate(); + } catch (CrtRuntimeException e) { + /* If activate failed, complete callback will not be invoked */ + streamHandler.onResponseComplete(stream, e.errorCode); + /* Release the connection back */ + connManager.releaseConnection(conn); + } + } catch (Exception ex) { + connManager.releaseConnection(conn); + completionFuture.completeExceptionally(ex); + } + } + }); + return completionFuture; + } + + @Override + public void close() { + /** + * TODO: is this async? Should we wait for the underlying structure cleaned up? + */ + if (this.connectionManager != null) { + this.connectionManager.close(); + } + if (this.h2StreamManager != null) { + this.h2StreamManager.close(); + } + } +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java new file mode 100644 index 000000000..01657a722 --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -0,0 +1,260 @@ +package software.amazon.awssdk.crt.test; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.management.RuntimeErrorException; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.http.HttpStreamManager; +import software.amazon.awssdk.crt.http.Http2StreamManager; +import software.amazon.awssdk.crt.http.Http2Request; +import software.amazon.awssdk.crt.http.Http2Stream; +import software.amazon.awssdk.crt.http.HttpStreamManagerOptions; +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpProxyOptions; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; +import software.amazon.awssdk.crt.http.HttpVersion; +import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.io.TlsContextOptions; +import software.amazon.awssdk.crt.utils.ByteBufferUtils; +import software.amazon.awssdk.crt.Log; +import software.amazon.awssdk.crt.Log.LogLevel; + +public class HttpStreamManagerTest extends HttpClientTestFixture { + private final static String endpoint = "https://httpbin.org"; + private final static String path = "/anything"; + private final String EMPTY_BODY = ""; + private final static int NUM_CONNECTIONS = 20; + private final static Charset UTF8 = StandardCharsets.UTF_8; + private final static int EXPECTED_HTTP_STATUS = 200; + + private HttpStreamManager createStreamManager(URI uri, int numConnections, HttpVersion expectedVersion) { + + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContextOptions tlsOpts = expectedVersion == HttpVersion.HTTP_2 + ? TlsContextOptions.createDefaultClient().withAlpnList("h2") + : TlsContextOptions.createDefaultClient().withAlpnList("http/1.1"); + TlsContext tlsContext = createHttpClientTlsContext(tlsOpts)) { + HttpStreamManagerOptions options = new HttpStreamManagerOptions(); + options.withClientBootstrap(bootstrap) + .withSocketOptions(sockOpts) + .withTlsContext(tlsContext) + .withUri(uri) + .withMaxConnections(numConnections); + + return HttpStreamManager.create(options); + } + } + + private Http2Request createHttp2Request(String method, String endpoint, String path, String requestBody) + throws Exception { + URI uri = new URI(endpoint); + HttpHeader[] requestHeaders = new HttpHeader[] { + new HttpHeader(":method", method), + new HttpHeader(":path", path), + new HttpHeader(":scheme", uri.getScheme()), + new HttpHeader(":authority", uri.getHost()), + new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)) + }; + final ByteBuffer payload = ByteBuffer.wrap(requestBody.getBytes()); + HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() { + @Override + public boolean sendRequestBody(ByteBuffer outBuffer) { + ByteBufferUtils.transferData(payload, outBuffer); + return payload.remaining() == 0; + } + + @Override + public boolean resetPosition() { + return true; + } + + @Override + public long getLength() { + return payload.capacity(); + } + }; + Http2Request request = new Http2Request(requestHeaders, payloadStream); + + return request; + } + + private HttpRequest createHttp1Request(String method, String endpoint, String path, String requestBody) + throws Exception { + URI uri = new URI(endpoint); + HttpHeader[] requestHeaders = new HttpHeader[] { + new HttpHeader("host", uri.getHost()), + new HttpHeader("content-length", Integer.toString(requestBody.getBytes(UTF8).length)) + }; + final ByteBuffer payload = ByteBuffer.wrap(requestBody.getBytes()); + HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() { + @Override + public boolean sendRequestBody(ByteBuffer outBuffer) { + ByteBufferUtils.transferData(payload, outBuffer); + return payload.remaining() == 0; + } + + @Override + public boolean resetPosition() { + return true; + } + + @Override + public long getLength() { + return payload.capacity(); + } + }; + return new HttpRequest(method, path, requestHeaders, payloadStream); + } + + @Test + public void testSanitizerHTTP1() throws Exception { + URI uri = new URI(endpoint); + try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { + } + + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testSanitizerHTTP2() throws Exception { + URI uri = new URI(endpoint); + try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_2)) { + } + + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testSingleHTTP2Requests() throws Exception { + URI uri = new URI(endpoint); + try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_2)) { + Http2Request request = createHttp2Request("GET", endpoint, path, EMPTY_BODY); + + streamManager.acquireStream(request, new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + Assert.assertEquals(responseStatusCode, EXPECTED_HTTP_STATUS); + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + Assert.assertEquals(errorCode, CRT.AWS_CRT_SUCCESS); + stream.close(); + } + }).get(); + } + + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testSingleHTTP1Request() throws Throwable { + Log.initLoggingToStderr(LogLevel.Trace); + URI uri = new URI(endpoint); + try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { + /* + * http2 request which will have :method headers that is not allowed for + * HTTP/1.1 + */ + HttpRequest request = createHttp1Request("GET", endpoint, path, EMPTY_BODY); + + streamManager.acquireStream(request, new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + Assert.assertEquals(responseStatusCode, EXPECTED_HTTP_STATUS); + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + Assert.assertEquals(errorCode, CRT.AWS_CRT_SUCCESS); + stream.close(); + } + }).get(); + } catch (ExecutionException e) { + Assert.assertNull(e); + } + + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + /* + * Create HTTP/1.1 stream manager, with HTTP/2 request, which should fail with + * invalid header name. Make sure the exception pops out and everything clean up + * correctly. + */ + @Test + public void testSingleHTTP1RequestsFailure() throws Throwable { + URI uri = new URI(endpoint); + try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { + /* + * http2 request which will have :method headers that is not allowed for + * HTTP/1.1 + */ + Http2Request request = createHttp2Request("GET", endpoint, path, EMPTY_BODY); + + streamManager.acquireStream(request, new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + Assert.assertEquals(responseStatusCode, EXPECTED_HTTP_STATUS); + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + Assert.assertEquals(errorCode, CRT.AWS_CRT_SUCCESS); + stream.close(); + } + }).get(); + } catch (ExecutionException e) { + try { + throw e.getCause(); + } catch (CrtRuntimeException causeException) { + /** + * Assert the exceptions are set correctly. + */ + Assert.assertTrue(causeException.errorName.equals("AWS_ERROR_HTTP_INVALID_HEADER_NAME")); + } + } + + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } +} From 82fd0dda045bc2e9d34d1359a133229a28395504 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 10:26:25 -0800 Subject: [PATCH 099/142] update test and add shutdown complete future --- .../awssdk/crt/http/HttpStreamManager.java | 12 +- .../crt/test/HttpStreamManagerTest.java | 124 +++++++++++------- 2 files changed, 85 insertions(+), 51 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java index 33905333e..89382bb0c 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -20,6 +20,7 @@ public class HttpStreamManager implements AutoCloseable { private HttpClientConnectionManager connectionManager = null; private Http2StreamManager h2StreamManager = null; private Exception initException = null; + private CompletableFuture shutdownComplete = null; /** * Factory function for HttpStreamManager instances @@ -60,8 +61,10 @@ private HttpStreamManager(HttpStreamManagerOptions options) { this.connectionManager.releaseConnection(conn); this.connectionManager.close(); this.connectionManager = null; + this.shutdownComplete = this.h2StreamManager.getShutdownCompleteFuture(); } else { this.connectionManager.releaseConnection(conn); + this.shutdownComplete = this.connectionManager.getShutdownCompleteFuture(); } } catch (Exception ex) { this.initException = ex; @@ -98,7 +101,7 @@ public CompletableFuture acquireStream(HttpRequest request, return this.acquireStream((HttpRequestBase) request, streamHandler); } - private CompletableFuture acquireStream(HttpRequestBase request, + public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); @@ -165,11 +168,12 @@ public void onResponseComplete(HttpStream stream, int errorCode) { return completionFuture; } + public CompletableFuture getShutdownCompleteFuture() { + return shutdownComplete; + } + @Override public void close() { - /** - * TODO: is this async? Should we wait for the underlying structure cleaned up? - */ if (this.connectionManager != null) { this.connectionManager.close(); } diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index 01657a722..44a181351 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -14,6 +14,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.TimeUnit; +import java.util.Arrays; import javax.management.RuntimeErrorException; @@ -34,6 +36,7 @@ import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpProxyOptions; import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpRequestBase; import software.amazon.awssdk.crt.http.HttpRequestBodyStream; import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpVersion; @@ -48,7 +51,7 @@ import software.amazon.awssdk.crt.Log; import software.amazon.awssdk.crt.Log.LogLevel; -public class HttpStreamManagerTest extends HttpClientTestFixture { +public class HttpStreamManagerTest extends HttpRequestResponseFixture { private final static String endpoint = "https://httpbin.org"; private final static String path = "/anything"; private final String EMPTY_BODY = ""; @@ -138,12 +141,67 @@ public long getLength() { return new HttpRequest(method, path, requestHeaders, payloadStream); } + private TestHttpResponse getResponseFromManager(HttpStreamManager streamManager, HttpRequestBase request) + throws Exception { + + final CompletableFuture reqCompleted = new CompletableFuture<>(); + + final TestHttpResponse response = new TestHttpResponse(); + + try { + HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { + @Override + public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + response.statusCode = responseStatusCode; + Assert.assertEquals(responseStatusCode, stream.getResponseStatusCode()); + response.headers.addAll(Arrays.asList(nextHeaders)); + } + + @Override + public void onResponseHeadersDone(HttpStream stream, int blockType) { + response.blockType = blockType; + } + + @Override + public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + response.bodyBuffer.put(bodyBytesIn); + int amountRead = bodyBytesIn.length; + + // Slide the window open by the number of bytes just read + return amountRead; + } + + @Override + public void onResponseComplete(HttpStream stream, int errorCode) { + response.onCompleteErrorCode = errorCode; + reqCompleted.complete(null); + stream.close(); + } + }; + streamManager.acquireStream(request, streamHandler).get(60, TimeUnit.SECONDS); + // Give the request up to 60 seconds to complete, otherwise throw a + // TimeoutException + reqCompleted.get(60, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException(e); + } + + CrtResource.waitForNoResources(); + + return response; + + } + @Test public void testSanitizerHTTP1() throws Exception { URI uri = new URI(endpoint); + CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { + shutdownComplete = streamManager.getShutdownCompleteFuture(); } + shutdownComplete.get(60, TimeUnit.SECONDS); CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } @@ -151,9 +209,12 @@ public void testSanitizerHTTP1() throws Exception { @Test public void testSanitizerHTTP2() throws Exception { URI uri = new URI(endpoint); + CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_2)) { + shutdownComplete = streamManager.getShutdownCompleteFuture(); } + shutdownComplete.get(60, TimeUnit.SECONDS); CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } @@ -161,24 +222,15 @@ public void testSanitizerHTTP2() throws Exception { @Test public void testSingleHTTP2Requests() throws Exception { URI uri = new URI(endpoint); + CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_2)) { + shutdownComplete = streamManager.getShutdownCompleteFuture(); Http2Request request = createHttp2Request("GET", endpoint, path, EMPTY_BODY); - - streamManager.acquireStream(request, new HttpStreamResponseHandler() { - @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, - HttpHeader[] nextHeaders) { - Assert.assertEquals(responseStatusCode, EXPECTED_HTTP_STATUS); - } - - @Override - public void onResponseComplete(HttpStream stream, int errorCode) { - Assert.assertEquals(errorCode, CRT.AWS_CRT_SUCCESS); - stream.close(); - } - }).get(); + TestHttpResponse response = this.getResponseFromManager(streamManager, request); + Assert.assertEquals(response.statusCode, EXPECTED_HTTP_STATUS); } + shutdownComplete.get(60, TimeUnit.SECONDS); CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } @@ -187,30 +239,17 @@ public void onResponseComplete(HttpStream stream, int errorCode) { public void testSingleHTTP1Request() throws Throwable { Log.initLoggingToStderr(LogLevel.Trace); URI uri = new URI(endpoint); + CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { - /* - * http2 request which will have :method headers that is not allowed for - * HTTP/1.1 - */ + shutdownComplete = streamManager.getShutdownCompleteFuture(); HttpRequest request = createHttp1Request("GET", endpoint, path, EMPTY_BODY); - - streamManager.acquireStream(request, new HttpStreamResponseHandler() { - @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, - HttpHeader[] nextHeaders) { - Assert.assertEquals(responseStatusCode, EXPECTED_HTTP_STATUS); - } - - @Override - public void onResponseComplete(HttpStream stream, int errorCode) { - Assert.assertEquals(errorCode, CRT.AWS_CRT_SUCCESS); - stream.close(); - } - }).get(); + TestHttpResponse response = this.getResponseFromManager(streamManager, request); + Assert.assertEquals(response.statusCode, EXPECTED_HTTP_STATUS); } catch (ExecutionException e) { Assert.assertNull(e); } + shutdownComplete.get(60, TimeUnit.SECONDS); CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } @@ -223,26 +262,16 @@ public void onResponseComplete(HttpStream stream, int errorCode) { @Test public void testSingleHTTP1RequestsFailure() throws Throwable { URI uri = new URI(endpoint); + CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { /* * http2 request which will have :method headers that is not allowed for * HTTP/1.1 */ Http2Request request = createHttp2Request("GET", endpoint, path, EMPTY_BODY); - - streamManager.acquireStream(request, new HttpStreamResponseHandler() { - @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, - HttpHeader[] nextHeaders) { - Assert.assertEquals(responseStatusCode, EXPECTED_HTTP_STATUS); - } - - @Override - public void onResponseComplete(HttpStream stream, int errorCode) { - Assert.assertEquals(errorCode, CRT.AWS_CRT_SUCCESS); - stream.close(); - } - }).get(); + shutdownComplete = streamManager.getShutdownCompleteFuture(); + TestHttpResponse response = this.getResponseFromManager(streamManager, request); + Assert.assertEquals(response.statusCode, EXPECTED_HTTP_STATUS); } catch (ExecutionException e) { try { throw e.getCause(); @@ -254,6 +283,7 @@ public void onResponseComplete(HttpStream stream, int errorCode) { } } + shutdownComplete.get(60, TimeUnit.SECONDS); CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } From 0a73ffc164fea77e5647a2628480c4b72984ea3d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 10:47:04 -0800 Subject: [PATCH 100/142] remove the check as the resource is created outside the function --- .../software/amazon/awssdk/crt/test/HttpStreamManagerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index 44a181351..67bf7481d 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -187,8 +187,6 @@ public void onResponseComplete(HttpStream stream, int errorCode) { throw new RuntimeException(e); } - CrtResource.waitForNoResources(); - return response; } From 6de6dc43e6acf0d66104442e9f89ca2fc8e9901b Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 10:59:00 -0800 Subject: [PATCH 101/142] remove log --- .../software/amazon/awssdk/crt/test/HttpStreamManagerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index 67bf7481d..5a20bdd9e 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -235,7 +235,6 @@ public void testSingleHTTP2Requests() throws Exception { @Test public void testSingleHTTP1Request() throws Throwable { - Log.initLoggingToStderr(LogLevel.Trace); URI uri = new URI(endpoint); CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { From 43fda92eb14a64ecf7a3eb0046d5c3162fb42b99 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 12:59:24 -0800 Subject: [PATCH 102/142] try to add assertion --- .../amazon/awssdk/crt/test/HttpStreamManagerTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index 5a20bdd9e..508b7daea 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -165,7 +165,11 @@ public void onResponseHeadersDone(HttpStream stream, int blockType) { @Override public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { - response.bodyBuffer.put(bodyBytesIn); + try { + response.bodyBuffer.put(bodyBytesIn); + } catch (Exception e) { + Assert.assertNull(e); + } int amountRead = bodyBytesIn.length; // Slide the window open by the number of bytes just read From 0165c1670e4a3e8af994dfba914fa78bdfefe6f1 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 13:40:27 -0800 Subject: [PATCH 103/142] throw the original exception --- .../software/amazon/awssdk/crt/test/HttpStreamManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index 508b7daea..02886e8ea 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -188,7 +188,7 @@ public void onResponseComplete(HttpStream stream, int errorCode) { // TimeoutException reqCompleted.get(60, TimeUnit.SECONDS); } catch (Exception e) { - throw new RuntimeException(e); + throw e; } return response; From 2b69707f9d2387952b4c70f3f78100a03c9546e9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 15:17:17 -0800 Subject: [PATCH 104/142] Add timeout and comments --- .../software/amazon/awssdk/crt/http/HttpStreamManager.java | 7 ++++++- .../amazon/awssdk/crt/test/HttpStreamManagerTest.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java index 89382bb0c..cef434676 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -8,6 +8,7 @@ import software.amazon.awssdk.crt.CrtRuntimeException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; /** * Manages a pool for either HTTP/1.1 or HTTP/2 connection. @@ -51,7 +52,8 @@ private HttpClientConnectionManagerOptions getConnManagerOptFromStreamManagerOpt private HttpStreamManager(HttpStreamManagerOptions options) { HttpClientConnectionManagerOptions connManagerOptions = getConnManagerOptFromStreamManagerOpt(options); this.connectionManager = HttpClientConnectionManager.create(connManagerOptions); - try (HttpClientConnection conn = this.connectionManager.acquireConnection().get()) { + /* Take a block call */ + try (HttpClientConnection conn = this.connectionManager.acquireConnection().get(30, TimeUnit.SECONDS)) { if (conn.getVersion() == HttpVersion.HTTP_2) { /* * Create Http2StreamManager and clean up connection manager later as we don't @@ -67,6 +69,9 @@ private HttpStreamManager(HttpStreamManagerOptions options) { this.shutdownComplete = this.connectionManager.getShutdownCompleteFuture(); } } catch (Exception ex) { + /** + * Not failing the initialization, but all the acquiring new streams will fail + */ this.initException = ex; } } diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index 02886e8ea..c2ada6388 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -201,6 +201,7 @@ public void testSanitizerHTTP1() throws Exception { CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_1_1)) { shutdownComplete = streamManager.getShutdownCompleteFuture(); + Assert.assertEquals(streamManager.getHttpVersion(), HttpVersion.HTTP_1_1); } shutdownComplete.get(60, TimeUnit.SECONDS); @@ -214,6 +215,7 @@ public void testSanitizerHTTP2() throws Exception { CompletableFuture shutdownComplete = null; try (HttpStreamManager streamManager = createStreamManager(uri, NUM_CONNECTIONS, HttpVersion.HTTP_2)) { shutdownComplete = streamManager.getShutdownCompleteFuture(); + Assert.assertEquals(streamManager.getHttpVersion(), HttpVersion.HTTP_2); } shutdownComplete.get(60, TimeUnit.SECONDS); @@ -246,8 +248,6 @@ public void testSingleHTTP1Request() throws Throwable { HttpRequest request = createHttp1Request("GET", endpoint, path, EMPTY_BODY); TestHttpResponse response = this.getResponseFromManager(streamManager, request); Assert.assertEquals(response.statusCode, EXPECTED_HTTP_STATUS); - } catch (ExecutionException e) { - Assert.assertNull(e); } shutdownComplete.get(60, TimeUnit.SECONDS); From 84e30a8b3dd00daaa417572f33dfae049c23c70e Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 21 Feb 2022 17:08:19 -0800 Subject: [PATCH 105/142] fix a previous bug --- .../amazon/awssdk/crt/test/Http2RequestResponseTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index 9af47e841..65592777e 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -183,6 +183,11 @@ public void testHttp2ResetStream() throws Exception { @Override public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { + } + + @Override + public void onResponseHeadersDone(HttpStream stream, int blockType) { + /* Only invoke once */ Http2Stream h2Stream = (Http2Stream) stream; h2Stream.resetStream(Http2ErrorCode.INTERNAL_ERROR); } From f1e9479f8f9790e0a421319220bba3e3c6813016 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 23 Feb 2022 10:36:05 -0800 Subject: [PATCH 106/142] make sure the exception throw out at the right place --- .../awssdk/crt/http/Http2StreamManager.java | 10 --- .../awssdk/crt/http/HttpStreamManager.java | 90 +++++++++++-------- 2 files changed, 54 insertions(+), 46 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index fe07bfffa..2f6f88f61 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -181,16 +181,6 @@ private Http2StreamManager(HttpStreamManagerOptions options) { * acquired. * @throws CrtRuntimeException */ - public CompletableFuture acquireStream(Http2Request request, - HttpStreamResponseHandler streamHandler) { - return this.acquireStream((HttpRequestBase) request, streamHandler); - } - - public CompletableFuture acquireStream(HttpRequest request, - HttpStreamResponseHandler streamHandler) { - return this.acquireStream((HttpRequestBase) request, streamHandler); - } - public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java index cef434676..0d9e599ce 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -9,6 +9,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * Manages a pool for either HTTP/1.1 or HTTP/2 connection. @@ -20,8 +21,9 @@ public class HttpStreamManager implements AutoCloseable { private HttpClientConnectionManager connectionManager = null; private Http2StreamManager h2StreamManager = null; - private Exception initException = null; private CompletableFuture shutdownComplete = null; + private AtomicLong shutdownNum = new AtomicLong(0); + private Throwable shutdownCompleteException = null; /** * Factory function for HttpStreamManager instances @@ -52,34 +54,59 @@ private HttpClientConnectionManagerOptions getConnManagerOptFromStreamManagerOpt private HttpStreamManager(HttpStreamManagerOptions options) { HttpClientConnectionManagerOptions connManagerOptions = getConnManagerOptFromStreamManagerOpt(options); this.connectionManager = HttpClientConnectionManager.create(connManagerOptions); - /* Take a block call */ - try (HttpClientConnection conn = this.connectionManager.acquireConnection().get(30, TimeUnit.SECONDS)) { - if (conn.getVersion() == HttpVersion.HTTP_2) { - /* - * Create Http2StreamManager and clean up connection manager later as we don't - * need it anymore - */ - this.h2StreamManager = Http2StreamManager.create(options); - this.connectionManager.releaseConnection(conn); - this.connectionManager.close(); - this.connectionManager = null; - this.shutdownComplete = this.h2StreamManager.getShutdownCompleteFuture(); - } else { - this.connectionManager.releaseConnection(conn); - this.shutdownComplete = this.connectionManager.getShutdownCompleteFuture(); + this.h2StreamManager = Http2StreamManager.create(options); + this.shutdownComplete = new CompletableFuture(); + this.connectionManager.getShutdownCompleteFuture().whenComplete((v, throwable) -> { + if (throwable != null) { + this.shutdownCompleteException = throwable; } - } catch (Exception ex) { - /** - * Not failing the initialization, but all the acquiring new streams will fail - */ - this.initException = ex; - } + long shutdownNum = this.shutdownNum.addAndGet(1); + if (shutdownNum == 2) { + /* both connectionManager and the h2StreamManager has been shutdown. */ + if (this.shutdownCompleteException != null) { + this.shutdownComplete.completeExceptionally(this.shutdownCompleteException); + } else { + this.shutdownComplete.complete(null); + } + } + }); + this.h2StreamManager.getShutdownCompleteFuture().whenComplete((v, throwable) -> { + if (throwable != null) { + this.shutdownCompleteException = throwable; + } + long shutdownNum = this.shutdownNum.addAndGet(1); + if (shutdownNum == 2) { + /* both connectionManager and the h2StreamManager has been shutdown. */ + if (this.shutdownCompleteException != null) { + this.shutdownComplete.completeExceptionally(this.shutdownCompleteException); + } else { + this.shutdownComplete.complete(null); + } + } + }); } /** * Get the protocol version the manager runs on. */ - public HttpVersion getHttpVersion() { + public HttpVersion getHttpVersion() throws Exception { + if (this.h2StreamManager != null && this.connectionManager != null) { + try (HttpClientConnection conn = this.connectionManager.acquireConnection().get(30, TimeUnit.SECONDS)) { + if (conn.getVersion() == HttpVersion.HTTP_2) { + /* + * Create Http2StreamManager and clean up connection manager later as we don't + * need it anymore + */ + this.connectionManager.releaseConnection(conn); + this.connectionManager.close(); + this.connectionManager = null; + } else { + this.connectionManager.releaseConnection(conn); + this.h2StreamManager.close(); + this.h2StreamManager = null; + } + } + } if (this.h2StreamManager != null) { return HttpVersion.HTTP_2; } @@ -96,22 +123,13 @@ public HttpVersion getHttpVersion() { * acquired. * @throws CrtRuntimeException */ - public CompletableFuture acquireStream(Http2Request request, - HttpStreamResponseHandler streamHandler) { - return this.acquireStream((HttpRequestBase) request, streamHandler); - } - - public CompletableFuture acquireStream(HttpRequest request, - HttpStreamResponseHandler streamHandler) { - return this.acquireStream((HttpRequestBase) request, streamHandler); - } - public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); - - if (this.initException != null) { - completionFuture.completeExceptionally(this.initException); + try { + this.getHttpVersion(); + } catch (Exception e) { + completionFuture.completeExceptionally(e); return completionFuture; } if (this.h2StreamManager != null) { From 3804371aba421234b37398a3b6552358193fb57f Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 23 Feb 2022 10:44:23 -0800 Subject: [PATCH 107/142] update the comments --- .../amazon/awssdk/crt/http/HttpStreamManager.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java index 0d9e599ce..498980751 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -93,14 +93,16 @@ public HttpVersion getHttpVersion() throws Exception { if (this.h2StreamManager != null && this.connectionManager != null) { try (HttpClientConnection conn = this.connectionManager.acquireConnection().get(30, TimeUnit.SECONDS)) { if (conn.getVersion() == HttpVersion.HTTP_2) { - /* - * Create Http2StreamManager and clean up connection manager later as we don't - * need it anymore + /** + * The connection is HTTP/2, close the connectionManager, which made for HTTP/1 */ this.connectionManager.releaseConnection(conn); this.connectionManager.close(); this.connectionManager = null; } else { + /** + * The connection is HTTP/1, close the h2StreamManager, which made for HTTP/2 + */ this.connectionManager.releaseConnection(conn); this.h2StreamManager.close(); this.h2StreamManager = null; @@ -127,6 +129,10 @@ public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); try { + /* + * Try get version first. If we haven't decided the version yet, this will help + * us to decide the version we are running on + */ this.getHttpVersion(); } catch (Exception e) { completionFuture.completeExceptionally(e); From 01f3475c578b7f6acc23424ba38a1a6332c6aa37 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 May 2022 15:44:11 -0700 Subject: [PATCH 108/142] fix a bunch of merge conflict --- .../crt/http/Http2ConnectionSettings.java | 52 ------------------- .../awssdk/crt/http/Http2StreamManager.java | 16 ++---- .../crt/http/HttpStreamManagerOptions.java | 23 ++++---- src/native/http2_stream_manager.c | 32 ++++++++---- src/native/http_request_response.c | 41 ++------------- src/native/http_request_response.h | 2 +- .../crt/test/Http2StreamManagerTest.java | 10 ++-- 7 files changed, 50 insertions(+), 126 deletions(-) delete mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java b/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java deleted file mode 100644 index cfaf2a544..000000000 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2ConnectionSettings.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -package software.amazon.awssdk.crt.http; - -import java.util.List; -import java.util.ArrayList; - -public class Http2ConnectionSettings { - - private List settings; - - public Http2ConnectionSettings(List settings) { - this.settings = new ArrayList(settings); - } - - public Http2ConnectionSettings() { - this.settings = new ArrayList(); - } - - public void addSetting(Http2ConnectionSetting setting) throws Exception { - settings.add(setting); - } - - public void enablePush(boolean push) throws Exception { - settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.ENABLE_PUSH, push ? 1 : 0)); - } - - public void headerTableSize(long headerTableSize) throws Exception { - settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.HEADER_TABLE_SIZE, headerTableSize)); - } - - public void maxConcurrentStreams(long maxConcurrentStreams) throws Exception { - settings.add( - new Http2ConnectionSetting(Http2ConnectionSetting.ID.MAX_CONCURRENT_STREAMS, maxConcurrentStreams)); - } - - public void initialWindowSize(long initialWindowSize) throws Exception { - settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.INITIAL_WINDOW_SIZE, initialWindowSize)); - } - - public void maxFrameSize(long maxFrameSize) throws Exception { - settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.MAX_FRAME_SIZE, maxFrameSize)); - } - - public void maxHeaderListSize(long maxHeaderListSize) throws Exception { - settings.add(new Http2ConnectionSetting(Http2ConnectionSetting.ID.MAX_HEADER_LIST_SIZE, maxHeaderListSize)); - } - -} diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index ba9f8486d..cbb76242f 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -23,7 +23,6 @@ public class Http2StreamManager extends CrtResource { private static final int DEFAULT_HTTPS_PORT = 443; private final static Charset UTF8 = java.nio.charset.StandardCharsets.UTF_8; - private final int windowSize; private final URI uri; private final int port; private final int maxConnections; @@ -72,10 +71,6 @@ private Http2StreamManager(HttpStreamManagerOptions options) { throw new IllegalArgumentException("TlsContext must not be null if https is used"); } - int windowSize = options.getWindowSize(); - if (windowSize <= 0) { - throw new IllegalArgumentException("Window Size must be greater than zero."); - } int maxConnections = options.getMaxConnections(); if (maxConnections <= 0) { @@ -108,7 +103,6 @@ private Http2StreamManager(HttpStreamManagerOptions options) { HttpProxyOptions proxyOptions = options.getProxyOptions(); - this.windowSize = windowSize; this.uri = uri; this.port = port; this.maxConnections = maxConnections; @@ -145,7 +139,7 @@ private Http2StreamManager(HttpStreamManagerOptions options) { clientBootstrap.getNativeHandle(), socketOptions.getNativeHandle(), useTls ? tlsContext.getNativeHandle() : 0, - windowSize, + Http2ConnectionSetting.marshallSettingsForJNI(options.getInitialSettingsList()), uri.getHost().getBytes(UTF8), port, proxyConnectionType, @@ -182,17 +176,17 @@ private Http2StreamManager(HttpStreamManagerOptions options) { * @throws CrtRuntimeException */ public CompletableFuture acquireStream(Http2Request request, - HttpStreamResponseHandler streamHandler) { + HttpStreamBaseResponseHandler streamHandler) { return this.acquireStream((HttpRequestBase) request, streamHandler); } public CompletableFuture acquireStream(HttpRequest request, - HttpStreamResponseHandler streamHandler) { + HttpStreamBaseResponseHandler streamHandler) { return this.acquireStream((HttpRequestBase) request, streamHandler); } private CompletableFuture acquireStream(HttpRequestBase request, - HttpStreamResponseHandler streamHandler) { + HttpStreamBaseResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); AsyncCallback acquireStreamCompleted = AsyncCallback.wrapFuture(completionFuture, null); if (isNull()) { @@ -261,7 +255,7 @@ private static native long http2StreamManagerNew(Http2StreamManager thisObj, long client_bootstrap, long socketOptions, long tlsContext, - int windowSize, + long[] marshalledSettings, byte[] endpoint, int port, int proxyConnectionType, diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java index be4477d33..18530da12 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java @@ -1,6 +1,8 @@ package software.amazon.awssdk.crt.http; import java.net.URI; +import java.util.List; +import java.util.ArrayList; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.SocketOptions; import software.amazon.awssdk.crt.io.TlsContext; @@ -20,7 +22,6 @@ public class HttpStreamManagerOptions { private URI uri; private int port = -1; private boolean manualWindowManagement = false; - private int windowSize = DEFAULT_MAX_WINDOW_SIZE; private HttpMonitoringOptions monitoringOptions; private HttpProxyOptions proxyOptions; @@ -29,6 +30,8 @@ public class HttpStreamManagerOptions { private int maxConnections = DEFAULT_MAX_CONNECTIONS; + private List initialSettingsList = new ArrayList(); + /** * Default constructor */ @@ -93,22 +96,24 @@ public TlsContext getTlsContext() { } /** - * Sets the IO channel window size to use for connections in the connection pool + * For HTTP/2 stream manager only. + * + * The initial settings for the HTTP/2 connections made by stream manger. + * `Http2ConnectionSettingListBuilder` can help to build the settings list * - * @param windowSize The initial window size to use for each connection + * @param initialSettingsList The List of initial settings * @return this */ - public HttpStreamManagerOptions withWindowSize(int windowSize) { - this.windowSize = windowSize; + public HttpStreamManagerOptions withInitialSettingsList(List initialSettingsList) { + this.initialSettingsList.addAll(initialSettingsList); return this; } /** - * @return the IO channel window size to use for connections in the connection - * pool + * @return The List of initial settings */ - public int getWindowSize() { - return windowSize; + public List getInitialSettingsList() { + return this.initialSettingsList; } /** diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c index 596ab3e52..38443480f 100644 --- a/src/native/http2_stream_manager.c +++ b/src/native/http2_stream_manager.c @@ -85,7 +85,7 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ jlong jni_client_bootstrap, jlong jni_socket_options, jlong jni_tls_ctx, - jint jni_window_size, + jlongArray java_marshalled_settings, jbyteArray jni_endpoint, jint jni_port, jint jni_proxy_connection_type, @@ -108,6 +108,7 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ struct aws_socket_options *socket_options = (struct aws_socket_options *)jni_socket_options; struct aws_tls_ctx *tls_ctx = (struct aws_tls_ctx *)jni_tls_ctx; struct aws_http2_stream_manager_binding *binding = NULL; + struct aws_allocator *allocator = aws_jni_get_allocator(); if (!client_bootstrap) { aws_jni_throw_runtime_exception(env, "ClientBootstrap can't be null"); @@ -119,7 +120,21 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ return (jlong)NULL; } - struct aws_allocator *allocator = aws_jni_get_allocator(); + const size_t marshalled_len = (*env)->GetArrayLength(env, java_marshalled_settings); + AWS_ASSERT(marshalled_len % 2 == 0); + + size_t num_initial_settings = marshalled_len / 2; + struct aws_http2_setting *initial_settings = + num_initial_settings ? aws_mem_calloc(allocator, num_initial_settings, sizeof(struct aws_http2_setting)) : NULL; + + jlong *marshalled_settings = (*env)->GetLongArrayElements(env, java_marshalled_settings, NULL); + for (size_t i = 0; i < num_initial_settings; i++) { + jlong id = marshalled_settings[i * 2]; + initial_settings[i].id = id; + jlong value = marshalled_settings[i * 2 + 1]; + initial_settings[i].value = (uint32_t)value; + } + struct aws_byte_cursor endpoint = aws_jni_byte_cursor_from_jbyteArray_acquire(env, jni_endpoint); if (jni_port <= 0 || 65535 < jni_port) { @@ -127,11 +142,6 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ goto cleanup; } - if (jni_window_size <= 0) { - aws_jni_throw_runtime_exception(env, "Window Size must be > 0"); - goto cleanup; - } - if (jni_max_conns <= 0) { aws_jni_throw_runtime_exception(env, "Max Connections must be > 0"); goto cleanup; @@ -161,7 +171,9 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ AWS_ZERO_STRUCT(manager_options); manager_options.bootstrap = client_bootstrap; - manager_options.initial_window_size = (size_t)jni_window_size; + manager_options.initial_settings_array = initial_settings; + manager_options.num_initial_settings = num_initial_settings; + manager_options.socket_options = socket_options; manager_options.tls_connection_options = NULL; manager_options.host = endpoint; @@ -294,12 +306,12 @@ static void s_on_stream_acquired(struct aws_http_stream *stream, int error_code, } else { /* Stream is activated once we acquired from the Stream Manager */ aws_atomic_store_int(&callback_data->stream_binding->activated, 1); - callback_data->stream_binding->java_http_stream = (*env)->NewGlobalRef(env, j_http_stream); + callback_data->stream_binding->java_http_stream_base = (*env)->NewGlobalRef(env, j_http_stream); (*env)->CallVoidMethod( env, callback_data->java_async_callback, async_callback_properties.on_success_with_object, - callback_data->stream_binding->java_http_stream); + callback_data->stream_binding->java_http_stream_base); (*env)->DeleteLocalRef(env, j_http_stream); } } diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 2a0e77b4d..3aa286ee2 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -75,7 +75,7 @@ void aws_http_stream_binding_destroy(JNIEnv *env, struct http_stream_binding *ca } if (callback->java_http_stream_base) { - s_java_http_stream_from_native_delete(env, callback->java_http_stream_base); + aws_java_http_stream_from_native_delete(env, callback->java_http_stream_base); } if (callback->java_http_response_stream_handler != NULL) { @@ -83,7 +83,6 @@ void aws_http_stream_binding_destroy(JNIEnv *env, struct http_stream_binding *ca } if (callback->native_request) { - struct aws_input_stream *input_stream = aws_http_message_get_body_stream(callback->native_request); aws_http_message_destroy(callback->native_request); } aws_byte_buf_clean_up(&callback->headers_buf); @@ -343,7 +342,7 @@ static jobject s_make_request_general( (void *)native_conn, (void *)callback_data->native_stream); - jHttpStreamBase = s_java_http_stream_from_native_new(env, callback_data, version); + jHttpStreamBase = aws_java_http_stream_from_native_new(env, callback_data, version); } /* Check for errors that might have occurred while holding the lock. */ @@ -400,40 +399,6 @@ JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnec AWS_HTTP_VERSION_2); } -JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_HttpClientConnection_httpClientConnectionMakeRequest( - JNIEnv *env, - jclass jni_class, - jlong jni_connection, - jbyteArray marshalled_request, - jobject jni_http_request_body_stream, - jobject jni_http_response_callback_handler) { - (void)jni_class; - return s_make_request_general( - env, - jni_connection, - marshalled_request, - jni_http_request_body_stream, - jni_http_response_callback_handler, - AWS_HTTP_VERSION_1_1); -} - -JNIEXPORT jobject JNICALL Java_software_amazon_awssdk_crt_http_Http2ClientConnection_http2ClientConnectionMakeRequest( - JNIEnv *env, - jclass jni_class, - jlong jni_connection, - jbyteArray marshalled_request, - jobject jni_http_request_body_stream, - jobject jni_http_response_callback_handler) { - (void)jni_class; - return s_make_request_general( - env, - jni_connection, - marshalled_request, - jni_http_request_body_stream, - jni_http_response_callback_handler, - AWS_HTTP_VERSION_2); -} - struct http_stream_chunked_callback_data { struct http_stream_binding *stream_cb_data; struct aws_byte_buf chunk_data; @@ -633,7 +598,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2Stream_http2Str (void)jni_class; - struct http_stream_callback_data *cb_data = (struct http_stream_callback_data *)jni_cb_data; + struct http_stream_binding *cb_data = (struct http_stream_binding *)jni_cb_data; struct aws_http_stream *stream = cb_data->native_stream; if (stream == NULL) { diff --git a/src/native/http_request_response.h b/src/native/http_request_response.h index d707b2b73..528c14148 100644 --- a/src/native/http_request_response.h +++ b/src/native/http_request_response.h @@ -22,7 +22,7 @@ struct http_stream_binding { struct aws_http_message *native_request; jobject java_http_response_stream_handler; - jobject java_http_stream; + jobject java_http_stream_base; struct aws_http_stream *native_stream; struct aws_byte_buf headers_buf; int response_status; diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java index ae8050aaa..d7173dcc4 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java @@ -26,8 +26,8 @@ import software.amazon.awssdk.crt.http.HttpHeader; import software.amazon.awssdk.crt.http.HttpProxyOptions; import software.amazon.awssdk.crt.http.HttpRequest; -import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; -import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.http.HttpStreamBaseResponseHandler; +import software.amazon.awssdk.crt.http.HttpStreamBase; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.EventLoopGroup; import software.amazon.awssdk.crt.io.HostResolver; @@ -101,15 +101,15 @@ private void testParallelStreams(Http2StreamManager streamManager, Http2Request threadPool.execute(() -> { // Request a connection from the connection pool int requestId = numRequestsMade.incrementAndGet(); - streamManager.acquireStream(request, new HttpStreamResponseHandler() { + streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { reqIdToStatus.put(requestId, responseStatusCode); } @Override - public void onResponseComplete(HttpStream stream, int errorCode) { + public void onResponseComplete(HttpStreamBase stream, int errorCode) { if (errorCode != CRT.AWS_CRT_SUCCESS || stream.getResponseStatusCode() != EXPECTED_HTTP_STATUS) { Log.log(Log.LogLevel.Error, Log.LogSubject.HttpConnectionManager, From 57794d956f392c86fb62e678154792a647ed6dfc Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 May 2022 16:06:00 -0700 Subject: [PATCH 109/142] input stream refcount stuff --- .../software/amazon/awssdk/crt/http/Http2StreamManager.java | 6 +++--- .../amazon/awssdk/crt/http/HttpStreamManagerOptions.java | 6 +++--- src/native/aws_signing.c | 4 ---- src/native/http_request_response.c | 2 +- src/native/http_request_utils.c | 4 ++++ src/native/s3_client.c | 2 -- .../amazon/awssdk/crt/test/CredentialsProviderTest.java | 2 +- 7 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index cbb76242f..ed184e49a 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -169,11 +169,11 @@ private Http2StreamManager(HttpStreamManagerOptions options) { /** * Request a Http2Stream from StreamManager. * - * @param request - * @param streamHandler + * @param request The Request to make to the Server. + * @param streamHandler The Stream Handler to be called from the Native + * EventLoop * @return A future for a Http2Stream that will be completed when the stream is * acquired. - * @throws CrtRuntimeException */ public CompletableFuture acquireStream(Http2Request request, HttpStreamBaseResponseHandler streamHandler) { diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java index 18530da12..879e12afb 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java @@ -149,7 +149,7 @@ public int getIdealConcurrentStreamsPerConnection() { * * @param maxConcurrentStreamsPerConnection The max number of concurrent * streams for a connection - * @return + * @return this */ public HttpStreamManagerOptions withMaxConcurrentStreamsPerConnection(int maxConcurrentStreamsPerConnection) { this.maxConcurrentStreamsPerConnection = maxConcurrentStreamsPerConnection; @@ -208,8 +208,8 @@ public int getPort() { * connections are full, manager will wait until available to vender more * streams * - * @param maxConnections - * @return + * @param maxConnections The max number of connections will be open at same time. + * @return this */ public HttpStreamManagerOptions withMaxConnections(int maxConnections) { this.maxConnections = maxConnections; diff --git a/src/native/aws_signing.c b/src/native/aws_signing.c index ae5ad1d13..f49dac3a2 100644 --- a/src/native/aws_signing.c +++ b/src/native/aws_signing.c @@ -74,10 +74,6 @@ static void s_cleanup_callback_data(struct s_aws_sign_request_callback_data *cal } if (callback_data->native_request) { - struct aws_input_stream *input_stream = aws_http_message_get_body_stream(callback_data->native_request); - if (input_stream != NULL) { - aws_input_stream_destroy(input_stream); - } aws_http_message_release(callback_data->native_request); } diff --git a/src/native/http_request_response.c b/src/native/http_request_response.c index 3aa286ee2..f2956a78c 100644 --- a/src/native/http_request_response.c +++ b/src/native/http_request_response.c @@ -83,7 +83,7 @@ void aws_http_stream_binding_destroy(JNIEnv *env, struct http_stream_binding *ca } if (callback->native_request) { - aws_http_message_destroy(callback->native_request); + aws_http_message_release(callback->native_request); } aws_byte_buf_clean_up(&callback->headers_buf); aws_mem_release(aws_jni_get_allocator(), callback); diff --git a/src/native/http_request_utils.c b/src/native/http_request_utils.c index 8a1ed1eae..9a09b969b 100644 --- a/src/native/http_request_utils.c +++ b/src/native/http_request_utils.c @@ -365,6 +365,8 @@ int aws_apply_java_http_request_changes_to_native_request( aws_input_stream_new_from_java_http_request_body_stream(aws_jni_get_allocator(), env, jni_body_stream); aws_http_message_set_body_stream(message, body_stream); + /* request controls the lifetime of body stream fully */ + aws_input_stream_release(body_stream); } return result; @@ -407,6 +409,8 @@ struct aws_http_message *aws_http_request_new_from_java_http_request( } aws_http_message_set_body_stream(request, body_stream); + /* request controls the lifetime of body stream fully */ + aws_input_stream_release(body_stream); } return request; diff --git a/src/native/s3_client.c b/src/native/s3_client.c index 24de39803..5e871a15d 100644 --- a/src/native/s3_client.c +++ b/src/native/s3_client.c @@ -396,7 +396,6 @@ static void s_s3_meta_request_callback_cleanup( JNIEnv *env, struct s3_client_make_meta_request_callback_data *callback_data) { if (callback_data) { - aws_input_stream_destroy(callback_data->input_stream); (*env)->DeleteGlobalRef(env, callback_data->java_s3_meta_request); (*env)->DeleteGlobalRef(env, callback_data->java_s3_meta_request_response_handler_native_adapter); aws_mem_release(aws_jni_get_allocator(), callback_data); @@ -450,7 +449,6 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_s3_S3Client_s3ClientMake AWS_FATAL_ASSERT( AWS_OP_SUCCESS == aws_apply_java_http_request_changes_to_native_request( env, jni_marshalled_message_data, jni_http_request_body_stream, request_message)); - callback_data->input_stream = aws_http_message_get_body_stream(request_message); struct aws_uri endpoint; AWS_ZERO_STRUCT(endpoint); diff --git a/src/test/java/software/amazon/awssdk/crt/test/CredentialsProviderTest.java b/src/test/java/software/amazon/awssdk/crt/test/CredentialsProviderTest.java index ffa54979f..15706991c 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/CredentialsProviderTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/CredentialsProviderTest.java @@ -182,7 +182,7 @@ public void testDelegateException() { DelegateCredentialsHandler credentialsHandler = new DelegateCredentialsHandler() { @Override public Credentials getCredentials() { - throw new RuntimeException("hate coding"); + throw new RuntimeException("Some exception. =)"); } }; boolean failed = false; From 838066793459fa96a584b182d7a79fcdfdb22ee2 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 May 2022 16:18:59 -0700 Subject: [PATCH 110/142] trivial update --- src/main/java/software/amazon/awssdk/crt/http/HttpStream.java | 3 --- src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java | 2 -- 2 files changed, 5 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java index 6a4b1a352..00e301375 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStream.java @@ -24,9 +24,6 @@ protected HttpStream(long ptr) { super(ptr); } - /******************************************************************************* - * Shared method - ******************************************************************************/ /** * Completion interface for writing chunks to an http stream */ diff --git a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java index 9897ccd5f..8308f2bbe 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Elasticurl.java @@ -31,8 +31,6 @@ import software.amazon.awssdk.crt.http.HttpRequest; import software.amazon.awssdk.crt.http.HttpRequestBase; import software.amazon.awssdk.crt.http.HttpRequestBodyStream; -import software.amazon.awssdk.crt.http.HttpStream; -import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpStreamBase; import software.amazon.awssdk.crt.http.HttpStreamBaseResponseHandler; import software.amazon.awssdk.crt.io.ClientBootstrap; From d760f08193045637e5ea134df30de26701e88ae7 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 May 2022 17:07:54 -0700 Subject: [PATCH 111/142] try localhost ci --- .github/workflows/ci.yml | 49 +++++++++++++++++++ .../awssdk/crt/http/Http2StreamManager.java | 3 +- src/native/http_request_response.h | 2 +- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea07dc493..347c7a16f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,3 +171,52 @@ jobs: # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work # https://github.com/actions/runner/issues/480 uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main + + + localhost-test-linux: + runs-on: ubuntu-20.04 # latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure local host + run: | + python3 -m pip install h2 + cd crt/aws-c-http/tests/py_localhost/ + python3 server.py & + python3 non_tls_server.py & + - name: Build and test + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream + + localhost-test-mac: + runs-on: macos-11 # latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure local host + run: | + python3 -m pip install h2 + cd crt/aws-c-http/tests/py_localhost/ + python3 server.py & + python3 non_tls_server.py & + - name: Build and test + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream + + localhost-test-win: + runs-on: windows-2022 # latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure local host + run: | + python -m pip install h2 + - name: Build and test + run: | + cd ./tests/py_localhost/ + Start-Process -NoNewWindow python .\server.py + Start-Process -NoNewWindow python .\non_tls_server.py + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index ed184e49a..71c7c6839 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -71,15 +71,16 @@ private Http2StreamManager(HttpStreamManagerOptions options) { throw new IllegalArgumentException("TlsContext must not be null if https is used"); } - int maxConnections = options.getMaxConnections(); if (maxConnections <= 0) { throw new IllegalArgumentException("Max Connections must be greater than zero."); } + int maxConcurrentStreamsPerConnection = options.getMaxConcurrentStreamsPerConnection(); if (maxConcurrentStreamsPerConnection <= 0) { throw new IllegalArgumentException("Max Concurrent Streams Per Connection must be greater than zero."); } + int idealConcurrentStreamsPerConnection = options.getIdealConcurrentStreamsPerConnection(); if (idealConcurrentStreamsPerConnection <= 0 || idealConcurrentStreamsPerConnection > maxConcurrentStreamsPerConnection) { diff --git a/src/native/http_request_response.h b/src/native/http_request_response.h index 528c14148..057200442 100644 --- a/src/native/http_request_response.h +++ b/src/native/http_request_response.h @@ -28,7 +28,7 @@ struct http_stream_binding { int response_status; /* - * Unactivated streams must have their callback data destroyed at release time + * Inactivated streams must have their callback data destroyed at release time */ struct aws_atomic_var activated; }; From 7878f616a4ce30d61459bb6166a0318c5f893e74 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 May 2022 18:22:56 -0700 Subject: [PATCH 112/142] checkout with submodules --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 347c7a16f..6e24c0d9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,6 +178,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: true - name: Configure local host run: | python3 -m pip install h2 @@ -194,6 +196,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: true - name: Configure local host run: | python3 -m pip install h2 @@ -210,6 +214,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: true - name: Configure local host run: | python -m pip install h2 From 26ed971786d932078328b1f64bfbb181d945144e Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 May 2022 10:46:32 -0700 Subject: [PATCH 113/142] what's the file? --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e24c0d9f..7d5658f67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,6 +183,8 @@ jobs: - name: Configure local host run: | python3 -m pip install h2 + ls + ls crt cd crt/aws-c-http/tests/py_localhost/ python3 server.py & python3 non_tls_server.py & From 35a23148f7d489d410c128c5402350cc62872962 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 May 2022 10:51:28 -0700 Subject: [PATCH 114/142] the submodule is not updated --- .github/workflows/ci.yml | 3 +-- crt/aws-c-event-stream | 2 +- crt/aws-c-http | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d5658f67..2948e4c0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,8 +183,7 @@ jobs: - name: Configure local host run: | python3 -m pip install h2 - ls - ls crt + ls crt/aws-c-http/ cd crt/aws-c-http/tests/py_localhost/ python3 server.py & python3 non_tls_server.py & diff --git a/crt/aws-c-event-stream b/crt/aws-c-event-stream index e87537be5..9141f54ca 160000 --- a/crt/aws-c-event-stream +++ b/crt/aws-c-event-stream @@ -1 +1 @@ -Subproject commit e87537be561d753ec82e783bc0929b1979c585f8 +Subproject commit 9141f54caa20376cce5633e785ffd628df569429 diff --git a/crt/aws-c-http b/crt/aws-c-http index b76ebf220..73af2aa20 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit b76ebf2207c989e1d8e1c5e1a9ab1b0c32de3483 +Subproject commit 73af2aa20d220dfbb26f22ef15df9651583cd5cb From 516acca6bd5abefb1ad463d2b456ca10bae4fbcb Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 May 2022 10:56:52 -0700 Subject: [PATCH 115/142] update more --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2948e4c0b..c2bdcb2cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,7 +207,7 @@ jobs: python3 non_tls_server.py & - name: Build and test run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream localhost-test-win: @@ -222,7 +222,7 @@ jobs: python -m pip install h2 - name: Build and test run: | - cd ./tests/py_localhost/ + cd crt/aws-c-http/tests/py_localhost/ Start-Process -NoNewWindow python .\server.py Start-Process -NoNewWindow python .\non_tls_server.py python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" From afedcd9bad3fed29e51a9ddbabe24b4bea28174a Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 May 2022 11:23:07 -0700 Subject: [PATCH 116/142] fix osx --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2bdcb2cf..f7f3cbc54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,7 +208,8 @@ jobs: - name: Build and test run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream localhost-test-win: runs-on: windows-2022 # latest From bdce86c779c42b3e8044f98b79c67b374ba92200 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 May 2022 13:54:47 -0700 Subject: [PATCH 117/142] try just use mvn? --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7f3cbc54..fff615585 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,8 +189,7 @@ jobs: python3 non_tls_server.py & - name: Build and test run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream + mvn install localhost-test-mac: runs-on: macos-11 # latest From 66d07cdccb5f70bf57bb0db8dadc7de0914d13be Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 May 2022 14:18:05 -0700 Subject: [PATCH 118/142] try to use action --- .builder/actions/localhost_test.py | 12 ++++++++++++ .github/workflows/ci.yml | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 .builder/actions/localhost_test.py diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py new file mode 100644 index 000000000..82cc27937 --- /dev/null +++ b/.builder/actions/localhost_test.py @@ -0,0 +1,12 @@ +import Builder +import sys +import os + + +class LocalhostTest(Builder.Action): + + def run(self, env): + # tests must run with leak detection turned on + actions = ["echo thisShouldBeLocalTest"] + + return Builder.Script(actions, name='localhost-test') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fff615585..eb5feeac7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,8 @@ jobs: python3 non_tls_server.py & - name: Build and test run: | - mvn install + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} --spec=downstream localhost-test-mac: runs-on: macos-11 # latest @@ -208,7 +209,7 @@ jobs: run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream + ./builder localhost-test -p ${{ env.PACKAGE_NAME }} --spec=downstream localhost-test-win: runs-on: windows-2022 # latest @@ -226,4 +227,4 @@ jobs: Start-Process -NoNewWindow python .\server.py Start-Process -NoNewWindow python .\non_tls_server.py python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream + python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} downstream From 8f6a2ac635fc7cb42ff8e131b9a1ef3122cb3c09 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 20 May 2022 15:36:45 -0700 Subject: [PATCH 119/142] add localhost test --- .../awssdk/crt/test/CrtTestFixture.java | 4 + .../crt/test/Http2ClientLocalHostTest.java | 174 ++++++++++++++++++ .../crt/test/Http2StreamManagerTest.java | 6 +- 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java diff --git a/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java index 3dc46572b..125a682a0 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java @@ -109,4 +109,8 @@ protected boolean hasAwsCredentials() { protected void skipIfNetworkUnavailable() { Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); } + + protected void skipIfLocalhostUnavailable() { + Assume.assumeTrue(System.getProperty("aws.crt.localhost") != null); + } } diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java new file mode 100644 index 000000000..40b84bf75 --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -0,0 +1,174 @@ + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package software.amazon.awssdk.crt.test; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.Http2StreamManager; +import software.amazon.awssdk.crt.http.Http2Request; +import software.amazon.awssdk.crt.http.Http2Stream; +import software.amazon.awssdk.crt.http.HttpStreamManagerOptions; +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpProxyOptions; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpStreamBaseResponseHandler; +import software.amazon.awssdk.crt.http.HttpStreamBase; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.io.TlsContextOptions; +import software.amazon.awssdk.crt.utils.ByteBufferUtils; +import software.amazon.awssdk.crt.Log; + +public class Http2ClientLocalHostTest extends HttpClientTestFixture { + + private Http2StreamManager createStreamManager(URI uri, int numConnections) { + + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContextOptions tlsOpts = TlsContextOptions.createDefaultClient().withAlpnList("h2").withVerifyPeer(false); + TlsContext tlsContext = createHttpClientTlsContext(tlsOpts)) { + HttpStreamManagerOptions options = new HttpStreamManagerOptions(); + options.withClientBootstrap(bootstrap) + .withSocketOptions(sockOpts) + .withTlsContext(tlsContext) + .withUri(uri) + .withMaxConnections(numConnections); + + return Http2StreamManager.create(options); + } + } + + private HttpRequestBodyStream createBodyStreamWithLength(long bodyLength) { + final long payloadSize = bodyLength; + final String payloadString = "This is CRT HTTP test."; + + HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() { + + private long remainingBody = payloadSize; + + @Override + public boolean sendRequestBody(ByteBuffer outBuffer) { + + byte[] payloadBytes = null; + + try { + payloadBytes = payloadString.getBytes("ASCII"); + } catch (Exception ex) { + System.out.println("Encountered error trying to get payload bytes."); + return true; + } + + while (remainingBody > 0 && outBuffer.remaining() > 0) { + long amtToTransfer = Math.min(remainingBody, (long) outBuffer.remaining()); + amtToTransfer = Math.min(amtToTransfer, (long) payloadBytes.length); + + // Transfer the data + outBuffer.put(payloadBytes, 0, (int) amtToTransfer); + + remainingBody -= amtToTransfer; + } + + return remainingBody == 0; + } + + @Override + public boolean resetPosition() { + return true; + } + + @Override + public long getLength() { + return payloadSize; + } + }; + return payloadStream; + } + + private Http2Request createHttp2Request(String method, URI uri, long bodyLength) { + HttpHeader[] requestHeaders = new HttpHeader[] { + new HttpHeader(":method", method), + new HttpHeader(":path", uri.getPath()), + new HttpHeader(":scheme", uri.getScheme()), + new HttpHeader(":authority", uri.getHost()), + new HttpHeader("content-length", Long.toString(bodyLength)) + }; + HttpRequestBodyStream bodyStream = null; + if(bodyLength > 0) { + bodyStream = createBodyStreamWithLength(bodyLength); + } + Http2Request request = new Http2Request(requestHeaders, bodyStream); + + return request; + } + + @Test + public void testParallelRequestsStress() throws Exception { + skipIfLocalhostUnavailable(); + Log.initLoggingToStderr(Log.LogLevel.Trace); + URI uri = new URI("https://localhost:8443/echo"); + try( + Http2StreamManager streamManager = createStreamManager(uri, 100)) { + int numberToAcquire = 500 * 100; + + Http2Request request = createHttp2Request("GET", uri, 0); + List> requestCompleteFutures = new ArrayList<>(); + List> acquireCompleteFutures = new ArrayList<>(); + final AtomicInteger numStreamsFailures = new AtomicInteger(0); + for (int i = 0; i < numberToAcquire; i++) { + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + requestCompleteFutures.add(requestCompleteFuture); + acquireCompleteFutures.add(streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + if (responseStatusCode != 200) { + numStreamsFailures.incrementAndGet(); + } + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + if (errorCode != CRT.AWS_CRT_SUCCESS) { + numStreamsFailures.incrementAndGet(); + } + stream.close(); + requestCompleteFuture.complete(null); + } + })); + } + for (CompletableFuture f : acquireCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + // Wait for all Requests to complete + for (CompletableFuture f : requestCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + Assert.assertTrue(numStreamsFailures.get() == 0); + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java index d7173dcc4..36c8c51d9 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java @@ -1,3 +1,7 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ package software.amazon.awssdk.crt.test; import java.net.URI; @@ -139,7 +143,7 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { // Verify we got some Http Status Code for each Request Assert.assertTrue(reqIdToStatus.size() >= requiredSuccesses); - // Verify that the failure counts aren't too high ????? + // Verify that the failure counts aren't too high Assert.assertTrue(numErrorCode.get() <= allowedFailures); Assert.assertTrue(numStreamsFailures.get() <= allowedFailures); } From 78aebd81890b4511593afa98411a40b91b3ba3e5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 20 May 2022 15:38:18 -0700 Subject: [PATCH 120/142] forgot to commit action --- .builder/actions/localhost_test.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py index 82cc27937..ad079cae6 100644 --- a/.builder/actions/localhost_test.py +++ b/.builder/actions/localhost_test.py @@ -7,6 +7,19 @@ class LocalhostTest(Builder.Action): def run(self, env): # tests must run with leak detection turned on - actions = ["echo thisShouldBeLocalTest"] + env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') + actions = [] + if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + -DrerunFailingTestsCount=5 \ + -Daws.crt.memory.tracing=2 \ + -Daws.crt.debugnative=true \ + -Daws.crt.localhost=true"): + # Failed + actions.append("exit 1") + os.system("cat log.txt") + python = sys.executable + actions.append( + [python, 'crt/aws-c-http/integration-testing/http_client_test.py', + python, 'integration-testing/java_elasticurl_runner.py']) - return Builder.Script(actions, name='localhost-test') + return Builder.Script(actions, name='aws-crt-java-test') From a07b6bc97b9de2a7a442f3864f5562416eab4571 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 20 May 2022 15:50:28 -0700 Subject: [PATCH 121/142] remove log --- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index 40b84bf75..b76bfdce1 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -127,7 +127,6 @@ private Http2Request createHttp2Request(String method, URI uri, long bodyLength) @Test public void testParallelRequestsStress() throws Exception { skipIfLocalhostUnavailable(); - Log.initLoggingToStderr(Log.LogLevel.Trace); URI uri = new URI("https://localhost:8443/echo"); try( Http2StreamManager streamManager = createStreamManager(uri, 100)) { From 1b6d42620ae9c4085852cbcb5caa7ab1ef128c34 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Sat, 21 May 2022 11:56:57 -0700 Subject: [PATCH 122/142] add test wtih body --- crt/aws-c-http | 2 +- .../crt/test/Http2ClientLocalHostTest.java | 80 +++++++++++++++++-- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/crt/aws-c-http b/crt/aws-c-http index 73af2aa20..bf656bff0 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 73af2aa20d220dfbb26f22ef15df9651583cd5cb +Subproject commit bf656bff0cfc441281f3ace05ac868eb19ea2e12 diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index b76bfdce1..e6358b2e7 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -48,7 +48,8 @@ private Http2StreamManager createStreamManager(URI uri, int numConnections) { HostResolver resolver = new HostResolver(eventLoopGroup); ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); SocketOptions sockOpts = new SocketOptions(); - TlsContextOptions tlsOpts = TlsContextOptions.createDefaultClient().withAlpnList("h2").withVerifyPeer(false); + TlsContextOptions tlsOpts = TlsContextOptions.createDefaultClient().withAlpnList("h2") + .withVerifyPeer(false); TlsContext tlsContext = createHttpClientTlsContext(tlsOpts)) { HttpStreamManagerOptions options = new HttpStreamManagerOptions(); options.withClientBootstrap(bootstrap) @@ -116,7 +117,7 @@ private Http2Request createHttp2Request(String method, URI uri, long bodyLength) new HttpHeader("content-length", Long.toString(bodyLength)) }; HttpRequestBodyStream bodyStream = null; - if(bodyLength > 0) { + if (bodyLength > 0) { bodyStream = createBodyStreamWithLength(bodyLength); } Http2Request request = new Http2Request(requestHeaders, bodyStream); @@ -128,8 +129,7 @@ private Http2Request createHttp2Request(String method, URI uri, long bodyLength) public void testParallelRequestsStress() throws Exception { skipIfLocalhostUnavailable(); URI uri = new URI("https://localhost:8443/echo"); - try( - Http2StreamManager streamManager = createStreamManager(uri, 100)) { + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { int numberToAcquire = 500 * 100; Http2Request request = createHttp2Request("GET", uri, 0); @@ -142,7 +142,7 @@ public void testParallelRequestsStress() throws Exception { acquireCompleteFutures.add(streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { @Override public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, - HttpHeader[] nextHeaders) { + HttpHeader[] nextHeaders) { if (responseStatusCode != 200) { numStreamsFailures.incrementAndGet(); } @@ -170,4 +170,74 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } + + @Test + public void testParallelRequestsStressWithBody() throws Exception { + skipIfLocalhostUnavailable(); + URI uri = new URI("https://localhost:8443/uploadTest"); + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { + int numberToAcquire = 500 * 100; + if (CRT.getOSIdentifier() == "linux") { + /* + * Using Python hyper h2 server frame work, met a weird upload performance issue + * on Linux. Our client against nginx platform has not met the same issue. + * We assume it's because the server framework implementation. + * Use lower number of linux + */ + numberToAcquire = 500; + } + int bodyLength = 2000; + + List> requestCompleteFutures = new ArrayList<>(); + List> acquireCompleteFutures = new ArrayList<>(); + final AtomicInteger numStreamsFailures = new AtomicInteger(0); + for (int i = 0; i < numberToAcquire; i++) { + Http2Request request = createHttp2Request("PUT", uri, bodyLength); + + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + final int expectedLength = bodyLength; + requestCompleteFutures.add(requestCompleteFuture); + acquireCompleteFutures.add(streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + if (responseStatusCode != 200) { + numStreamsFailures.incrementAndGet(); + } + } + + @Override + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ + String bodyString = new String(bodyBytesIn); + int receivedLength = Integer.parseInt(bodyString); + + Assert.assertTrue(receivedLength == expectedLength); + if(receivedLength!=expectedLength) { + numStreamsFailures.incrementAndGet(); + } + return bodyString.length(); + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + if (errorCode != CRT.AWS_CRT_SUCCESS) { + numStreamsFailures.incrementAndGet(); + } + stream.close(); + requestCompleteFuture.complete(null); + } + })); + } + for (CompletableFuture f : acquireCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + // Wait for all Requests to complete + for (CompletableFuture f : requestCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + Assert.assertTrue(numStreamsFailures.get() == 0); + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } } From 372410d761663e59e555ff980c86667b468778d0 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Sun, 22 May 2022 14:38:00 -0700 Subject: [PATCH 123/142] test only 5? --- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index e6358b2e7..e169fb327 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -184,7 +184,7 @@ public void testParallelRequestsStressWithBody() throws Exception { * We assume it's because the server framework implementation. * Use lower number of linux */ - numberToAcquire = 500; + numberToAcquire = 5; } int bodyLength = 2000; From 945ca20838430d29945d400ac813fb7629a43217 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Sun, 22 May 2022 14:49:36 -0700 Subject: [PATCH 124/142] test 500 --- .builder/actions/localhost_test.py | 2 +- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py index ad079cae6..d65a1da7c 100644 --- a/.builder/actions/localhost_test.py +++ b/.builder/actions/localhost_test.py @@ -9,7 +9,7 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - if os.system("mvn -B test -DredirectTestOutputToFile=true -DforkCount=0 \ + if os.system("mvn -Dtest=Http2ClientLocalHostTest test -DredirectTestOutputToFile=true -DforkCount=0 \ -DrerunFailingTestsCount=5 \ -Daws.crt.memory.tracing=2 \ -Daws.crt.debugnative=true \ diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index e169fb327..e6358b2e7 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -184,7 +184,7 @@ public void testParallelRequestsStressWithBody() throws Exception { * We assume it's because the server framework implementation. * Use lower number of linux */ - numberToAcquire = 5; + numberToAcquire = 500; } int bodyLength = 2000; From eb7bd25be1c5c1ae09046938ef404404d7b1d989 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 09:58:28 -0700 Subject: [PATCH 125/142] added upload and download stress test --- .../crt/test/Http2ClientLocalHostTest.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index e6358b2e7..93a3f4c3b 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -13,6 +13,8 @@ import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + import org.junit.Assert; import org.junit.Assume; import org.junit.Test; @@ -240,4 +242,104 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } + + @Test + public void testRequestsUploadStress() throws Exception { + /* Test that upload a 2.5GB data from local server (0.25GB for linux) */ + skipIfLocalhostUnavailable(); + URI uri = new URI("https://localhost:8443/uploadTest"); + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { + long bodyLength = 2500000000L; + if (CRT.getOSIdentifier() == "linux") { + /* + * Using Python hyper h2 server frame work, met a weird upload performance issue + * on Linux. Our client against nginx platform has not met the same issue. + * We assume it's because the server framework implementation. + * Use lower number of linux + */ + bodyLength = 250000000L; + } + + Http2Request request = createHttp2Request("PUT", uri, bodyLength); + + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + final long expectedLength = bodyLength; + CompletableFuture acquireCompleteFuture = streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + + Assert.assertTrue(responseStatusCode == 200); + } + + @Override + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ + String bodyString = new String(bodyBytesIn); + long receivedLength = Long.parseLong(bodyString); + + Assert.assertTrue(receivedLength == expectedLength); + return bodyString.length(); + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + + Assert.assertTrue(errorCode == CRT.AWS_CRT_SUCCESS); + stream.close(); + requestCompleteFuture.complete(null); + } + }); + + acquireCompleteFuture.get(30, TimeUnit.SECONDS); + requestCompleteFuture.get(30, TimeUnit.SECONDS); + + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testRequestsDownloadStress() throws Exception { + /* Test that download a 2.5GB data from local server */ + skipIfLocalhostUnavailable(); + URI uri = new URI("https://localhost:8443/downloadTest"); + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { + long bodyLength = 2500000000L; + + Http2Request request = createHttp2Request("GET", uri, 0); + + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + final AtomicLong receivedLength = new AtomicLong(0); + CompletableFuture acquireCompleteFuture = streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + + Assert.assertTrue(responseStatusCode == 200); + } + + @Override + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ + receivedLength.addAndGet(bodyBytesIn.length); + + return bodyBytesIn.length; + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + + Assert.assertTrue(errorCode == CRT.AWS_CRT_SUCCESS); + stream.close(); + requestCompleteFuture.complete(null); + } + }); + + acquireCompleteFuture.get(30, TimeUnit.SECONDS); + requestCompleteFuture.get(30, TimeUnit.SECONDS); + + Assert.assertTrue(receivedLength.get() == bodyLength); + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } } From d84b0320f68182006c304fc7fabb435ce4e9253d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 10:10:18 -0700 Subject: [PATCH 126/142] add error level log --- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index 93a3f4c3b..5ebb8c397 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -45,6 +45,7 @@ public class Http2ClientLocalHostTest extends HttpClientTestFixture { private Http2StreamManager createStreamManager(URI uri, int numConnections) { + Log.initLoggingToStderr(Log.LogLevel.Error); try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); HostResolver resolver = new HostResolver(eventLoopGroup); From 0194696b41252f967251e0970c7711a8af987055 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 10:16:52 -0700 Subject: [PATCH 127/142] run again --- .builder/actions/localhost_test.py | 2 +- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py index d65a1da7c..d112af6b2 100644 --- a/.builder/actions/localhost_test.py +++ b/.builder/actions/localhost_test.py @@ -9,7 +9,7 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - if os.system("mvn -Dtest=Http2ClientLocalHostTest test -DredirectTestOutputToFile=true -DforkCount=0 \ + if os.system("mvn -Dtest=Http2ClientLocalHostTest#testRequestsUploadStress test -DredirectTestOutputToFile=true -DforkCount=0 \ -DrerunFailingTestsCount=5 \ -Daws.crt.memory.tracing=2 \ -Daws.crt.debugnative=true \ diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index 5ebb8c397..c0a5fb90f 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -45,7 +45,6 @@ public class Http2ClientLocalHostTest extends HttpClientTestFixture { private Http2StreamManager createStreamManager(URI uri, int numConnections) { - Log.initLoggingToStderr(Log.LogLevel.Error); try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); HostResolver resolver = new HostResolver(eventLoopGroup); @@ -248,6 +247,7 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { public void testRequestsUploadStress() throws Exception { /* Test that upload a 2.5GB data from local server (0.25GB for linux) */ skipIfLocalhostUnavailable(); + Log.initLoggingToStderr(Log.LogLevel.Debug); URI uri = new URI("https://localhost:8443/uploadTest"); try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { long bodyLength = 2500000000L; From fca7b3e01f66f8574766ac73e33f74ed8f3b942b Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 14:42:10 -0700 Subject: [PATCH 128/142] checkout the body received? --- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index c0a5fb90f..6fafb9395 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -247,10 +247,10 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { public void testRequestsUploadStress() throws Exception { /* Test that upload a 2.5GB data from local server (0.25GB for linux) */ skipIfLocalhostUnavailable(); - Log.initLoggingToStderr(Log.LogLevel.Debug); +// Log.initLoggingToStderr(Log.LogLevel.Debug); URI uri = new URI("https://localhost:8443/uploadTest"); try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { - long bodyLength = 2500000000L; + long bodyLength = 2500L; if (CRT.getOSIdentifier() == "linux") { /* * Using Python hyper h2 server frame work, met a weird upload performance issue @@ -277,7 +277,7 @@ public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ String bodyString = new String(bodyBytesIn); long receivedLength = Long.parseLong(bodyString); - + System.out.println(bodyString); Assert.assertTrue(receivedLength == expectedLength); return bodyString.length(); } From 4a25bed0e6dd7335eb685918d4c74c5aa5b90849 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 14:48:33 -0700 Subject: [PATCH 129/142] back to 2.5G --- .github/workflows/ci.yml | 288 +++++++++--------- .../crt/test/Http2ClientLocalHostTest.java | 2 +- 2 files changed, 145 insertions(+), 145 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb5feeac7..8b9407038 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,158 +19,158 @@ env: AWS_REGION: us-east-1 jobs: - linux-compat: - runs-on: ubuntu-20.04 # latest - strategy: - matrix: - image: - - manylinux2014-x64 - - al2-x64 - - fedora-34-x64 - - opensuse-leap - - rhel8-x64 - #- manylinux2014-x86 until we find 32-bit linux binaries we can use - steps: - - name: Checkout Sources - uses: actions/checkout@v2 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - - uses: actions/upload-artifact@v2 - if: failure() - with: - name: logs - path: | - hs_err_pid* - target/surefire-reports/*stream + # linux-compat: + # runs-on: ubuntu-20.04 # latest + # strategy: + # matrix: + # image: + # - manylinux2014-x64 + # - al2-x64 + # - fedora-34-x64 + # - opensuse-leap + # - rhel8-x64 + # #- manylinux2014-x86 until we find 32-bit linux binaries we can use + # steps: + # - name: Checkout Sources + # uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Build ${{ env.PACKAGE_NAME }} + # run: | + # aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + # ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + # - uses: actions/upload-artifact@v2 + # if: failure() + # with: + # name: logs + # path: | + # hs_err_pid* + # target/surefire-reports/*stream - linux-compiler-compat: - runs-on: ubuntu-20.04 # latest - strategy: - matrix: - compiler: - - clang-3 - - clang-6 - - clang-8 - - clang-9 - - clang-10 - - clang-11 - - gcc-4.8 - - gcc-5 - - gcc-6 - - gcc-7 - - gcc-8 - steps: - - name: Checkout Sources - uses: actions/checkout@v2 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} - - uses: actions/upload-artifact@v2 - if: failure() - with: - name: logs - path: | - hs_err_pid* - target/surefire-reports/*stream + # linux-compiler-compat: + # runs-on: ubuntu-20.04 # latest + # strategy: + # matrix: + # compiler: + # - clang-3 + # - clang-6 + # - clang-8 + # - clang-9 + # - clang-10 + # - clang-11 + # - gcc-4.8 + # - gcc-5 + # - gcc-6 + # - gcc-7 + # - gcc-8 + # steps: + # - name: Checkout Sources + # uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Build ${{ env.PACKAGE_NAME }} + # run: | + # aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + # ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} + # - uses: actions/upload-artifact@v2 + # if: failure() + # with: + # name: logs + # path: | + # hs_err_pid* + # target/surefire-reports/*stream - linux-arm: - name: ARM (${{ matrix.arch }}) - runs-on: ubuntu-20.04 # latest - strategy: - matrix: - arch: [armv6, armv7, arm64] - steps: - - name: Build ${{ env.PACKAGE_NAME }} - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream + # linux-arm: + # name: ARM (${{ matrix.arch }}) + # runs-on: ubuntu-20.04 # latest + # strategy: + # matrix: + # arch: [armv6, armv7, arm64] + # steps: + # - name: Build ${{ env.PACKAGE_NAME }} + # run: | + # python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + # chmod a+x builder + # ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream - windows: - runs-on: windows-2022 # latest - steps: - - name: Checkout Sources - uses: actions/checkout@v2 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream + # windows: + # runs-on: windows-2022 # latest + # steps: + # - name: Checkout Sources + # uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Build ${{ env.PACKAGE_NAME }} + consumers + # run: | + # python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + # python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream - windows-vc14: - runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2015 (v14.0) - strategy: - matrix: - arch: [x86, x64] - steps: - - name: Checkout Sources - uses: actions/checkout@v2 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - env: - AWS_CMAKE_TOOLSET: v140 - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream + # windows-vc14: + # runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2015 (v14.0) + # strategy: + # matrix: + # arch: [x86, x64] + # steps: + # - name: Checkout Sources + # uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Build ${{ env.PACKAGE_NAME }} + consumers + # env: + # AWS_CMAKE_TOOLSET: v140 + # run: | + # python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + # python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream - osx: - runs-on: macos-11 # latest - steps: - - name: Checkout Sources - uses: actions/checkout@v2 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream + # osx: + # runs-on: macos-11 # latest + # steps: + # - name: Checkout Sources + # uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Build ${{ env.PACKAGE_NAME }} + consumers + # run: | + # python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + # chmod a+x builder + # ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream - android: - name: Android - # ubuntu-20.04 comes with Android tooling, see: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md#android - runs-on: ubuntu-20.04 # latest - steps: - - name: Checkout Sources - uses: actions/checkout@v2 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} - run: | - ./gradlew :android:crt:build + # android: + # name: Android + # # ubuntu-20.04 comes with Android tooling, see: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md#android + # runs-on: ubuntu-20.04 # latest + # steps: + # - name: Checkout Sources + # uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Build ${{ env.PACKAGE_NAME }} + # run: | + # ./gradlew :android:crt:build - # check that docs can still build - check-docs: - runs-on: ubuntu-20.04 # latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Check docs - run: | - ./make-docs.sh + # # check that docs can still build + # check-docs: + # runs-on: ubuntu-20.04 # latest + # steps: + # - uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Check docs + # run: | + # ./make-docs.sh - check-submodules: - runs-on: ubuntu-20.04 # latest - steps: - - name: Checkout Source - uses: actions/checkout@v2 - with: - submodules: true - fetch-depth: 0 - - name: Check Submodules - # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work - # https://github.com/actions/runner/issues/480 - uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main + # check-submodules: + # runs-on: ubuntu-20.04 # latest + # steps: + # - name: Checkout Source + # uses: actions/checkout@v2 + # with: + # submodules: true + # fetch-depth: 0 + # - name: Check Submodules + # # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work + # # https://github.com/actions/runner/issues/480 + # uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main localhost-test-linux: diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index 6fafb9395..071859e8e 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -250,7 +250,7 @@ public void testRequestsUploadStress() throws Exception { // Log.initLoggingToStderr(Log.LogLevel.Debug); URI uri = new URI("https://localhost:8443/uploadTest"); try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { - long bodyLength = 2500L; + long bodyLength = 2500000000L; if (CRT.getOSIdentifier() == "linux") { /* * Using Python hyper h2 server frame work, met a weird upload performance issue From d01c96afd08bbbdfa764e09dbd99f9bb9a91ca92 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 14:56:12 -0700 Subject: [PATCH 130/142] why???? --- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index 071859e8e..717a9d1ff 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -291,10 +291,13 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { } }); + System.out.println("before acquire"); acquireCompleteFuture.get(30, TimeUnit.SECONDS); + System.out.println("before complete"); requestCompleteFuture.get(30, TimeUnit.SECONDS); } + System.out.println("before check resource"); CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } From dfce85fc579c5b178b3e9130b0834684b86583c7 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 15:12:20 -0700 Subject: [PATCH 131/142] more printout --- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index 717a9d1ff..92d39dad7 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -278,14 +278,18 @@ public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ String bodyString = new String(bodyBytesIn); long receivedLength = Long.parseLong(bodyString); System.out.println(bodyString); + System.out.println("before length check"); Assert.assertTrue(receivedLength == expectedLength); + System.out.println("after length check"); return bodyString.length(); } @Override public void onResponseComplete(HttpStreamBase stream, int errorCode) { + System.out.println("before error code check"); Assert.assertTrue(errorCode == CRT.AWS_CRT_SUCCESS); + System.out.println("after error code check"); stream.close(); requestCompleteFuture.complete(null); } @@ -295,6 +299,7 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { acquireCompleteFuture.get(30, TimeUnit.SECONDS); System.out.println("before complete"); requestCompleteFuture.get(30, TimeUnit.SECONDS); + System.out.println("after complete"); } System.out.println("before check resource"); From b9aa5bcf7517e25a626f16809c8aee65ae60c998 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 15:21:19 -0700 Subject: [PATCH 132/142] more? --- .../amazon/awssdk/crt/test/Http2ClientLocalHostTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index 92d39dad7..fff23ebbb 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -291,14 +291,16 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { Assert.assertTrue(errorCode == CRT.AWS_CRT_SUCCESS); System.out.println("after error code check"); stream.close(); + System.out.println("after stream close"); requestCompleteFuture.complete(null); + System.out.println("after complete"); } }); System.out.println("before acquire"); acquireCompleteFuture.get(30, TimeUnit.SECONDS); System.out.println("before complete"); - requestCompleteFuture.get(30, TimeUnit.SECONDS); + requestCompleteFuture.join(); System.out.println("after complete"); } From 82175156c6275e04335ec5e0d927eea245041e33 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 17:11:18 -0700 Subject: [PATCH 133/142] use join instead --- .builder/actions/localhost_test.py | 3 +-- .../awssdk/crt/test/Http2ClientLocalHostTest.java | 14 +------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py index d112af6b2..e49363679 100644 --- a/.builder/actions/localhost_test.py +++ b/.builder/actions/localhost_test.py @@ -9,14 +9,13 @@ def run(self, env): # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] - if os.system("mvn -Dtest=Http2ClientLocalHostTest#testRequestsUploadStress test -DredirectTestOutputToFile=true -DforkCount=0 \ + if os.system("mvn -Dtest=Http2ClientLocalHostTest test -DredirectTestOutputToFile=true -DforkCount=0 \ -DrerunFailingTestsCount=5 \ -Daws.crt.memory.tracing=2 \ -Daws.crt.debugnative=true \ -Daws.crt.localhost=true"): # Failed actions.append("exit 1") - os.system("cat log.txt") python = sys.executable actions.append( [python, 'crt/aws-c-http/integration-testing/http_client_test.py', diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java index fff23ebbb..15859eb4a 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -277,34 +277,22 @@ public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ String bodyString = new String(bodyBytesIn); long receivedLength = Long.parseLong(bodyString); - System.out.println(bodyString); - System.out.println("before length check"); Assert.assertTrue(receivedLength == expectedLength); - System.out.println("after length check"); return bodyString.length(); } @Override public void onResponseComplete(HttpStreamBase stream, int errorCode) { - - System.out.println("before error code check"); Assert.assertTrue(errorCode == CRT.AWS_CRT_SUCCESS); - System.out.println("after error code check"); stream.close(); - System.out.println("after stream close"); requestCompleteFuture.complete(null); - System.out.println("after complete"); } }); - System.out.println("before acquire"); acquireCompleteFuture.get(30, TimeUnit.SECONDS); - System.out.println("before complete"); requestCompleteFuture.join(); - System.out.println("after complete"); } - System.out.println("before check resource"); CrtResource.logNativeResources(); CrtResource.waitForNoResources(); } @@ -346,7 +334,7 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { }); acquireCompleteFuture.get(30, TimeUnit.SECONDS); - requestCompleteFuture.get(30, TimeUnit.SECONDS); + requestCompleteFuture.join(); Assert.assertTrue(receivedLength.get() == bodyLength); } From d86b30313c90368b364b38bb0afa6b5aea1065d5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 17:18:37 -0700 Subject: [PATCH 134/142] get the rest of CI back --- .github/workflows/ci.yml | 288 +++++++++++++++++++-------------------- 1 file changed, 144 insertions(+), 144 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b9407038..eb5feeac7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,158 +19,158 @@ env: AWS_REGION: us-east-1 jobs: - # linux-compat: - # runs-on: ubuntu-20.04 # latest - # strategy: - # matrix: - # image: - # - manylinux2014-x64 - # - al2-x64 - # - fedora-34-x64 - # - opensuse-leap - # - rhel8-x64 - # #- manylinux2014-x86 until we find 32-bit linux binaries we can use - # steps: - # - name: Checkout Sources - # uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Build ${{ env.PACKAGE_NAME }} - # run: | - # aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - # ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - # - uses: actions/upload-artifact@v2 - # if: failure() - # with: - # name: logs - # path: | - # hs_err_pid* - # target/surefire-reports/*stream + linux-compat: + runs-on: ubuntu-20.04 # latest + strategy: + matrix: + image: + - manylinux2014-x64 + - al2-x64 + - fedora-34-x64 + - opensuse-leap + - rhel8-x64 + #- manylinux2014-x86 until we find 32-bit linux binaries we can use + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: logs + path: | + hs_err_pid* + target/surefire-reports/*stream - # linux-compiler-compat: - # runs-on: ubuntu-20.04 # latest - # strategy: - # matrix: - # compiler: - # - clang-3 - # - clang-6 - # - clang-8 - # - clang-9 - # - clang-10 - # - clang-11 - # - gcc-4.8 - # - gcc-5 - # - gcc-6 - # - gcc-7 - # - gcc-8 - # steps: - # - name: Checkout Sources - # uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Build ${{ env.PACKAGE_NAME }} - # run: | - # aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - # ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} - # - uses: actions/upload-artifact@v2 - # if: failure() - # with: - # name: logs - # path: | - # hs_err_pid* - # target/surefire-reports/*stream + linux-compiler-compat: + runs-on: ubuntu-20.04 # latest + strategy: + matrix: + compiler: + - clang-3 + - clang-6 + - clang-8 + - clang-9 + - clang-10 + - clang-11 + - gcc-4.8 + - gcc-5 + - gcc-6 + - gcc-7 + - gcc-8 + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: logs + path: | + hs_err_pid* + target/surefire-reports/*stream - # linux-arm: - # name: ARM (${{ matrix.arch }}) - # runs-on: ubuntu-20.04 # latest - # strategy: - # matrix: - # arch: [armv6, armv7, arm64] - # steps: - # - name: Build ${{ env.PACKAGE_NAME }} - # run: | - # python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - # chmod a+x builder - # ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream + linux-arm: + name: ARM (${{ matrix.arch }}) + runs-on: ubuntu-20.04 # latest + strategy: + matrix: + arch: [armv6, armv7, arm64] + steps: + - name: Build ${{ env.PACKAGE_NAME }} + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-${{ matrix.arch }} --spec=downstream - # windows: - # runs-on: windows-2022 # latest - # steps: - # - name: Checkout Sources - # uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Build ${{ env.PACKAGE_NAME }} + consumers - # run: | - # python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - # python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream + windows: + runs-on: windows-2022 # latest + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream - # windows-vc14: - # runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2015 (v14.0) - # strategy: - # matrix: - # arch: [x86, x64] - # steps: - # - name: Checkout Sources - # uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Build ${{ env.PACKAGE_NAME }} + consumers - # env: - # AWS_CMAKE_TOOLSET: v140 - # run: | - # python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - # python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream + windows-vc14: + runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2015 (v14.0) + strategy: + matrix: + arch: [x86, x64] + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + env: + AWS_CMAKE_TOOLSET: v140 + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream - # osx: - # runs-on: macos-11 # latest - # steps: - # - name: Checkout Sources - # uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Build ${{ env.PACKAGE_NAME }} + consumers - # run: | - # python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - # chmod a+x builder - # ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream + osx: + runs-on: macos-11 # latest + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream - # android: - # name: Android - # # ubuntu-20.04 comes with Android tooling, see: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md#android - # runs-on: ubuntu-20.04 # latest - # steps: - # - name: Checkout Sources - # uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Build ${{ env.PACKAGE_NAME }} - # run: | - # ./gradlew :android:crt:build + android: + name: Android + # ubuntu-20.04 comes with Android tooling, see: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md#android + runs-on: ubuntu-20.04 # latest + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + run: | + ./gradlew :android:crt:build - # # check that docs can still build - # check-docs: - # runs-on: ubuntu-20.04 # latest - # steps: - # - uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Check docs - # run: | - # ./make-docs.sh + # check that docs can still build + check-docs: + runs-on: ubuntu-20.04 # latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Check docs + run: | + ./make-docs.sh - # check-submodules: - # runs-on: ubuntu-20.04 # latest - # steps: - # - name: Checkout Source - # uses: actions/checkout@v2 - # with: - # submodules: true - # fetch-depth: 0 - # - name: Check Submodules - # # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work - # # https://github.com/actions/runner/issues/480 - # uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main + check-submodules: + runs-on: ubuntu-20.04 # latest + steps: + - name: Checkout Source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 0 + - name: Check Submodules + # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work + # https://github.com/actions/runner/issues/480 + uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main localhost-test-linux: From 6517d5104573392c3a3cfea9a153f7b2da80ed85 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 May 2022 17:22:24 -0700 Subject: [PATCH 135/142] get rid of sutff not needed --- .builder/actions/localhost_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py index e49363679..0dd32b385 100644 --- a/.builder/actions/localhost_test.py +++ b/.builder/actions/localhost_test.py @@ -6,7 +6,6 @@ class LocalhostTest(Builder.Action): def run(self, env): - # tests must run with leak detection turned on env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') actions = [] if os.system("mvn -Dtest=Http2ClientLocalHostTest test -DredirectTestOutputToFile=true -DforkCount=0 \ @@ -16,9 +15,5 @@ def run(self, env): -Daws.crt.localhost=true"): # Failed actions.append("exit 1") - python = sys.executable - actions.append( - [python, 'crt/aws-c-http/integration-testing/http_client_test.py', - python, 'integration-testing/java_elasticurl_runner.py']) return Builder.Script(actions, name='aws-crt-java-test') From e7ad920e2f900c2aa1f3ab92856ecc6e5c3f44c9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 2 Jun 2022 15:55:11 -0700 Subject: [PATCH 136/142] add todo and remove things don't support anymore --- .../software/amazon/awssdk/crt/http/HttpStreamManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java index 498980751..72a0b4a94 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -36,12 +36,11 @@ public static HttpStreamManager create(HttpStreamManagerOptions options) { } private HttpClientConnectionManagerOptions getConnManagerOptFromStreamManagerOpt(HttpStreamManagerOptions options) { - + // TODO: split the HTTP/1 stream manager to its own thing HttpClientConnectionManagerOptions connManagerOptions = new HttpClientConnectionManagerOptions(); connManagerOptions.withClientBootstrap(options.getClientBootstrap()) .withSocketOptions(options.getSocketOptions()) .withTlsContext(options.getTlsContext()) - .withWindowSize(options.getWindowSize()) .withUri(options.getUri()) .withMaxConnections(options.getMaxConnections()) .withPort(options.getPort()) From 6e565e93bcd9cd0ec6c45b1c86d4755111b6987c Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 2 Jun 2022 16:26:33 -0700 Subject: [PATCH 137/142] fix broken api --- .../awssdk/crt/http/Http2StreamManager.java | 2 +- .../awssdk/crt/http/HttpStreamManager.java | 18 +++++++++--------- .../crt/test/Http2RequestResponseTest.java | 2 +- .../awssdk/crt/test/HttpStreamManagerTest.java | 12 +++++++----- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index 71c7c6839..f86db0e1b 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -186,7 +186,7 @@ public CompletableFuture acquireStream(HttpRequest request, return this.acquireStream((HttpRequestBase) request, streamHandler); } - private CompletableFuture acquireStream(HttpRequestBase request, + public CompletableFuture acquireStream(HttpRequestBase request, HttpStreamBaseResponseHandler streamHandler) { CompletableFuture completionFuture = new CompletableFuture<>(); AsyncCallback acquireStreamCompleted = AsyncCallback.wrapFuture(completionFuture, null); diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java index 72a0b4a94..94f98b9b7 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -124,9 +124,9 @@ public HttpVersion getHttpVersion() throws Exception { * acquired. * @throws CrtRuntimeException */ - public CompletableFuture acquireStream(HttpRequestBase request, - HttpStreamResponseHandler streamHandler) { - CompletableFuture completionFuture = new CompletableFuture<>(); + public CompletableFuture acquireStream(HttpRequestBase request, + HttpStreamBaseResponseHandler streamHandler) { + CompletableFuture completionFuture = new CompletableFuture<>(); try { /* * Try get version first. If we haven't decided the version yet, this will help @@ -142,7 +142,7 @@ public CompletableFuture acquireStream(HttpRequestBase request, if (throwable != null) { completionFuture.completeExceptionally(throwable); } else { - completionFuture.complete((HttpStream) stream); + completionFuture.complete((Http2Stream) stream); } }); return completionFuture; @@ -153,25 +153,25 @@ public CompletableFuture acquireStream(HttpRequestBase request, completionFuture.completeExceptionally(throwable); } else { try { - HttpStream stream = conn.makeRequest(request, new HttpStreamResponseHandler() { + HttpStreamBase stream = conn.makeRequest(request, new HttpStreamBaseResponseHandler() { @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { streamHandler.onResponseHeaders(stream, responseStatusCode, blockType, nextHeaders); } @Override - public void onResponseHeadersDone(HttpStream stream, int blockType) { + public void onResponseHeadersDone(HttpStreamBase stream, int blockType) { streamHandler.onResponseHeadersDone(stream, blockType); } @Override - public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn) { return streamHandler.onResponseBody(stream, bodyBytesIn); } @Override - public void onResponseComplete(HttpStream stream, int errorCode) { + public void onResponseComplete(HttpStreamBase stream, int errorCode) { streamHandler.onResponseComplete(stream, errorCode); /* Release the connection back */ connManager.releaseConnection(conn); diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java index 34fed45e8..c729e3c4a 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java @@ -186,7 +186,7 @@ public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int } @Override - public void onResponseHeadersDone(HttpStream stream, int blockType) { + public void onResponseHeadersDone(HttpStreamBase stream, int blockType) { /* Only invoke once */ Http2Stream h2Stream = (Http2Stream) stream; h2Stream.resetStream(Http2ErrorCode.INTERNAL_ERROR); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index c2ada6388..a81f388d9 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -41,6 +41,8 @@ import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.http.HttpVersion; import software.amazon.awssdk.crt.http.HttpStream; +import software.amazon.awssdk.crt.http.HttpStreamBase; +import software.amazon.awssdk.crt.http.HttpStreamBaseResponseHandler; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.EventLoopGroup; import software.amazon.awssdk.crt.io.HostResolver; @@ -149,9 +151,9 @@ private TestHttpResponse getResponseFromManager(HttpStreamManager streamManager, final TestHttpResponse response = new TestHttpResponse(); try { - HttpStreamResponseHandler streamHandler = new HttpStreamResponseHandler() { + HttpStreamBaseResponseHandler streamHandler = new HttpStreamBaseResponseHandler() { @Override - public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blockType, + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, HttpHeader[] nextHeaders) { response.statusCode = responseStatusCode; Assert.assertEquals(responseStatusCode, stream.getResponseStatusCode()); @@ -159,12 +161,12 @@ public void onResponseHeaders(HttpStream stream, int responseStatusCode, int blo } @Override - public void onResponseHeadersDone(HttpStream stream, int blockType) { + public void onResponseHeadersDone(HttpStreamBase stream, int blockType) { response.blockType = blockType; } @Override - public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn) { try { response.bodyBuffer.put(bodyBytesIn); } catch (Exception e) { @@ -177,7 +179,7 @@ public int onResponseBody(HttpStream stream, byte[] bodyBytesIn) { } @Override - public void onResponseComplete(HttpStream stream, int errorCode) { + public void onResponseComplete(HttpStreamBase stream, int errorCode) { response.onCompleteErrorCode = errorCode; reqCompleted.complete(null); stream.close(); From 77eac51c2c703da204704c05306499eee9bdf744 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 3 Jun 2022 18:02:46 -0700 Subject: [PATCH 138/142] update submodules --- crt/aws-c-event-stream | 2 +- crt/aws-c-http | 2 +- crt/aws-c-io | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crt/aws-c-event-stream b/crt/aws-c-event-stream index 9141f54ca..956f3442f 160000 --- a/crt/aws-c-event-stream +++ b/crt/aws-c-event-stream @@ -1 +1 @@ -Subproject commit 9141f54caa20376cce5633e785ffd628df569429 +Subproject commit 956f3442f6b9132f68b21d0604316fd11b698c7a diff --git a/crt/aws-c-http b/crt/aws-c-http index bf656bff0..aadc57a38 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit bf656bff0cfc441281f3ace05ac868eb19ea2e12 +Subproject commit aadc57a3829cb057acdd6d57ed0991ff61a71192 diff --git a/crt/aws-c-io b/crt/aws-c-io index 8f4508f5e..8b93aaed5 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit 8f4508f5ec7d2949d5545e2b1ddcd1beb47a76a8 +Subproject commit 8b93aaed5e2ae26f459380bbdbc90ee1e578f8d3 From 2c7cdab8d8526a9fe383578a0aeb6bbfba68d318 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 17 Jun 2022 15:02:53 -0700 Subject: [PATCH 139/142] adapt new api --- src/native/http2_stream_manager.c | 47 ++++++++++++++++++------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c index 38443480f..933c1ce2f 100644 --- a/src/native/http2_stream_manager.c +++ b/src/native/http2_stream_manager.c @@ -47,12 +47,10 @@ struct aws_http2_stream_manager_binding { struct aws_http2_stream_manager *stream_manager; }; -static void s_destroy_manager_binding(struct aws_http2_stream_manager_binding *binding) { +static void s_destroy_manager_binding(struct aws_http2_stream_manager_binding *binding, JNIEnv *env) { if (binding == NULL) { return; } - - JNIEnv *env = aws_jni_get_thread_env(binding->jvm); if (binding->java_http2_stream_manager != NULL) { (*env)->DeleteWeakGlobalRef(env, binding->java_http2_stream_manager); } @@ -60,22 +58,27 @@ static void s_destroy_manager_binding(struct aws_http2_stream_manager_binding *b aws_mem_release(aws_jni_get_allocator(), binding); } -static void s_on_http_conn_manager_shutdown_complete_callback(void *user_data) { +static void s_on_stream_manager_shutdown_complete_callback(void *user_data) { struct aws_http2_stream_manager_binding *binding = (struct aws_http2_stream_manager_binding *)user_data; - JNIEnv *env = aws_jni_get_thread_env(binding->jvm); + /********** JNI ENV ACQUIRE **********/ + JNIEnv *env = aws_jni_acquire_thread_env(binding->jvm); AWS_LOGF_DEBUG(AWS_LS_HTTP_STREAM_MANAGER, "Java Stream Manager Shutdown Complete"); jobject java_http2_stream_manager = (*env)->NewLocalRef(env, binding->java_http2_stream_manager); if (java_http2_stream_manager != NULL) { (*env)->CallVoidMethod(env, java_http2_stream_manager, http2_stream_manager_properties.onShutdownComplete); - AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); + /* If exception raised from Java callback, but we already closed the stream manager, just move on */ + aws_jni_check_and_clear_exception(env); + (*env)->DeleteLocalRef(env, java_http2_stream_manager); } - // We're done with this wrapper, free it. - s_destroy_manager_binding(binding); + /* We're done with this wrapper, free it. */ + s_destroy_manager_binding(binding, env); + aws_jni_release_thread_env(binding->jvm, env); + /********** JNI ENV RELEASE **********/ } JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_http2StreamManagerNew( @@ -111,12 +114,12 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ struct aws_allocator *allocator = aws_jni_get_allocator(); if (!client_bootstrap) { - aws_jni_throw_runtime_exception(env, "ClientBootstrap can't be null"); + aws_jni_throw_illegal_argument_exception(env, "ClientBootstrap can't be null"); return (jlong)NULL; } if (!socket_options) { - aws_jni_throw_runtime_exception(env, "SocketOptions can't be null"); + aws_jni_throw_illegal_argument_exception(env, "SocketOptions can't be null"); return (jlong)NULL; } @@ -130,20 +133,21 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ jlong *marshalled_settings = (*env)->GetLongArrayElements(env, java_marshalled_settings, NULL); for (size_t i = 0; i < num_initial_settings; i++) { jlong id = marshalled_settings[i * 2]; - initial_settings[i].id = id; + initial_settings[i].id = (uint32_t)id; jlong value = marshalled_settings[i * 2 + 1]; + /* We checked the value can fit into uint32_t in Java already */ initial_settings[i].value = (uint32_t)value; } struct aws_byte_cursor endpoint = aws_jni_byte_cursor_from_jbyteArray_acquire(env, jni_endpoint); if (jni_port <= 0 || 65535 < jni_port) { - aws_jni_throw_runtime_exception(env, "Port must be between 1 and 65535"); + aws_jni_throw_illegal_argument_exception(env, "Port must be between 1 and 65535"); goto cleanup; } if (jni_max_conns <= 0) { - aws_jni_throw_runtime_exception(env, "Max Connections must be > 0"); + aws_jni_throw_illegal_argument_exception(env, "Max Connections must be > 0"); goto cleanup; } @@ -178,7 +182,7 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ manager_options.tls_connection_options = NULL; manager_options.host = endpoint; manager_options.port = port; - manager_options.shutdown_complete_callback = &s_on_http_conn_manager_shutdown_complete_callback; + manager_options.shutdown_complete_callback = &s_on_stream_manager_shutdown_complete_callback; manager_options.shutdown_complete_user_data = binding; manager_options.monitoring_options = NULL; /* TODO: this variable needs to be renamed in aws-c-http. Come back and change it next revision. */ @@ -241,7 +245,7 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_ aws_jni_byte_cursor_from_jbyteArray_release(env, jni_endpoint, endpoint); if (binding->stream_manager == NULL) { - s_destroy_manager_binding(binding); + s_destroy_manager_binding(binding, env); binding = NULL; } @@ -257,9 +261,9 @@ struct aws_sm_acquire_stream_callback_data { jobject java_async_callback; }; -static void s_cleanup_sm_acquire_stream_callback_data(struct aws_sm_acquire_stream_callback_data *callback_data) { - - JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); +static void s_cleanup_sm_acquire_stream_callback_data( + struct aws_sm_acquire_stream_callback_data *callback_data, + JNIEnv *env) { if (callback_data->java_async_callback) { (*env)->DeleteGlobalRef(env, callback_data->java_async_callback); @@ -286,7 +290,8 @@ static struct aws_sm_acquire_stream_callback_data *s_new_sm_acquire_stream_callb static void s_on_stream_acquired(struct aws_http_stream *stream, int error_code, void *user_data) { struct aws_sm_acquire_stream_callback_data *callback_data = user_data; - JNIEnv *env = aws_jni_get_thread_env(callback_data->jvm); + /********** JNI ENV ACQUIRE **********/ + JNIEnv *env = aws_jni_acquire_thread_env(callback_data->jvm); if (error_code) { jobject crt_exception = aws_jni_new_crt_exception_from_error_code(env, error_code); (*env)->CallVoidMethod( @@ -316,7 +321,9 @@ static void s_on_stream_acquired(struct aws_http_stream *stream, int error_code, } } AWS_FATAL_ASSERT(!aws_jni_check_and_clear_exception(env)); - s_cleanup_sm_acquire_stream_callback_data(callback_data); + s_cleanup_sm_acquire_stream_callback_data(callback_data, env); + aws_jni_release_thread_env(callback_data->jvm, env); + /********** JNI ENV RELEASE **********/ } JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_http2StreamManagerAcquireStream( From a1f6f9002238aef57e93f7fc8d283ac21159f420 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 20 Jun 2022 15:49:30 -0700 Subject: [PATCH 140/142] comments addressed --- .../awssdk/crt/http/Http2StreamManager.java | 82 ++---- .../crt/http/Http2StreamManagerOptions.java | 159 +++++++++++ .../crt/http/HttpClientConnectionManager.java | 17 +- .../HttpClientConnectionManagerOptions.java | 43 +++ .../crt/http/HttpStreamManagerOptions.java | 265 ------------------ src/native/http2_stream_manager.c | 2 +- .../crt/test/Http2StreamManagerTest.java | 8 +- 7 files changed, 233 insertions(+), 343 deletions(-) create mode 100644 src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java delete mode 100644 src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java index ed184e49a..0559283a7 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManager.java @@ -36,72 +36,32 @@ public class Http2StreamManager extends CrtResource { * @param options configuration options * @return a new instance of an Http2StreamManager */ - public static Http2StreamManager create(HttpStreamManagerOptions options) { + public static Http2StreamManager create(Http2StreamManagerOptions options) { return new Http2StreamManager(options); } - private Http2StreamManager(HttpStreamManagerOptions options) { - URI uri = options.getUri(); - if (uri == null) { - throw new IllegalArgumentException("URI must not be null"); - } - if (uri.getScheme() == null) { - throw new IllegalArgumentException("URI does not have a Scheme"); - } - if (!HTTP.equals(uri.getScheme()) && !HTTPS.equals(uri.getScheme())) { - throw new IllegalArgumentException("URI has unknown Scheme"); - } - if (uri.getHost() == null) { - throw new IllegalArgumentException("URI does not have a Host name"); - } - - ClientBootstrap clientBootstrap = options.getClientBootstrap(); - if (clientBootstrap == null) { - throw new IllegalArgumentException("ClientBootstrap must not be null"); - } - - SocketOptions socketOptions = options.getSocketOptions(); - if (socketOptions == null) { - throw new IllegalArgumentException("SocketOptions must not be null"); - } + private Http2StreamManager(Http2StreamManagerOptions options) { + options.validateOptions(); + HttpClientConnectionManagerOptions connectionManagerOptions = options.getConnectionManagerOptions(); + URI uri = connectionManagerOptions.getUri(); + ClientBootstrap clientBootstrap = connectionManagerOptions.getClientBootstrap(); + SocketOptions socketOptions = connectionManagerOptions.getSocketOptions(); boolean useTls = HTTPS.equals(uri.getScheme()); - TlsContext tlsContext = options.getTlsContext(); - if (useTls && tlsContext == null) { - throw new IllegalArgumentException("TlsContext must not be null if https is used"); - } - - - int maxConnections = options.getMaxConnections(); - if (maxConnections <= 0) { - throw new IllegalArgumentException("Max Connections must be greater than zero."); - } - int maxConcurrentStreamsPerConnection = options.getMaxConcurrentStreamsPerConnection(); - if (maxConcurrentStreamsPerConnection <= 0) { - throw new IllegalArgumentException("Max Concurrent Streams Per Connection must be greater than zero."); - } - int idealConcurrentStreamsPerConnection = options.getIdealConcurrentStreamsPerConnection(); - if (idealConcurrentStreamsPerConnection <= 0 - || idealConcurrentStreamsPerConnection > maxConcurrentStreamsPerConnection) { - throw new IllegalArgumentException( - "Ideal Concurrent Streams Per Connection must be greater than zero and smaller than max."); - } - - int port = options.getPort(); + TlsContext tlsContext = connectionManagerOptions.getTlsContext(); + int maxConnections = connectionManagerOptions.getMaxConnections(); + int port = connectionManagerOptions.getPort(); if (port == -1) { port = uri.getPort(); /* Pick a default port based on the scheme if one wasn't set */ if (port == -1) { - if (HTTP.equals(uri.getScheme())) { - port = DEFAULT_HTTP_PORT; - } - if (HTTPS.equals(uri.getScheme())) { - port = DEFAULT_HTTPS_PORT; - } + if (HTTP.equals(uri.getScheme())) { port = DEFAULT_HTTP_PORT; } + if (HTTPS.equals(uri.getScheme())) { port = DEFAULT_HTTPS_PORT; } } } - HttpProxyOptions proxyOptions = options.getProxyOptions(); + int maxConcurrentStreamsPerConnection = options.getMaxConcurrentStreamsPerConnection(); + int idealConcurrentStreamsPerConnection = options.getIdealConcurrentStreamsPerConnection(); this.uri = uri; this.port = port; @@ -116,6 +76,7 @@ private Http2StreamManager(HttpStreamManagerOptions options) { int proxyAuthorizationType = 0; String proxyAuthorizationUsername = null; String proxyAuthorizationPassword = null; + HttpProxyOptions proxyOptions = connectionManagerOptions.getProxyOptions(); if (proxyOptions != null) { proxyConnectionType = proxyOptions.getConnectionType().getValue(); @@ -127,7 +88,7 @@ private Http2StreamManager(HttpStreamManagerOptions options) { proxyAuthorizationPassword = proxyOptions.getAuthorizationPassword(); } - HttpMonitoringOptions monitoringOptions = options.getMonitoringOptions(); + HttpMonitoringOptions monitoringOptions = connectionManagerOptions.getMonitoringOptions(); long monitoringThroughputThresholdInBytesPerSecond = 0; int monitoringFailureIntervalInSeconds = 0; if (monitoringOptions != null) { @@ -149,7 +110,7 @@ private Http2StreamManager(HttpStreamManagerOptions options) { proxyAuthorizationType, proxyAuthorizationUsername != null ? proxyAuthorizationUsername.getBytes(UTF8) : null, proxyAuthorizationPassword != null ? proxyAuthorizationPassword.getBytes(UTF8) : null, - options.isManualWindowManagement(), + connectionManagerOptions.isManualWindowManagement(), monitoringThroughputThresholdInBytesPerSecond, monitoringFailureIntervalInSeconds, maxConnections, @@ -176,17 +137,20 @@ private Http2StreamManager(HttpStreamManagerOptions options) { * acquired. */ public CompletableFuture acquireStream(Http2Request request, - HttpStreamBaseResponseHandler streamHandler) { + HttpStreamBaseResponseHandler streamHandler) { + return this.acquireStream((HttpRequestBase) request, streamHandler); } public CompletableFuture acquireStream(HttpRequest request, - HttpStreamBaseResponseHandler streamHandler) { + HttpStreamBaseResponseHandler streamHandler) { + return this.acquireStream((HttpRequestBase) request, streamHandler); } private CompletableFuture acquireStream(HttpRequestBase request, - HttpStreamBaseResponseHandler streamHandler) { + HttpStreamBaseResponseHandler streamHandler) { + CompletableFuture completionFuture = new CompletableFuture<>(); AsyncCallback acquireStreamCompleted = AsyncCallback.wrapFuture(completionFuture, null); if (isNull()) { diff --git a/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java new file mode 100644 index 000000000..d0c8c985d --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/http/Http2StreamManagerOptions.java @@ -0,0 +1,159 @@ +package software.amazon.awssdk.crt.http; + +import java.util.List; +import java.util.ArrayList; + +/** + * Contains all the configuration options for a Http2StreamManager + * instance + */ +public class Http2StreamManagerOptions { + public static final int DEFAULT_MAX_WINDOW_SIZE = Integer.MAX_VALUE; + public static final int DEFAULT_MAX = Integer.MAX_VALUE; + public static final int DEFAULT_MAX_CONNECTIONS = 2; + + private HttpClientConnectionManagerOptions connectionManagerOptions; + + private int idealConcurrentStreamsPerConnection = 100; + private boolean connectionManualWindowManagement = false; + private int maxConcurrentStreamsPerConnection = DEFAULT_MAX; + + private List initialSettingsList = new ArrayList(); + + /** + * Default constructor + */ + public Http2StreamManagerOptions() { + } + + /** + * For HTTP/2 stream manager only. + * + * The initial settings for the HTTP/2 connections made by stream manger. + * `Http2ConnectionSettingListBuilder` can help to build the settings list. + * + * To control the initial stream-level flow-control window, set the INITIAL_WINDOW_SIZE setting in the initial settings. + * + * @param initialSettingsList The List of initial settings + * @return this + */ + public Http2StreamManagerOptions withInitialSettingsList(List initialSettingsList) { + this.initialSettingsList.addAll(initialSettingsList); + return this; + } + + /** + * @return The List of initial settings + */ + public List getInitialSettingsList() { + return this.initialSettingsList; + } + + /** + * For HTTP/2 stream manager only. + * + * The ideal number of concurrent streams for a connection. Stream manager will + * try to create a new connection if one connection reaches this number. But, if + * the max connections reaches, manager will reuse connections to create the + * acquired steams as much as possible. + * + * @param idealConcurrentStreamsPerConnection The ideal number of concurrent + * streams for a connection + * @return this + */ + public Http2StreamManagerOptions withIdealConcurrentStreamsPerConnection(int idealConcurrentStreamsPerConnection) { + this.idealConcurrentStreamsPerConnection = idealConcurrentStreamsPerConnection; + return this; + } + + /** + * @return The ideal number of concurrent streams for a connection used for + * manager + */ + public int getIdealConcurrentStreamsPerConnection() { + return idealConcurrentStreamsPerConnection; + } + + /** + * Default is no limit, which will use the limit from the server. 0 will be + * considered as using the default value. + * The real number of concurrent streams per connection will be controlled by + * the minimal value of the setting from other end and the value here. + * + * @param maxConcurrentStreamsPerConnection The max number of concurrent + * streams for a connection + * @return this + */ + public Http2StreamManagerOptions withMaxConcurrentStreamsPerConnection(int maxConcurrentStreamsPerConnection) { + this.maxConcurrentStreamsPerConnection = maxConcurrentStreamsPerConnection; + return this; + } + + /** + * @return The max number of concurrent streams for a connection set for + * manager. + * It could be different than the real limits, which is the minimal set + * for manager and the settings from the other side. + */ + public int getMaxConcurrentStreamsPerConnection() { + return maxConcurrentStreamsPerConnection; + } + + /** + * @return The connection level manual flow control enabled or not. + */ + public boolean isConnectionManualWindowManagement() { + return connectionManualWindowManagement; + } + + /** + * Set to true to manually manage the flow-control window of whole HTTP/2 connection. + * The stream level flow-control window is controlled by the manualWindowManagement in connectionManagerOptions. + * + * @param connectionManualWindowManagement Enable connection level manual flow control or not. + * @return this + */ + public Http2StreamManagerOptions withConnectionManualWindowManagement(boolean connectionManualWindowManagement) { + this.connectionManualWindowManagement = connectionManualWindowManagement; + return this; + } + + /** + * @return The connection manager options for the underlying connection manager. + */ + public HttpClientConnectionManagerOptions getConnectionManagerOptions() { + return connectionManagerOptions; + } + + /** + * The configuration options for the connection manager under the hood. + * It controls the connection specific thing for the stream manager. See `HttpClientConnectionManagerOptions` for details. + * + * Note: + * 1. the windowSize of connection manager will be ignored, as the initial flow-control window size for HTTP/2 stream + * is controlled by the initial settings. + * 2. The expectedHttpVersion will also be ignored. + * + * @param connectionManagerOptions The connection manager options for the underlying connection manager + * @return this + */ + public Http2StreamManagerOptions withConnectionManagerOptions(HttpClientConnectionManagerOptions connectionManagerOptions) { + this.connectionManagerOptions = connectionManagerOptions; + return this; + } + + /** + * Validate the stream manager options are valid to use. Throw exceptions if not. + */ + public void validateOptions() { + connectionManagerOptions.validateOptions(); + if (maxConcurrentStreamsPerConnection <= 0) { + throw new IllegalArgumentException("Max Concurrent Streams Per Connection must be greater than zero."); + } + if (idealConcurrentStreamsPerConnection <= 0 + || idealConcurrentStreamsPerConnection > maxConcurrentStreamsPerConnection) { + throw new IllegalArgumentException( + "Ideal Concurrent Streams Per Connection must be greater than zero and smaller than max."); + } + } +} diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java index 7b4705aaf..666f5ac21 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManager.java @@ -45,28 +45,15 @@ public static HttpClientConnectionManager create(HttpClientConnectionManagerOpti } private HttpClientConnectionManager(HttpClientConnectionManagerOptions options) { - URI uri = options.getUri(); - if (uri == null) { throw new IllegalArgumentException("URI must not be null"); } - if (uri.getScheme() == null) { throw new IllegalArgumentException("URI does not have a Scheme"); } - if (!HTTP.equals(uri.getScheme()) && !HTTPS.equals(uri.getScheme())) { throw new IllegalArgumentException("URI has unknown Scheme"); } - if (uri.getHost() == null) { throw new IllegalArgumentException("URI does not have a Host name"); } + options.validateOptions(); + URI uri = options.getUri(); ClientBootstrap clientBootstrap = options.getClientBootstrap(); - if (clientBootstrap == null) { throw new IllegalArgumentException("ClientBootstrap must not be null"); } - SocketOptions socketOptions = options.getSocketOptions(); - if (socketOptions == null) { throw new IllegalArgumentException("SocketOptions must not be null"); } - boolean useTls = HTTPS.equals(uri.getScheme()); TlsContext tlsContext = options.getTlsContext(); - if (useTls && tlsContext == null) { throw new IllegalArgumentException("TlsContext must not be null if https is used"); } - int windowSize = options.getWindowSize(); - if (windowSize <= 0) { throw new IllegalArgumentException("Window Size must be greater than zero."); } - int maxConnections = options.getMaxConnections(); - if (maxConnections <= 0) { throw new IllegalArgumentException("Max Connections must be greater than zero."); } - int port = options.getPort(); if (port == -1) { port = uri.getPort(); diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java index ea7344c3a..1766bee7a 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpClientConnectionManagerOptions.java @@ -13,6 +13,7 @@ * Contains all the configuration options for a HttpConnectionPoolManager instance */ public class HttpClientConnectionManagerOptions { + public static final int DEFAULT_MAX_BUFFER_SIZE = 16 * 1024; public static final int DEFAULT_MAX_WINDOW_SIZE = Integer.MAX_VALUE; public static final int DEFAULT_MAX_CONNECTIONS = 2; @@ -20,6 +21,7 @@ public class HttpClientConnectionManagerOptions { private SocketOptions socketOptions; private TlsContext tlsContext; private int windowSize = DEFAULT_MAX_WINDOW_SIZE; + private int bufferSize = DEFAULT_MAX_BUFFER_SIZE; private URI uri; private int port = -1; private int maxConnections = DEFAULT_MAX_CONNECTIONS; @@ -29,6 +31,9 @@ public class HttpClientConnectionManagerOptions { private long maxConnectionIdleInMilliseconds = 0; private HttpVersion expectedHttpVersion = HttpVersion.HTTP_1_1; + private static final String HTTP = "http"; + private static final String HTTPS = "https"; + /** * Default constructor */ @@ -96,6 +101,22 @@ public HttpClientConnectionManagerOptions withWindowSize(int windowSize) { */ public int getWindowSize() { return windowSize; } + /** + * @deprecated Sets the IO buffer size to use for connections in the connection pool + * @param bufferSize Size of I/O buffer per connection + * @return this + */ + public HttpClientConnectionManagerOptions withBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + return this; + } + + /** + * @deprecated + * @return the IO buffer size to use for connections in the connection pool + */ + public int getBufferSize() { return bufferSize; } + /** * Sets the URI to use for connections in the connection pool * @param uri The endpoint URI to connect to @@ -231,4 +252,26 @@ public HttpClientConnectionManagerOptions withMonitoringOptions(HttpMonitoringOp * @return the monitoring options for connections in the connection pool */ public HttpMonitoringOptions getMonitoringOptions() { return monitoringOptions; } + + /** + * Validate the connection manager options are valid to use. Throw exceptions if not. + */ + public void validateOptions() { + URI uri = this.getUri(); + if (uri == null) { throw new IllegalArgumentException("URI must not be null"); } + if (uri.getScheme() == null) { throw new IllegalArgumentException("URI does not have a Scheme"); } + if (!HTTP.equals(uri.getScheme()) && !HTTPS.equals(uri.getScheme())) { throw new IllegalArgumentException("URI has unknown Scheme"); } + if (uri.getHost() == null) { throw new IllegalArgumentException("URI does not have a Host name"); } + + if (clientBootstrap == null) { throw new IllegalArgumentException("ClientBootstrap must not be null"); } + + if (socketOptions == null) { throw new IllegalArgumentException("SocketOptions must not be null"); } + + boolean useTls = HTTPS.equals(uri.getScheme()); + if (useTls && tlsContext == null) { throw new IllegalArgumentException("TlsContext must not be null if https is used"); } + + if (windowSize <= 0) { throw new IllegalArgumentException("Window Size must be greater than zero."); } + + if (maxConnections <= 0) { throw new IllegalArgumentException("Max Connections must be greater than zero."); } + } } diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java deleted file mode 100644 index 879e12afb..000000000 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManagerOptions.java +++ /dev/null @@ -1,265 +0,0 @@ -package software.amazon.awssdk.crt.http; - -import java.net.URI; -import java.util.List; -import java.util.ArrayList; -import software.amazon.awssdk.crt.io.ClientBootstrap; -import software.amazon.awssdk.crt.io.SocketOptions; -import software.amazon.awssdk.crt.io.TlsContext; - -/** - * Contains all the configuration options for a Http2StreamManager - * instance - */ -public class HttpStreamManagerOptions { - public static final int DEFAULT_MAX_WINDOW_SIZE = Integer.MAX_VALUE; - public static final int DEFAULT_MAX = Integer.MAX_VALUE; - public static final int DEFAULT_MAX_CONNECTIONS = 2; - - private ClientBootstrap clientBootstrap; - private SocketOptions socketOptions; - private TlsContext tlsContext; - private URI uri; - private int port = -1; - private boolean manualWindowManagement = false; - private HttpMonitoringOptions monitoringOptions; - private HttpProxyOptions proxyOptions; - - private int idealConcurrentStreamsPerConnection = 100; - private int maxConcurrentStreamsPerConnection = DEFAULT_MAX; - - private int maxConnections = DEFAULT_MAX_CONNECTIONS; - - private List initialSettingsList = new ArrayList(); - - /** - * Default constructor - */ - public HttpStreamManagerOptions() { - } - - /** - * Sets the client bootstrap instance to use to create the pool's connections - * - * @param clientBootstrap ClientBootstrap to use - * @return this - */ - public HttpStreamManagerOptions withClientBootstrap(ClientBootstrap clientBootstrap) { - this.clientBootstrap = clientBootstrap; - return this; - } - - /** - * Gets the client bootstrap instance to use to create the pool's connections - * - * @return ClientBootstrap used by this connection manager - */ - public ClientBootstrap getClientBootstrap() { - return clientBootstrap; - } - - /** - * Sets the socket options to use for connections in the connection pool - * - * @param socketOptions The socket options to use for all connections in the - * manager - * @return this - */ - public HttpStreamManagerOptions withSocketOptions(SocketOptions socketOptions) { - this.socketOptions = socketOptions; - return this; - } - - /** - * @return the socket options to use for connections in the connection pool - */ - public SocketOptions getSocketOptions() { - return socketOptions; - } - - /** - * Sets the tls context to use for connections in the connection pool - * - * @param tlsContext The TlsContext to use - * @return this - */ - public HttpStreamManagerOptions withTlsContext(TlsContext tlsContext) { - this.tlsContext = tlsContext; - return this; - } - - /** - * @return the tls context used by connections in the connection pool - */ - public TlsContext getTlsContext() { - return tlsContext; - } - - /** - * For HTTP/2 stream manager only. - * - * The initial settings for the HTTP/2 connections made by stream manger. - * `Http2ConnectionSettingListBuilder` can help to build the settings list - * - * @param initialSettingsList The List of initial settings - * @return this - */ - public HttpStreamManagerOptions withInitialSettingsList(List initialSettingsList) { - this.initialSettingsList.addAll(initialSettingsList); - return this; - } - - /** - * @return The List of initial settings - */ - public List getInitialSettingsList() { - return this.initialSettingsList; - } - - /** - * For HTTP/2 stream manager only. - * - * The ideal number of concurrent streams for a connection. Stream manager will - * try to create a new connection if one connection reaches this number. But, if - * the max connections reaches, manager will reuse connections to create the - * acquired steams as much as possible. - * - * @param idealConcurrentStreamsPerConnection The ideal number of concurrent - * streams for a connection - * @return this - */ - public HttpStreamManagerOptions withIdealConcurrentStreamsPerConnection(int idealConcurrentStreamsPerConnection) { - this.idealConcurrentStreamsPerConnection = idealConcurrentStreamsPerConnection; - return this; - } - - /** - * @return The ideal number of concurrent streams for a connection used for - * manager - */ - public int getIdealConcurrentStreamsPerConnection() { - return idealConcurrentStreamsPerConnection; - } - - /** - * Default is no limit, which will use the limit from the server. 0 will be - * considered as using the default value. - * The real number of concurrent streams per connection will be controlled by - * the minimal value of the setting from other end and the value here. - * - * @param maxConcurrentStreamsPerConnection The max number of concurrent - * streams for a connection - * @return this - */ - public HttpStreamManagerOptions withMaxConcurrentStreamsPerConnection(int maxConcurrentStreamsPerConnection) { - this.maxConcurrentStreamsPerConnection = maxConcurrentStreamsPerConnection; - return this; - } - - /** - * @return The max number of concurrent streams for a connection set for - * manager. - * It could be different than the real limits, which is the minimal set - * for manager and the settings from the other side. - */ - public int getMaxConcurrentStreamsPerConnection() { - return maxConcurrentStreamsPerConnection; - } - - /** - * Sets the URI to use for connections in the connection pool - * - * @param uri The endpoint URI to connect to - * @return this - */ - public HttpStreamManagerOptions withUri(URI uri) { - this.uri = uri; - return this; - } - - /** - * @return the URI to use for connections in the connection pool - */ - public URI getUri() { - return uri; - } - - /** - * Sets the port to connect to for connections in the connection pool - * - * @param port The port to connect to - * @return this - */ - public HttpStreamManagerOptions withPort(int port) { - this.port = port; - return this; - } - - /** - * @return the port to connect to for connections in the connection pool. - * Returns -1 if none has been explicitly set. - */ - public int getPort() { - return port; - } - - /** - * The max number of connections will be open at same time. If all the - * connections are full, manager will wait until available to vender more - * streams - * - * @param maxConnections The max number of connections will be open at same time. - * @return this - */ - public HttpStreamManagerOptions withMaxConnections(int maxConnections) { - this.maxConnections = maxConnections; - return this; - } - - /** - * @return the maximum number of connections allowed in the connection pool - */ - public int getMaxConnections() { - return maxConnections; - } - - public HttpStreamManagerOptions withProxyOptions(HttpProxyOptions proxyOptions) { - this.proxyOptions = proxyOptions; - return this; - } - - /** - * @return the proxy options for connections in the connection pool - */ - public HttpProxyOptions getProxyOptions() { - return proxyOptions; - } - - public boolean isManualWindowManagement() { - return manualWindowManagement; - } - - public HttpStreamManagerOptions withManualWindowManagement(boolean manualWindowManagement) { - this.manualWindowManagement = manualWindowManagement; - return this; - } - - /** - * Sets the monitoring options for connections in the connection pool - * - * @param monitoringOptions Monitoring options for this connection manager, or - * null to disable monitoring - * @return this - */ - public HttpStreamManagerOptions withMonitoringOptions(HttpMonitoringOptions monitoringOptions) { - this.monitoringOptions = monitoringOptions; - return this; - } - - /** - * @return the monitoring options for connections in the connection pool - */ - public HttpMonitoringOptions getMonitoringOptions() { - return monitoringOptions; - } -} diff --git a/src/native/http2_stream_manager.c b/src/native/http2_stream_manager.c index 933c1ce2f..92c0e88e4 100644 --- a/src/native/http2_stream_manager.c +++ b/src/native/http2_stream_manager.c @@ -339,7 +339,7 @@ JNIEXPORT void JNICALL Java_software_amazon_awssdk_crt_http_Http2StreamManager_h struct aws_http2_stream_manager *stream_manager = sm_binding->stream_manager; if (!stream_manager) { - aws_jni_throw_runtime_exception(env, "Stream Manager can't be null"); + aws_jni_throw_illegal_argument_exception(env, "Stream Manager can't be null"); return; } diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java index d7173dcc4..fbee5ae2b 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java @@ -19,7 +19,7 @@ import software.amazon.awssdk.crt.http.Http2StreamManager; import software.amazon.awssdk.crt.http.Http2Request; import software.amazon.awssdk.crt.http.Http2Stream; -import software.amazon.awssdk.crt.http.HttpStreamManagerOptions; +import software.amazon.awssdk.crt.http.Http2StreamManagerOptions; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; @@ -56,12 +56,14 @@ private Http2StreamManager createStreamManager(URI uri, int numConnections) { SocketOptions sockOpts = new SocketOptions(); TlsContextOptions tlsOpts = TlsContextOptions.createDefaultClient().withAlpnList("h2"); TlsContext tlsContext = createHttpClientTlsContext(tlsOpts)) { - HttpStreamManagerOptions options = new HttpStreamManagerOptions(); - options.withClientBootstrap(bootstrap) + Http2StreamManagerOptions options = new Http2StreamManagerOptions(); + HttpClientConnectionManagerOptions connectionManagerOptions = new HttpClientConnectionManagerOptions(); + connectionManagerOptions.withClientBootstrap(bootstrap) .withSocketOptions(sockOpts) .withTlsContext(tlsContext) .withUri(uri) .withMaxConnections(numConnections); + options.withConnectionManagerOptions(connectionManagerOptions); return Http2StreamManager.create(options); } From 28d7417ceb6ca6cf2642687cd2345855794a4601 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 21 Jun 2022 10:50:19 -0700 Subject: [PATCH 141/142] Localhost test (#490) --- .builder/actions/localhost_test.py | 19 + .github/workflows/ci.yml | 57 +++ crt/aws-c-http | 2 +- src/native/http_request_response.h | 2 +- .../awssdk/crt/test/CrtTestFixture.java | 4 + .../crt/test/Http2ClientLocalHostTest.java | 346 ++++++++++++++++++ .../crt/test/Http2StreamManagerTest.java | 6 +- 7 files changed, 433 insertions(+), 3 deletions(-) create mode 100644 .builder/actions/localhost_test.py create mode 100644 src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py new file mode 100644 index 000000000..0dd32b385 --- /dev/null +++ b/.builder/actions/localhost_test.py @@ -0,0 +1,19 @@ +import Builder +import sys +import os + + +class LocalhostTest(Builder.Action): + + def run(self, env): + env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2') + actions = [] + if os.system("mvn -Dtest=Http2ClientLocalHostTest test -DredirectTestOutputToFile=true -DforkCount=0 \ + -DrerunFailingTestsCount=5 \ + -Daws.crt.memory.tracing=2 \ + -Daws.crt.debugnative=true \ + -Daws.crt.localhost=true"): + # Failed + actions.append("exit 1") + + return Builder.Script(actions, name='aws-crt-java-test') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb59ab1aa..68a24e898 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,3 +182,60 @@ jobs: # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work # https://github.com/actions/runner/issues/480 uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main + + + localhost-test-linux: + runs-on: ubuntu-20.04 # latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + - name: Configure local host + run: | + python3 -m pip install h2 + ls crt/aws-c-http/ + cd crt/aws-c-http/tests/py_localhost/ + python3 server.py & + python3 non_tls_server.py & + - name: Build and test + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} --spec=downstream + + localhost-test-mac: + runs-on: macos-11 # latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + - name: Configure local host + run: | + python3 -m pip install h2 + cd crt/aws-c-http/tests/py_localhost/ + python3 server.py & + python3 non_tls_server.py & + - name: Build and test + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder localhost-test -p ${{ env.PACKAGE_NAME }} --spec=downstream + + localhost-test-win: + runs-on: windows-2022 # latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + - name: Configure local host + run: | + python -m pip install h2 + - name: Build and test + run: | + cd crt/aws-c-http/tests/py_localhost/ + Start-Process -NoNewWindow python .\server.py + Start-Process -NoNewWindow python .\non_tls_server.py + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} downstream diff --git a/crt/aws-c-http b/crt/aws-c-http index 73af2aa20..aadc57a38 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 73af2aa20d220dfbb26f22ef15df9651583cd5cb +Subproject commit aadc57a3829cb057acdd6d57ed0991ff61a71192 diff --git a/src/native/http_request_response.h b/src/native/http_request_response.h index 528c14148..057200442 100644 --- a/src/native/http_request_response.h +++ b/src/native/http_request_response.h @@ -28,7 +28,7 @@ struct http_stream_binding { int response_status; /* - * Unactivated streams must have their callback data destroyed at release time + * Inactivated streams must have their callback data destroyed at release time */ struct aws_atomic_var activated; }; diff --git a/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java index 3dc46572b..125a682a0 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java +++ b/src/test/java/software/amazon/awssdk/crt/test/CrtTestFixture.java @@ -109,4 +109,8 @@ protected boolean hasAwsCredentials() { protected void skipIfNetworkUnavailable() { Assume.assumeTrue(System.getProperty("NETWORK_TESTS_DISABLED") == null); } + + protected void skipIfLocalhostUnavailable() { + Assume.assumeTrue(System.getProperty("aws.crt.localhost") != null); + } } diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java new file mode 100644 index 000000000..f94bf46cc --- /dev/null +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java @@ -0,0 +1,346 @@ + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package software.amazon.awssdk.crt.test; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.http.Http2StreamManager; +import software.amazon.awssdk.crt.http.Http2Request; +import software.amazon.awssdk.crt.http.Http2Stream; +import software.amazon.awssdk.crt.http.Http2StreamManagerOptions; +import software.amazon.awssdk.crt.http.HttpClientConnection; +import software.amazon.awssdk.crt.http.HttpClientConnectionManager; +import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpProxyOptions; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpStreamBaseResponseHandler; +import software.amazon.awssdk.crt.http.HttpStreamBase; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.crt.io.EventLoopGroup; +import software.amazon.awssdk.crt.io.HostResolver; +import software.amazon.awssdk.crt.io.SocketOptions; +import software.amazon.awssdk.crt.io.TlsContext; +import software.amazon.awssdk.crt.io.TlsContextOptions; +import software.amazon.awssdk.crt.utils.ByteBufferUtils; +import software.amazon.awssdk.crt.Log; + +public class Http2ClientLocalHostTest extends HttpClientTestFixture { + + private Http2StreamManager createStreamManager(URI uri, int numConnections) { + + try (EventLoopGroup eventLoopGroup = new EventLoopGroup(1); + HostResolver resolver = new HostResolver(eventLoopGroup); + ClientBootstrap bootstrap = new ClientBootstrap(eventLoopGroup, resolver); + SocketOptions sockOpts = new SocketOptions(); + TlsContextOptions tlsOpts = TlsContextOptions.createDefaultClient().withAlpnList("h2") + .withVerifyPeer(false); + TlsContext tlsContext = createHttpClientTlsContext(tlsOpts)) { + Http2StreamManagerOptions options = new Http2StreamManagerOptions(); + HttpClientConnectionManagerOptions connectionManagerOptions = new HttpClientConnectionManagerOptions(); + connectionManagerOptions.withClientBootstrap(bootstrap) + .withSocketOptions(sockOpts) + .withTlsContext(tlsContext) + .withUri(uri) + .withMaxConnections(numConnections); + options.withConnectionManagerOptions(connectionManagerOptions); + + return Http2StreamManager.create(options); + } + } + + private HttpRequestBodyStream createBodyStreamWithLength(long bodyLength) { + final long payloadSize = bodyLength; + final String payloadString = "This is CRT HTTP test."; + + HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() { + + private long remainingBody = payloadSize; + + @Override + public boolean sendRequestBody(ByteBuffer outBuffer) { + + byte[] payloadBytes = null; + + try { + payloadBytes = payloadString.getBytes("ASCII"); + } catch (Exception ex) { + System.out.println("Encountered error trying to get payload bytes."); + return true; + } + + while (remainingBody > 0 && outBuffer.remaining() > 0) { + long amtToTransfer = Math.min(remainingBody, (long) outBuffer.remaining()); + amtToTransfer = Math.min(amtToTransfer, (long) payloadBytes.length); + + // Transfer the data + outBuffer.put(payloadBytes, 0, (int) amtToTransfer); + + remainingBody -= amtToTransfer; + } + + return remainingBody == 0; + } + + @Override + public boolean resetPosition() { + return true; + } + + @Override + public long getLength() { + return payloadSize; + } + }; + return payloadStream; + } + + private Http2Request createHttp2Request(String method, URI uri, long bodyLength) { + HttpHeader[] requestHeaders = new HttpHeader[] { + new HttpHeader(":method", method), + new HttpHeader(":path", uri.getPath()), + new HttpHeader(":scheme", uri.getScheme()), + new HttpHeader(":authority", uri.getHost()), + new HttpHeader("content-length", Long.toString(bodyLength)) + }; + HttpRequestBodyStream bodyStream = null; + if (bodyLength > 0) { + bodyStream = createBodyStreamWithLength(bodyLength); + } + Http2Request request = new Http2Request(requestHeaders, bodyStream); + + return request; + } + + @Test + public void testParallelRequestsStress() throws Exception { + skipIfLocalhostUnavailable(); + URI uri = new URI("https://localhost:8443/echo"); + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { + int numberToAcquire = 500 * 100; + + Http2Request request = createHttp2Request("GET", uri, 0); + List> requestCompleteFutures = new ArrayList<>(); + List> acquireCompleteFutures = new ArrayList<>(); + final AtomicInteger numStreamsFailures = new AtomicInteger(0); + for (int i = 0; i < numberToAcquire; i++) { + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + requestCompleteFutures.add(requestCompleteFuture); + acquireCompleteFutures.add(streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + if (responseStatusCode != 200) { + numStreamsFailures.incrementAndGet(); + } + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + if (errorCode != CRT.AWS_CRT_SUCCESS) { + numStreamsFailures.incrementAndGet(); + } + stream.close(); + requestCompleteFuture.complete(null); + } + })); + } + for (CompletableFuture f : acquireCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + // Wait for all Requests to complete + for (CompletableFuture f : requestCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + Assert.assertTrue(numStreamsFailures.get() == 0); + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testParallelRequestsStressWithBody() throws Exception { + skipIfLocalhostUnavailable(); + URI uri = new URI("https://localhost:8443/uploadTest"); + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { + int numberToAcquire = 500 * 100; + if (CRT.getOSIdentifier() == "linux") { + /* + * Using Python hyper h2 server frame work, met a weird upload performance issue + * on Linux. Our client against nginx platform has not met the same issue. + * We assume it's because the server framework implementation. + * Use lower number of linux + */ + numberToAcquire = 500; + } + int bodyLength = 2000; + + List> requestCompleteFutures = new ArrayList<>(); + List> acquireCompleteFutures = new ArrayList<>(); + final AtomicInteger numStreamsFailures = new AtomicInteger(0); + for (int i = 0; i < numberToAcquire; i++) { + Http2Request request = createHttp2Request("PUT", uri, bodyLength); + + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + final int expectedLength = bodyLength; + requestCompleteFutures.add(requestCompleteFuture); + acquireCompleteFutures.add(streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + if (responseStatusCode != 200) { + numStreamsFailures.incrementAndGet(); + } + } + + @Override + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ + String bodyString = new String(bodyBytesIn); + int receivedLength = Integer.parseInt(bodyString); + + Assert.assertTrue(receivedLength == expectedLength); + if(receivedLength!=expectedLength) { + numStreamsFailures.incrementAndGet(); + } + return bodyString.length(); + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + if (errorCode != CRT.AWS_CRT_SUCCESS) { + numStreamsFailures.incrementAndGet(); + } + stream.close(); + requestCompleteFuture.complete(null); + } + })); + } + for (CompletableFuture f : acquireCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + // Wait for all Requests to complete + for (CompletableFuture f : requestCompleteFutures) { + f.get(30, TimeUnit.SECONDS); + } + Assert.assertTrue(numStreamsFailures.get() == 0); + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testRequestsUploadStress() throws Exception { + /* Test that upload a 2.5GB data from local server (0.25GB for linux) */ + skipIfLocalhostUnavailable(); +// Log.initLoggingToStderr(Log.LogLevel.Debug); + URI uri = new URI("https://localhost:8443/uploadTest"); + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { + long bodyLength = 2500000000L; + if (CRT.getOSIdentifier() == "linux") { + /* + * Using Python hyper h2 server frame work, met a weird upload performance issue + * on Linux. Our client against nginx platform has not met the same issue. + * We assume it's because the server framework implementation. + * Use lower number of linux + */ + bodyLength = 250000000L; + } + + Http2Request request = createHttp2Request("PUT", uri, bodyLength); + + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + final long expectedLength = bodyLength; + CompletableFuture acquireCompleteFuture = streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + + Assert.assertTrue(responseStatusCode == 200); + } + + @Override + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ + String bodyString = new String(bodyBytesIn); + long receivedLength = Long.parseLong(bodyString); + Assert.assertTrue(receivedLength == expectedLength); + return bodyString.length(); + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + Assert.assertTrue(errorCode == CRT.AWS_CRT_SUCCESS); + stream.close(); + requestCompleteFuture.complete(null); + } + }); + + acquireCompleteFuture.get(30, TimeUnit.SECONDS); + requestCompleteFuture.join(); + + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } + + @Test + public void testRequestsDownloadStress() throws Exception { + /* Test that download a 2.5GB data from local server */ + skipIfLocalhostUnavailable(); + URI uri = new URI("https://localhost:8443/downloadTest"); + try (Http2StreamManager streamManager = createStreamManager(uri, 100)) { + long bodyLength = 2500000000L; + + Http2Request request = createHttp2Request("GET", uri, 0); + + final CompletableFuture requestCompleteFuture = new CompletableFuture(); + final AtomicLong receivedLength = new AtomicLong(0); + CompletableFuture acquireCompleteFuture = streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() { + @Override + public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType, + HttpHeader[] nextHeaders) { + + Assert.assertTrue(responseStatusCode == 200); + } + + @Override + public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){ + receivedLength.addAndGet(bodyBytesIn.length); + + return bodyBytesIn.length; + } + + @Override + public void onResponseComplete(HttpStreamBase stream, int errorCode) { + + Assert.assertTrue(errorCode == CRT.AWS_CRT_SUCCESS); + stream.close(); + requestCompleteFuture.complete(null); + } + }); + + acquireCompleteFuture.get(30, TimeUnit.SECONDS); + requestCompleteFuture.join(); + + Assert.assertTrue(receivedLength.get() == bodyLength); + } + CrtResource.logNativeResources(); + CrtResource.waitForNoResources(); + } +} diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java index fbee5ae2b..b354c46e1 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/Http2StreamManagerTest.java @@ -1,3 +1,7 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ package software.amazon.awssdk.crt.test; import java.net.URI; @@ -141,7 +145,7 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) { // Verify we got some Http Status Code for each Request Assert.assertTrue(reqIdToStatus.size() >= requiredSuccesses); - // Verify that the failure counts aren't too high ????? + // Verify that the failure counts aren't too high Assert.assertTrue(numErrorCode.get() <= allowedFailures); Assert.assertTrue(numStreamsFailures.get() <= allowedFailures); } From d2902ccca5740605aeb0ed138428f6ef8e7b6753 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 21 Jun 2022 13:23:12 -0700 Subject: [PATCH 142/142] adapt new api --- .../awssdk/crt/http/HttpStreamManager.java | 21 +++---------------- .../crt/test/HttpStreamManagerTest.java | 16 +++++++------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java index 94f98b9b7..a85ac6ef8 100644 --- a/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java +++ b/src/main/java/software/amazon/awssdk/crt/http/HttpStreamManager.java @@ -31,27 +31,12 @@ public class HttpStreamManager implements AutoCloseable { * @param options configuration options * @return a new instance of an HttpStreamManager */ - public static HttpStreamManager create(HttpStreamManagerOptions options) { + public static HttpStreamManager create(Http2StreamManagerOptions options) { return new HttpStreamManager(options); } - private HttpClientConnectionManagerOptions getConnManagerOptFromStreamManagerOpt(HttpStreamManagerOptions options) { - // TODO: split the HTTP/1 stream manager to its own thing - HttpClientConnectionManagerOptions connManagerOptions = new HttpClientConnectionManagerOptions(); - connManagerOptions.withClientBootstrap(options.getClientBootstrap()) - .withSocketOptions(options.getSocketOptions()) - .withTlsContext(options.getTlsContext()) - .withUri(options.getUri()) - .withMaxConnections(options.getMaxConnections()) - .withPort(options.getPort()) - .withProxyOptions(options.getProxyOptions()) - .withManualWindowManagement(options.isManualWindowManagement()) - .withMonitoringOptions(options.getMonitoringOptions()); - return connManagerOptions; - } - - private HttpStreamManager(HttpStreamManagerOptions options) { - HttpClientConnectionManagerOptions connManagerOptions = getConnManagerOptFromStreamManagerOpt(options); + private HttpStreamManager(Http2StreamManagerOptions options) { + HttpClientConnectionManagerOptions connManagerOptions = options.getConnectionManagerOptions(); this.connectionManager = HttpClientConnectionManager.create(connManagerOptions); this.h2StreamManager = Http2StreamManager.create(options); this.shutdownComplete = new CompletableFuture(); diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java index a81f388d9..46afaae65 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/HttpStreamManagerTest.java @@ -29,7 +29,7 @@ import software.amazon.awssdk.crt.http.Http2StreamManager; import software.amazon.awssdk.crt.http.Http2Request; import software.amazon.awssdk.crt.http.Http2Stream; -import software.amazon.awssdk.crt.http.HttpStreamManagerOptions; +import software.amazon.awssdk.crt.http.Http2StreamManagerOptions; import software.amazon.awssdk.crt.http.HttpClientConnection; import software.amazon.awssdk.crt.http.HttpClientConnectionManager; import software.amazon.awssdk.crt.http.HttpClientConnectionManagerOptions; @@ -71,12 +71,14 @@ private HttpStreamManager createStreamManager(URI uri, int numConnections, HttpV ? TlsContextOptions.createDefaultClient().withAlpnList("h2") : TlsContextOptions.createDefaultClient().withAlpnList("http/1.1"); TlsContext tlsContext = createHttpClientTlsContext(tlsOpts)) { - HttpStreamManagerOptions options = new HttpStreamManagerOptions(); - options.withClientBootstrap(bootstrap) - .withSocketOptions(sockOpts) - .withTlsContext(tlsContext) - .withUri(uri) - .withMaxConnections(numConnections); + Http2StreamManagerOptions options = new Http2StreamManagerOptions(); + HttpClientConnectionManagerOptions connectionManagerOptions = new HttpClientConnectionManagerOptions(); + connectionManagerOptions.withClientBootstrap(bootstrap) + .withSocketOptions(sockOpts) + .withTlsContext(tlsContext) + .withUri(uri) + .withMaxConnections(numConnections); + options.withConnectionManagerOptions(connectionManagerOptions); return HttpStreamManager.create(options); }