Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Api gateway response parsing #1903

Merged
merged 4 commits into from
Dec 20, 2024
Merged

Api gateway response parsing #1903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8cfbc0a
Add api gateway extensions
gcbeattyAWS Dec 10, 2024
b4cd801
First set of PR comments
gcbeattyAWS Dec 14, 2024
7e88575
PR comments
gcbeattyAWS Dec 18, 2024
66a0c9c
PR comments
gcbeattyAWS Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions Tools/LambdaTestTool-v2/Amazon.Lambda.TestTool.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.12.35527.113
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Amazon.Lambda.TestTool", "src\Amazon.Lambda.TestTool\Amazon.Lambda.TestTool.csproj", "{97EE2E8A-D1F4-CB11-B664-B99B036E9F7B}"
Expand All @@ -8,6 +11,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Amazon.Lambda.TestTool.UnitTests", "tests\Amazon.Lambda.TestTool.UnitTests\Amazon.Lambda.TestTool.UnitTests.csproj", "{80A4F809-28B7-61EC-6539-DF3C7A0733FD}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Amazon.Lambda.TestTool.IntegrationTests", "tests\Amazon.Lambda.TestTool.IntegrationTests\Amazon.Lambda.TestTool.IntegrationTests.csproj", "{5C1B3E1C-DFEA-425B-8ED2-BB43BAECC3CB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -22,6 +27,13 @@ Global
{80A4F809-28B7-61EC-6539-DF3C7A0733FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{80A4F809-28B7-61EC-6539-DF3C7A0733FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{80A4F809-28B7-61EC-6539-DF3C7A0733FD}.Release|Any CPU.Build.0 = Release|Any CPU
{5C1B3E1C-DFEA-425B-8ED2-BB43BAECC3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5C1B3E1C-DFEA-425B-8ED2-BB43BAECC3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5C1B3E1C-DFEA-425B-8ED2-BB43BAECC3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5C1B3E1C-DFEA-425B-8ED2-BB43BAECC3CB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{97EE2E8A-D1F4-CB11-B664-B99B036E9F7B} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
Expand Down
7 changes: 4 additions & 3 deletions Tools/LambdaTestTool-v2/Amazon.Lambda.TestTool.slnx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
<Solution>
<Solution>
<Folder Name="/src/">
<Project Path="src\Amazon.Lambda.TestTool\Amazon.Lambda.TestTool.csproj" Type="Classic C#" />
<Project Path="src\Amazon.Lambda.TestTool\Amazon.Lambda.TestTool.csproj" />
</Folder>
<Folder Name="/tests/">
<Project Path="tests\Amazon.Lambda.TestTool.UnitTests\Amazon.Lambda.TestTool.UnitTests.csproj" Type="Classic C#" />
<Project Path="tests\Amazon.Lambda.TestTool.UnitTests\Amazon.Lambda.TestTool.UnitTests.csproj" />
<Project Path="tests\Amazon.Lambda.TestTool.IntegrationTests\Amazon.Lambda.TestTool.IntegrationTests.csproj" />
</Folder>
</Solution>
1 change: 1 addition & 0 deletions Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<ItemGroup>
<PackageReference Include="Spectre.Console" Version="0.49.1" />
<PackageReference Include="Spectre.Console.Cli" Version="0.49.1" />
<PackageReference Include="Amazon.Lambda.APIGatewayEvents" Version="2.7.1" />
<PackageReference Include="Blazored.Modal" Version="7.3.1" />
<PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="8.0.11" />
</ItemGroup>
Expand Down
270 changes: 270 additions & 0 deletions ...s/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Extensions/ApiGatewayResponseExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

using Amazon.Lambda.APIGatewayEvents;
using Amazon.Lambda.TestTool.Models;
using Microsoft.Extensions.Primitives;
using System.Text;

namespace Amazon.Lambda.TestTool.Extensions;

/// <summary>
/// Provides extension methods for converting API Gateway responses to <see cref="HttpResponse"/> objects.
/// </summary>
public static class ApiGatewayResponseExtensions
{
/// <summary>
/// Converts an <see cref="APIGatewayProxyResponse"/> to an <see cref="HttpResponse"/>.
/// </summary>
/// <param name="apiResponse">The API Gateway proxy response to convert.</param>
/// <param name="context">The <see cref="HttpContext"/> to use for the conversion.</param>
/// <param name="emulatorMode">The <see cref="ApiGatewayEmulatorMode"/> to use for the conversion.</param>
/// <returns>An <see cref="HttpResponse"/> representing the API Gateway response.</returns>
public static void ToHttpResponse(this APIGatewayProxyResponse apiResponse, HttpContext httpContext, ApiGatewayEmulatorMode emulatorMode)
{
var response = httpContext.Response;
response.Clear();

SetResponseHeaders(response, apiResponse.Headers, emulatorMode, apiResponse.MultiValueHeaders);
SetResponseBody(response, apiResponse.Body, apiResponse.IsBase64Encoded);
SetContentTypeAndStatusCodeV1(response, apiResponse.Headers, apiResponse.MultiValueHeaders, apiResponse.StatusCode, emulatorMode);
}

/// <summary>
/// Converts an <see cref="APIGatewayHttpApiV2ProxyResponse"/> to an <see cref="HttpResponse"/>.
/// </summary>
/// <param name="apiResponse">The API Gateway HTTP API v2 proxy response to convert.</param>
/// <param name="context">The <see cref="HttpContext"/> to use for the conversion.</param>
public static void ToHttpResponse(this APIGatewayHttpApiV2ProxyResponse apiResponse, HttpContext httpContext)
{
var response = httpContext.Response;
response.Clear();

SetResponseHeaders(response, apiResponse.Headers, ApiGatewayEmulatorMode.HttpV2);
SetResponseBody(response, apiResponse.Body, apiResponse.IsBase64Encoded);
SetContentTypeAndStatusCodeV2(response, apiResponse.Headers, apiResponse.StatusCode);
}

/// <summary>
/// Sets the response headers on the <see cref="HttpResponse"/>, including default API Gateway headers based on the emulator mode.
/// </summary>
/// <param name="response">The <see cref="HttpResponse"/> to set headers on.</param>
/// <param name="headers">The single-value headers to set.</param>
/// <param name="emulatorMode">The <see cref="ApiGatewayEmulatorMode"/> determining which default headers to include.</param>
/// <param name="multiValueHeaders">The multi-value headers to set.</param>
private static void SetResponseHeaders(HttpResponse response, IDictionary<string, string>? headers, ApiGatewayEmulatorMode emulatorMode, IDictionary<string, IList<string>>? multiValueHeaders = null)
{
// Add default API Gateway headers
var defaultHeaders = GetDefaultApiGatewayHeaders(emulatorMode);
foreach (var header in defaultHeaders)
{
response.Headers[header.Key] = header.Value;
}

if (multiValueHeaders != null)
{
foreach (var header in multiValueHeaders)
{
response.Headers[header.Key] = new StringValues(header.Value.ToArray());
}
}

if (headers != null)
{
foreach (var header in headers)
{
if (!response.Headers.ContainsKey(header.Key))
{
response.Headers[header.Key] = header.Value;
}
else
{
response.Headers.Append(header.Key, header.Value);
}
}
}
}

/// <summary>
/// Generates default API Gateway headers based on the specified emulator mode.
/// </summary>
/// <param name="emulatorMode">The <see cref="ApiGatewayEmulatorMode"/> determining which headers to generate.</param>
/// <returns>A dictionary of default headers appropriate for the specified emulator mode.</returns>
private static Dictionary<string, string> GetDefaultApiGatewayHeaders(ApiGatewayEmulatorMode emulatorMode)
{
var headers = new Dictionary<string, string>
{
{ "Date", DateTime.UtcNow.ToString("r") },
{ "Connection", "keep-alive" }
};

switch (emulatorMode)
{
case ApiGatewayEmulatorMode.Rest:
headers.Add("x-amzn-RequestId", Guid.NewGuid().ToString("D"));
headers.Add("x-amz-apigw-id", GenerateRequestId());
headers.Add("X-Amzn-Trace-Id", GenerateTraceId());
break;
case ApiGatewayEmulatorMode.HttpV1:
case ApiGatewayEmulatorMode.HttpV2:
headers.Add("Apigw-Requestid", GenerateRequestId());
break;
}

return headers;
}

/// <summary>
/// Generates a random X-Amzn-Trace-Id for REST API mode.
/// </summary>
/// <returns>A string representing a random X-Amzn-Trace-Id in the format used by API Gateway for REST APIs.</returns>
/// <remarks>
/// The generated trace ID includes:
/// - A root ID with a timestamp and two partial GUIDs
/// - A parent ID
/// - A sampling decision (always set to 0 in this implementation)
/// - A lineage identifier
/// </remarks>
private static string GenerateTraceId()
{
var timestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString("x");
var guid1 = Guid.NewGuid().ToString("N");
var guid2 = Guid.NewGuid().ToString("N");
return $"Root=1-{timestamp}-{guid1.Substring(0, 12)}{guid2.Substring(0, 12)};Parent={Guid.NewGuid().ToString("N").Substring(0, 16)};Sampled=0;Lineage=1:{Guid.NewGuid().ToString("N").Substring(0, 8)}:0";
}

/// <summary>
/// Generates a random API Gateway request ID for HTTP API v1 and v2.
/// </summary>
/// <returns>A string representing a random request ID in the format used by API Gateway for HTTP APIs.</returns>
/// <remarks>
gcbeattyAWS marked this conversation as resolved.
Show resolved Hide resolved
/// The generated ID is a 15-character string consisting of lowercase letters and numbers, followed by an equals sign.
/// </remarks>
private static string GenerateRequestId()
{
return Guid.NewGuid().ToString("N").Substring(0, 8) + Guid.NewGuid().ToString("N").Substring(0, 7) + "=";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use string interpolation

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

didnt get what you meant by this

}

/// <summary>
/// Sets the response body on the <see cref="HttpResponse"/>.
/// </summary>
/// <param name="response">The <see cref="HttpResponse"/> to set the body on.</param>
/// <param name="body">The body content.</param>
/// <param name="isBase64Encoded">Whether the body is Base64 encoded.</param>
gcbeattyAWS marked this conversation as resolved.
Show resolved Hide resolved
private static void SetResponseBody(HttpResponse response, string? body, bool isBase64Encoded)
{
if (!string.IsNullOrEmpty(body))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we will update the existing HttpResponse, the body and content length might already have values. You should add an else to clear them out if body is null or empty.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i added response.clear up above

{
byte[] bodyBytes;
if (isBase64Encoded)
{
bodyBytes = Convert.FromBase64String(body);
}
else
{
bodyBytes = Encoding.UTF8.GetBytes(body);
}

response.Body = new MemoryStream(bodyBytes);
response.ContentLength = bodyBytes.Length;
}
}

/// <summary>
/// Sets the content type and status code for API Gateway v1 responses.
/// </summary>
/// <param name="response">The <see cref="HttpResponse"/> to set the content type and status code on.</param>
/// <param name="headers">The single-value headers.</param>
/// <param name="multiValueHeaders">The multi-value headers.</param>
/// <param name="statusCode">The status code to set.</param>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ApiGatewayEmulatorMode emulatorMode does not have a docs param

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

/// <param name="emulatorMode">The <see cref="ApiGatewayEmulatorMode"/> being used.</param>
private static void SetContentTypeAndStatusCodeV1(HttpResponse response, IDictionary<string, string>? headers, IDictionary<string, IList<string>>? multiValueHeaders, int statusCode, ApiGatewayEmulatorMode emulatorMode)
{
string? contentType = null;

if (headers != null && headers.TryGetValue("Content-Type", out var headerContentType))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: you could just do:

if (headers != null && headers.TryGetValue("Content-Type", out contentType))

{
contentType = headerContentType;
}
else if (multiValueHeaders != null && multiValueHeaders.TryGetValue("Content-Type", out var multiValueContentType))
{
contentType = multiValueContentType.FirstOrDefault();
}

if (contentType != null)
{
response.ContentType = contentType;
}
else
{
if (emulatorMode == ApiGatewayEmulatorMode.HttpV1)
{
response.ContentType = "text/plain; charset=utf-8";
}
else if (emulatorMode == ApiGatewayEmulatorMode.Rest)
{
response.ContentType = "application/json";
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for the sake of completion, add an else and handle the fact that we could mistakenly make this call for V2. Maybe throw an exception as this is dev error and needs to be updated.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

else
{
throw new ArgumentException("This function should only be called for ApiGatewayEmulatorMode.HttpV1 or ApiGatewayEmulatorMode.Rest");
}
}

if (statusCode != 0)
{
response.StatusCode = statusCode;
}
else
{
if (emulatorMode == ApiGatewayEmulatorMode.Rest) // rest api text for this message/error code is slightly different
{
response.StatusCode = 502;
response.ContentType = "application/json";
var errorBytes = Encoding.UTF8.GetBytes("{\"message\": \"Internal server error\"}");
response.Body = new MemoryStream(errorBytes);
response.ContentLength = errorBytes.Length;
response.Headers["x-amzn-ErrorType"] = "InternalServerErrorException";
}
else
{
response.StatusCode = 500;
response.ContentType = "application/json";
var errorBytes = Encoding.UTF8.GetBytes("{\"message\":\"Internal Server Error\"}");
response.Body = new MemoryStream(errorBytes);
response.ContentLength = errorBytes.Length;
}
}
}

/// <summary>
/// Sets the content type and status code for API Gateway v2 responses.
/// </summary>
/// <param name="response">The <see cref="HttpResponse"/> to set the content type and status code on.</param>
/// <param name="headers">The headers.</param>
/// <param name="statusCode">The status code to set.</param>
private static void SetContentTypeAndStatusCodeV2(HttpResponse response, IDictionary<string, string>? headers, int statusCode)
{
if (headers != null && headers.TryGetValue("Content-Type", out var contentType))
{
response.ContentType = contentType;
}
else
{
response.ContentType = "text/plain; charset=utf-8"; // api gateway v2 defaults to this content type if none is provided
}

if (statusCode != 0)
{
response.StatusCode = statusCode;
}
else
{
response.StatusCode = 500;
response.ContentType = "application/json";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should you check to see if content type was already set from the headers collection coming back from the Lambda function?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ill add an integration test case to confirm what happens in that scenario and update accordingly

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ive added V2_TestJsonInfersAlwaysUsesApplicationJson. i believe my current logic is right. the reason being is for this scenario, since status code is 0, api gateway just uses the whole payload as the body response and doesnt consider any lambda provided headers.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i updated this logic to just be straightforward - with the results from my manual testing. that if status code is not set, then api gateway http 2.0 will default to application/json

Copy link
Author

@gcbeattyAWS gcbeattyAWS Dec 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i guess what you meant by this (based on your other comment with the binary type for rest api), is that if the user configured custom response headers for this error code that it might be in the response already? ill have to check that

edit: actually i guess my above crossed out comment only applies to the other code up above where i do internal server error. so ill see what happens in that case. i think the user can specify custom response headers in rest api depending on certain error codes

Copy link
Author

@gcbeattyAWS gcbeattyAWS Dec 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so i was thinking about this more with the way the v2 infers things. i think we need another layer before this api gateway response -> http response. i think we need a lambda memory stream -> api gateway response, which will do these inferences

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i removed the json inference check from here. i think we should add the lambda -> api gateay translation inference in another pr

var errorBytes = Encoding.UTF8.GetBytes("{\"message\":\"Internal Server Error\"}");
response.Body = new MemoryStream(errorBytes);
response.ContentLength = errorBytes.Length;
}
}
}
41 changes: 41 additions & 0 deletions ...ts/Amazon.Lambda.TestTool.IntegrationTests/Amazon.Lambda.TestTool.IntegrationTests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Amazon.Lambda.APIGatewayEvents" Version="2.7.1" />
<PackageReference Include="AWSSDK.APIGateway" Version="3.7.401.7" />
<PackageReference Include="AWSSDK.CloudFormation" Version="3.7.401.11" />
<PackageReference Include="AWSSDK.IdentityManagement" Version="3.7.402" />
<PackageReference Include="AWSSDK.ApiGatewayV2" Version="3.7.400.63" />
<PackageReference Include="AWSSDK.Lambda" Version="3.7.408.1" />
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="System.Configuration.ConfigurationManager" Version="9.0.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
</ItemGroup>


<ItemGroup>
<ProjectReference Include="..\..\src\Amazon.Lambda.TestTool\Amazon.Lambda.TestTool.csproj" />
<ProjectReference Include="..\Amazon.Lambda.TestTool.UnitTests\Amazon.Lambda.TestTool.UnitTests.csproj" />
</ItemGroup>

<ItemGroup>
<EmbeddedResource Include="cloudformation-template-apigateway.yaml" />
</ItemGroup>


<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>

</Project>
25 changes: 25 additions & 0 deletions ...tests/Amazon.Lambda.TestTool.IntegrationTests/Amazon.Lambda.TestTool.IntegrationTests.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.5.002.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Amazon.Lambda.TestTool.IntegrationTests", "Amazon.Lambda.TestTool.IntegrationTests.csproj", "{94C7903E-A21A-43EC-BB04-C9DA404F1C02}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{94C7903E-A21A-43EC-BB04-C9DA404F1C02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{94C7903E-A21A-43EC-BB04-C9DA404F1C02}.Debug|Any CPU.Build.0 = Debug|Any CPU
{94C7903E-A21A-43EC-BB04-C9DA404F1C02}.Release|Any CPU.ActiveCfg = Release|Any CPU
{94C7903E-A21A-43EC-BB04-C9DA404F1C02}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {429CE21F-1692-4C50-A9E6-299AB413D027}
EndGlobalSection
EndGlobal
Loading
Loading