Add raw data logging support for LambdaLogger

Libraries/src/Amazon.Lambda.Core/LambdaLogger.cs

@@ -12,12 +12,41 @@ public static class LambdaLogger
         // Logging action, logs to Console by default
         private static Action<string> _loggingAction = LogToConsole;
 
+#if NET6_0_OR_GREATER
+        private static System.Buffers.ReadOnlySpanAction<byte, object> _dataLoggingAction = LogUtf8BytesToConsole;
+#endif
+
         // Logs message to console
         private static void LogToConsole(string message)
         {
             Console.WriteLine(message);
         }
 
+#if NET6_0_OR_GREATER
+        private static void LogUtf8BytesToConsole(ReadOnlySpan<byte> utf8Message, object state)
+        {
+            try
+            {
+                Console.WriteLine(Console.OutputEncoding.GetString(utf8Message));
+            }
+            catch
+            {
+                // ignore any encoding error
+            }
+        }
+
+        /// <summary>
+        /// Logs a message to AWS CloudWatch Logs. <br/>
+        /// Logging will not be done:
+        /// If the role provided to the function does not have sufficient permissions.
+        /// </summary>
+        /// <param name="utf8Message">The message as UTF-8 encoded data.</param>
+        public static void Log(ReadOnlySpan<byte> utf8Message)
+        {
+            _dataLoggingAction(utf8Message, null);
+        }
+#endif
+
         /// <summary>
         /// Logs a message to AWS CloudWatch Logs.
         /// 

Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs

@@ -106,6 +106,25 @@ private LambdaBootstrap(HttpClient httpClient, LambdaBootstrapHandler handler, L
             Client = new RuntimeApiClient(new SystemEnvironmentVariables(), _httpClient);
         }
 
+        /// <summary>
+        /// Create a LambdaBootstrap that will call the given initializer and handler.
+        /// </summary>
+        /// <param name="httpClient">The HTTP client to use with the Lambda runtime.</param>
+        /// <param name="handler">Delegate called for each invocation of the Lambda function.</param>
+        /// <param name="initializer">Delegate called to initialize the Lambda function.  If not provided the initialization step is skipped.</param>
+        /// <param name="ownsHttpClient">Whether the instance owns the HTTP client and should dispose of it.</param>
+        /// <param name="runtimeApiClient">Instance of <see cref="IRuntimeApiClient"/> to call the runtime API.</param>
+        /// <returns></returns>
+        internal LambdaBootstrap(HttpClient httpClient, LambdaBootstrapHandler handler, LambdaBootstrapInitializer initializer, bool ownsHttpClient, IRuntimeApiClient runtimeApiClient)
+        {
+            _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+            _handler = handler ?? throw new ArgumentNullException(nameof(handler));
+            _ownsHttpClient = ownsHttpClient;
+            _initializer = initializer;
+            _httpClient.Timeout = RuntimeApiHttpTimeout;
+            Client = runtimeApiClient;
+        }
+
         /// <summary>
         /// Run the initialization Func if provided.
         /// Then run the invoke loop, calling the handler for each invocation.

Libraries/src/Amazon.Lambda.RuntimeSupport/Helpers/ConsoleLoggerWriter.cs

@@ -1,5 +1,6 @@
 using Amazon.Lambda.RuntimeSupport.Bootstrap;
 using System;
+using System.Buffers;
 using System.IO;
 using System.Reflection;
 using System.Text;
@@ -82,6 +83,8 @@ public void FormattedWriteLine(string level, string message)
 
 #if NET6_0_OR_GREATER
 
+    internal enum LogFormatType { Default, Unformatted }
+
     /// <summary>
     /// Formats log messages with time, request id, log level and message
     /// </summary>
@@ -145,18 +148,18 @@ public LogLevelLoggerWriter()
                 {
                     var stdOutWriter = FileDescriptorLogFactory.GetWriter(fileDescriptorLogId);
                     var stdErrorWriter = FileDescriptorLogFactory.GetWriter(fileDescriptorLogId);
-                    Initialize(stdOutWriter, stdErrorWriter);
+                    Initialize(stdOutWriter, stdErrorWriter, stdOutWriter.BaseStream);
                     InternalLogger.GetDefaultLogger().LogInformation("Using file descriptor stream writer for logging.");
                 }
-                catch(Exception ex)
+                catch (Exception ex)
                 {
                     InternalLogger.GetDefaultLogger().LogError(ex, "Error creating file descriptor log stream writer. Fallback to stdout and stderr.");
-                    Initialize(Console.Out, Console.Error);
+                    Initialize(Console.Out, Console.Error, null);
                 }
             }
             else
             {
-                Initialize(Console.Out, Console.Error);
+                Initialize(Console.Out, Console.Error, null);
                 InternalLogger.GetDefaultLogger().LogInformation("Using stdout and stderr for logging.");
             }
 
@@ -171,15 +174,16 @@ public LogLevelLoggerWriter()
             ConfigureLoggingActionField();
         }
 
-        public LogLevelLoggerWriter(TextWriter stdOutWriter, TextWriter stdErrorWriter)
+        public LogLevelLoggerWriter(TextWriter stdOutWriter, TextWriter stdErrorWriter, Stream outputStream)
         {
-            Initialize(stdOutWriter, stdErrorWriter);
+            Initialize(stdOutWriter, stdErrorWriter, outputStream);
+            ConfigureLoggingActionField();
         }
 
-        private void Initialize(TextWriter stdOutWriter, TextWriter stdErrorWriter)
+        private void Initialize(TextWriter stdOutWriter, TextWriter stdErrorWriter, Stream outputStream)
         {
-            _wrappedStdOutWriter = new WrapperTextWriter(stdOutWriter, LogLevel.Information.ToString());
-            _wrappedStdErrorWriter = new WrapperTextWriter(stdErrorWriter, LogLevel.Error.ToString());
+            _wrappedStdOutWriter = new WrapperTextWriter(stdOutWriter, outputStream, LogLevel.Information.ToString());
+            _wrappedStdErrorWriter = new WrapperTextWriter(stdErrorWriter, outputStream, LogLevel.Error.ToString());
         }
 
         /// <summary>
@@ -199,6 +203,10 @@ private void ConfigureLoggingActionField()
 
             Action<string> callback = (message => FormattedWriteLine(null, message));
             loggingActionField.SetValue(null, callback);
+
+            var dataLoggingActionField = lambdaILoggerType.GetTypeInfo().GetField("_dataLoggingAction", BindingFlags.NonPublic | BindingFlags.Static);
+            ReadOnlySpanAction<byte, object> dataLoggingAction = (data, state) => FormattedWriteBytes(null, data);
+            dataLoggingActionField.SetValue(null, dataLoggingAction);
         }
 
         public void SetCurrentAwsRequestId(string awsRequestId)
@@ -217,6 +225,16 @@ public void FormattedWriteLine(string level, string message)
             _wrappedStdOutWriter.FormattedWriteLine(level, message);
         }
 
+        public void FormattedWriteBytes(string level, ReadOnlySpan<byte> message)
+        {
+            _wrappedStdOutWriter.FormattedWriteBytes(level, message);
+        }
+
+        internal void SetLogFormatType(LogFormatType logFormatType)
+        {
+            _wrappedStdOutWriter.SetLogFormatType(logFormatType);
+            _wrappedStdErrorWriter.SetLogFormatType(logFormatType);
+        }
 
         /// <summary>
         /// Wraps around a provided TextWriter. In normal usage the wrapped TextWriter will either be stdout or stderr. 
@@ -226,15 +244,14 @@ public void FormattedWriteLine(string level, string message)
         class WrapperTextWriter : TextWriter
         {
             private readonly TextWriter _innerWriter;
+            private readonly Stream _innerOutputStream;
             private string _defaultLogLevel;
 
             const string LOG_LEVEL_ENVIRONMENT_VARAIBLE = "AWS_LAMBDA_HANDLER_LOG_LEVEL";
             const string LOG_FORMAT_ENVIRONMENT_VARAIBLE = "AWS_LAMBDA_HANDLER_LOG_FORMAT";
 
             private LogLevel _minmumLogLevel = LogLevel.Information;
 
-            enum LogFormatType { Default, Unformatted }
-
             private LogFormatType _logFormatType = LogFormatType.Default;
 
             public string CurrentAwsRequestId { get; set; } = string.Empty;
@@ -247,9 +264,19 @@ enum LogFormatType { Default, Unformatted }
             /// </summary>
             internal object LockObject { get; set; } = new object();
 
-            public WrapperTextWriter(TextWriter innerWriter, string defaultLogLevel)
+            /// <summary>
+            /// Initializes an instance of <see cref="WrapperTextWriter"/>.
+            /// </summary>
+            /// <param name="innerWriter">The inner <see cref="TextWriter"/> instance used to write the output message.</param>
+            /// <param name="innerOutputStream">
+            /// The inner output <see cref="Stream"/> that provides direct write access to Telemetry file descriptor.
+            /// In case Telemetry file descriptor is not available this parameter is set to NULL.
+            /// </param>
+            /// <param name="defaultLogLevel">The default logging level.</param>
+            public WrapperTextWriter(TextWriter innerWriter, Stream innerOutputStream, string defaultLogLevel)
             {
                 _innerWriter = innerWriter;
+                _innerOutputStream = innerOutputStream;
                 _defaultLogLevel = defaultLogLevel;
 
                 var envLogLevel = Environment.GetEnvironmentVariable(LOG_LEVEL_ENVIRONMENT_VARAIBLE);
@@ -271,14 +298,19 @@ public WrapperTextWriter(TextWriter innerWriter, string defaultLogLevel)
                 }
             }
 
+            internal void SetLogFormatType(LogFormatType logFormatType)
+            {
+                _logFormatType = logFormatType;
+            }
+
             internal void FormattedWriteLine(string message)
             {
                 FormattedWriteLine(_defaultLogLevel, message);
             }
 
             internal void FormattedWriteLine(string level, string message)
             {
-                lock(LockObject)
+                lock (LockObject)
                 {
                     var displayLevel = level;
                     if (Enum.TryParse<LogLevel>(level, true, out var levelEnum))
@@ -310,6 +342,58 @@ internal void FormattedWriteLine(string level, string message)
                 }
             }
 
+            internal void FormattedWriteBytes(string level, ReadOnlySpan<byte> message)
+            {
+                if (_innerOutputStream is null)
+                {
+                    // the telemetry FD output stream is not present, we delegate to FormattedWriteLine()
+                    FormattedWriteLine(level, _innerWriter.Encoding.GetString(message));
+                    return;
+                }
+
+                lock (LockObject)
+                {
+                    var displayLevel = level;
+                    if (Enum.TryParse<LogLevel>(level, true, out var levelEnum))
+                    {
+                        if (levelEnum < _minmumLogLevel)
+                            return;
+
+                        displayLevel = ConvertLogLevelToLabel(levelEnum);
+                    }
+
+                    if (_logFormatType == LogFormatType.Unformatted)
+                    {
+                        _innerOutputStream.Write(message);
+                    }
+                    else
+                    {
+                        string linePrefix;
+                        if (!string.IsNullOrEmpty(displayLevel))
+                        {
+                            linePrefix = $"{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")}\t{CurrentAwsRequestId}\t{displayLevel}\t";
+                        }
+                        else
+                        {
+                            linePrefix = $"{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")}\t{CurrentAwsRequestId}\t";
+                        }
+
+                        // acquire memory to put the prefix and the message together
+                        var logEntryBuffer = ArrayPool<byte>.Shared.Rent(_innerWriter.Encoding.GetMaxByteCount(linePrefix.Length) + message.Length);
+                        try
+                        {
+                            var prefixLength = _innerWriter.Encoding.GetBytes(linePrefix, logEntryBuffer.AsSpan());
+                            message.CopyTo(logEntryBuffer.AsSpan().Slice(prefixLength));
+                            _innerOutputStream.Write(logEntryBuffer.AsSpan().Slice(0, prefixLength + message.Length));
+                        }
+                        finally
+                        {
+                            ArrayPool<byte>.Shared.Return(logEntryBuffer);
+                        }
+                    }
+                }
+            }
+
             private Task FormattedWriteLineAsync(string message)
             {
                 FormattedWriteLine(message);

Libraries/src/Amazon.Lambda.RuntimeSupport/Helpers/FileDescriptorLogStream.cs

@@ -4,6 +4,9 @@
 using System.Text;
 using Microsoft.Win32.SafeHandles;
 using System.Collections.Concurrent;
+#if NET6_0_OR_GREATER
+using System.Buffers.Binary;
+#endif
 
 namespace Amazon.Lambda.RuntimeSupport.Helpers
 {
@@ -27,7 +30,7 @@ public static class FileDescriptorLogFactory
         /// <returns></returns>
         public static StreamWriter GetWriter(string fileDescriptorId)
         {
-            var writer = _writers.GetOrAdd(fileDescriptorId, 
+            var writer = _writers.GetOrAdd(fileDescriptorId,
                 (x) => {
                     SafeFileHandle handle = new SafeFileHandle(new IntPtr(int.Parse(fileDescriptorId)), false);
                     return InitializeWriter(new FileStream(handle, FileAccess.Write));
@@ -107,6 +110,29 @@ public override void Write(byte[] buffer, int offset, int count)
                 }
             }
 
+#if NET6_0_OR_GREATER
+            public override void Write(ReadOnlySpan<byte> buffer)
+            {
+                while (buffer.Length > 0)
+                {
+                    var bufferToWrite = buffer.Length > MaxCloudWatchLogEventSize ? buffer.Slice(0, MaxCloudWatchLogEventSize) : buffer;
+                    DoWriteBuffer(bufferToWrite);
+                    buffer = buffer.Slice(bufferToWrite.Length);
+                }
+            }
+
+            private void DoWriteBuffer(ReadOnlySpan<byte> buffer)
+            {
+                Span<byte> typeAndLength = stackalloc byte[LambdaTelemetryLogHeaderLength];
+                BinaryPrimitives.WriteUInt32BigEndian(typeAndLength[..4], LambdaTelemetryLogHeaderFrameType);
+                BinaryPrimitives.WriteInt32BigEndian(typeAndLength.Slice(4, 4), buffer.Length);
+
+                _fileDescriptorStream.Write(typeAndLength);
+                _fileDescriptorStream.Write(buffer);
+                _fileDescriptorStream.Flush();
+            }
+#endif
+
             #region Not implemented read and seek operations
             public override bool CanRead => false;
 

Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/Amazon.Lambda.RuntimeSupport.UnitTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFrameworks>netcoreapp3.1;net6.0</TargetFrameworks>
   </PropertyGroup>
 
   <ItemGroup>