From 7a2e36d74d1aad4dcbd654e3b53c4e227875cfb4 Mon Sep 17 00:00:00 2001 From: Erikson Bahr Date: Thu, 19 Dec 2024 16:49:02 -0300 Subject: [PATCH] add amazon nova support (text) only --- .../Bedrock/Core/BedrockServiceFactory.cs | 2 +- .../Clients/BedrockTextGenerationClient.cs | 7 +- .../Core/IBedrockTextGenerationService.cs | 5 +- .../Core/Models/AI21Labs/AI21JambaService.cs | 4 +- .../Models/AI21Labs/AI21JurassicService.cs | 4 +- .../Core/Models/Amazon/AmazonService.cs | 269 +++++++++++++----- .../Bedrock/Core/Models/Amazon/NovaRequest.cs | 117 ++++++++ .../Core/Models/Amazon/NovaResponse.cs | 45 +++ .../Core/Models/Anthropic/AnthropicService.cs | 4 +- .../Models/Cohere/CohereCommandRService.cs | 4 +- .../Models/Cohere/CohereCommandService.cs | 4 +- .../Bedrock/Core/Models/Meta/MetaService.cs | 4 +- .../Core/Models/Mistral/MistralService.cs | 4 +- .../Settings/AmazonNovaExecutionSettings.cs | 125 ++++++++ 14 files changed, 511 insertions(+), 87 deletions(-) create mode 100644 dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaRequest.cs create mode 100644 dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaResponse.cs create mode 100644 dotnet/src/Connectors/Connectors.Amazon/Bedrock/Settings/AmazonNovaExecutionSettings.cs diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/BedrockServiceFactory.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/BedrockServiceFactory.cs index e5b59f0c465a..c02a5e37a2ae 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/BedrockServiceFactory.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/BedrockServiceFactory.cs @@ -32,7 +32,7 @@ internal IBedrockTextGenerationService CreateTextGenerationService(string modelI } throw new NotSupportedException($"Unsupported AI21 model: {modelId}"); case "AMAZON": - if (modelName.StartsWith("titan-", StringComparison.OrdinalIgnoreCase)) + if (modelName.StartsWith("titan-", StringComparison.OrdinalIgnoreCase) || modelName.StartsWith("nova-", StringComparison.OrdinalIgnoreCase)) { return new AmazonService(); } diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Clients/BedrockTextGenerationClient.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Clients/BedrockTextGenerationClient.cs index b659cb81bf74..71a615c6c6ce 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Clients/BedrockTextGenerationClient.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Clients/BedrockTextGenerationClient.cs @@ -7,6 +7,7 @@ using System.Runtime.CompilerServices; using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using Amazon.BedrockRuntime; @@ -76,7 +77,7 @@ internal async Task> InvokeBedrockModelAsync( try { var requestBody = this._ioTextService.GetInvokeModelRequestBody(this._modelId, prompt, executionSettings); - using var requestBodyStream = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(requestBody)); + using var requestBodyStream = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(requestBody, new JsonSerializerOptions { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull })); invokeRequest.Body = requestBodyStream; response = await this._bedrockRuntime.InvokeModelAsync(invokeRequest, cancellationToken).ConfigureAwait(false); @@ -111,7 +112,7 @@ internal async Task> InvokeBedrockModelAsync( } activityStatus = BedrockClientUtilities.ConvertHttpStatusCodeToActivityStatusCode(response.HttpStatusCode); activity?.SetStatus(activityStatus); - IReadOnlyList textResponse = this._ioTextService.GetInvokeResponseBody(response); + IReadOnlyList textResponse = this._ioTextService.GetInvokeResponseBody(this._modelId, response); activity?.SetCompletionResponse(textResponse); return textResponse; } @@ -178,7 +179,7 @@ internal async IAsyncEnumerable StreamTextAsync( { continue; } - IEnumerable texts = this._ioTextService.GetTextStreamOutput(chunk); + IEnumerable texts = this._ioTextService.GetTextStreamOutput(this._modelId, chunk); foreach (var text in texts) { var content = new StreamingTextContent(text); diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/IBedrockTextGenerationService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/IBedrockTextGenerationService.cs index 1da2bb9f1790..5a8f57e08554 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/IBedrockTextGenerationService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/IBedrockTextGenerationService.cs @@ -23,14 +23,15 @@ internal interface IBedrockTextGenerationService /// /// Extracts the text contents from the . /// + /// The model ID to be used as a request parameter. /// The instance to be returned from the InvokeAsync Bedrock call. /// The list of TextContent objects for the Semantic Kernel output. - internal IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response); + internal IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response); /// /// Converts the streaming JSON into for output. /// /// The payloadPart bytes provided from the streaming response. /// output strings. - internal IEnumerable GetTextStreamOutput(JsonNode chunk); + internal IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk); } diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JambaService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JambaService.cs index 110782a2c8ad..4faf0aac6543 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JambaService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JambaService.cs @@ -46,7 +46,7 @@ public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExe } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { using var reader = new StreamReader(response.Body); var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); @@ -108,7 +108,7 @@ public ConverseRequest GetConverseRequest(string modelId, ChatHistory chatHistor } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { var choiceDeltaContent = chunk["choices"]?[0]?["delta"]?["content"]; if (choiceDeltaContent is not null) diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JurassicService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JurassicService.cs index 28087f77243b..7c65a6462ebf 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JurassicService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/AI21Labs/AI21JurassicService.cs @@ -34,7 +34,7 @@ public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExe } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { using var reader = new StreamReader(response.Body); var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); @@ -48,7 +48,7 @@ public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse resp } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { throw new NotSupportedException("Streaming not supported by this model."); } diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/AmazonService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/AmazonService.cs index 17a32450ed24..e8f50a3e2dd6 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/AmazonService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/AmazonService.cs @@ -1,12 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; using Amazon.BedrockRuntime.Model; using Amazon.Runtime.Documents; using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.Nova; +using Microsoft.SemanticKernel.Connectors.Nova.Core; +using static Microsoft.SemanticKernel.Connectors.Nova.Core.NovaRequest; namespace Microsoft.SemanticKernel.Connectors.Amazon.Core; @@ -18,39 +23,88 @@ internal sealed class AmazonService : IBedrockTextGenerationService, IBedrockCha /// public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExecutionSettings? executionSettings) { - var settings = AmazonTitanExecutionSettings.FromExecutionSettings(executionSettings); - var temperature = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "temperature") ?? settings.Temperature; - var topP = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "topP") ?? settings.TopP; - var maxTokenCount = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "maxTokenCount") ?? settings.MaxTokenCount; - var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(executionSettings?.ExtensionData, "stopSequences") ?? settings.StopSequences; + if (modelId.StartsWith("amazon.titan-", StringComparison.OrdinalIgnoreCase)) + { + var settings = AmazonTitanExecutionSettings.FromExecutionSettings(executionSettings); + var temperature = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "temperature") ?? settings.Temperature; + var topP = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "topP") ?? settings.TopP; + var maxTokenCount = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "maxTokenCount") ?? settings.MaxTokenCount; + var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(executionSettings?.ExtensionData, "stopSequences") ?? settings.StopSequences; + + var requestBody = new TitanRequest.TitanTextGenerationRequest() + { + InputText = prompt, + TextGenerationConfig = new TitanRequest.AmazonTitanTextGenerationConfig() + { + MaxTokenCount = maxTokenCount, + TopP = topP, + Temperature = temperature, + StopSequences = stopSequences + } + }; + return requestBody; + } - var requestBody = new TitanRequest.TitanTextGenerationRequest() + if (modelId.StartsWith("amazon.nova-", StringComparison.OrdinalIgnoreCase)) { - InputText = prompt, - TextGenerationConfig = new TitanRequest.AmazonTitanTextGenerationConfig() + var settings = AmazonNovaExecutionSettings.FromExecutionSettings(executionSettings); + var schemaVersion = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "schemaVersion") ?? settings.SchemaVersion; + var maxNewTokens = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "max_new_tokens") ?? settings.MaxNewTokens; + var topP = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "top_p") ?? settings.TopP; + var topK = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "top_k") ?? settings.TopK; + var temperature = BedrockModelUtilities.GetExtensionDataValue(executionSettings?.ExtensionData, "temperature") ?? settings.Temperature; + var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(executionSettings?.ExtensionData, "stopSequences") ?? settings.StopSequences; + + var requestBody = new NovaRequest.NovaTextGenerationRequest() { - MaxTokenCount = maxTokenCount, - TopP = topP, - Temperature = temperature, - StopSequences = stopSequences - } - }; - return requestBody; + InferenceConfig = new NovaRequest.NovaTextGenerationConfig + { + MaxNewTokens = maxNewTokens, + Temperature = temperature, + TopK = topK, + TopP = topP + }, + Messages = new List { new() { Role = AuthorRole.User.Label, Content = new List { new() { Text = prompt } } } }, + SchemaVersion = schemaVersion ?? "messages-v1", + }; + return requestBody; + } + + throw new NotSupportedException($"Amazon model '{modelId}' is not supported. Supported models: Titan and Nova."); } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { - using var reader = new StreamReader(response.Body); - var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); - List textContents = []; - if (responseBody?.Results is not { Count: > 0 }) + if (modelId.StartsWith("amazon.titan-", StringComparison.OrdinalIgnoreCase)) { + using var reader = new StreamReader(response.Body); + var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); + List textContents = []; + if (responseBody?.Results is not { Count: > 0 }) + { + return textContents; + } + string? outputText = responseBody.Results[0].OutputText; + textContents.Add(new TextContent(outputText)); return textContents; } - string? outputText = responseBody.Results[0].OutputText; - textContents.Add(new TextContent(outputText)); - return textContents; + + if (modelId.StartsWith("amazon.nova-", StringComparison.OrdinalIgnoreCase)) + { + using var reader = new StreamReader(response.Body); + var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd(), new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + List textContents = []; + if (responseBody?.Output?.Message?.Contents is not { Count: > 0 }) + { + return textContents; + } + string? outputText = responseBody.Output.Message.Contents[0].Text; + textContents.Add(new TextContent(outputText)); + return textContents; + } + + throw new NotSupportedException($"Amazon model '{modelId}' is not supported. Supported models: Titan and Nova."); } /// @@ -59,39 +113,86 @@ public ConverseRequest GetConverseRequest(string modelId, ChatHistory chatHistor var messages = BedrockModelUtilities.BuildMessageList(chatHistory); var systemMessages = BedrockModelUtilities.GetSystemMessages(chatHistory); - var executionSettings = AmazonTitanExecutionSettings.FromExecutionSettings(settings); - var temperature = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "temperature") ?? executionSettings.Temperature; - var topP = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "topP") ?? executionSettings.TopP; - var maxTokenCount = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "maxTokenCount") ?? executionSettings.MaxTokenCount; - var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(settings?.ExtensionData, "stopSequences") ?? executionSettings.StopSequences; + if (modelId.StartsWith("amazon.titan-", StringComparison.OrdinalIgnoreCase)) + { + var executionSettings = AmazonTitanExecutionSettings.FromExecutionSettings(settings); + var temperature = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "temperature") ?? executionSettings.Temperature; + var topP = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "topP") ?? executionSettings.TopP; + var maxTokenCount = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "maxTokenCount") ?? executionSettings.MaxTokenCount; + var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(settings?.ExtensionData, "stopSequences") ?? executionSettings.StopSequences; - var inferenceConfig = new InferenceConfiguration(); - BedrockModelUtilities.SetPropertyIfNotNull(() => temperature, value => inferenceConfig.Temperature = value); - BedrockModelUtilities.SetPropertyIfNotNull(() => topP, value => inferenceConfig.TopP = value); - BedrockModelUtilities.SetPropertyIfNotNull(() => maxTokenCount, value => inferenceConfig.MaxTokens = value); - BedrockModelUtilities.SetNullablePropertyIfNotNull(() => stopSequences, value => inferenceConfig.StopSequences = value); + var inferenceConfig = new InferenceConfiguration(); + BedrockModelUtilities.SetPropertyIfNotNull(() => temperature, value => inferenceConfig.Temperature = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => topP, value => inferenceConfig.TopP = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => maxTokenCount, value => inferenceConfig.MaxTokens = value); + BedrockModelUtilities.SetNullablePropertyIfNotNull(() => stopSequences, value => inferenceConfig.StopSequences = value); - var converseRequest = new ConverseRequest + var converseRequest = new ConverseRequest + { + ModelId = modelId, + Messages = messages, + System = systemMessages, + InferenceConfig = inferenceConfig, + AdditionalModelRequestFields = new Document(), + AdditionalModelResponseFieldPaths = new List() + }; + + return converseRequest; + } + + if (modelId.StartsWith("amazon.nova-", StringComparison.OrdinalIgnoreCase)) { - ModelId = modelId, - Messages = messages, - System = systemMessages, - InferenceConfig = inferenceConfig, - AdditionalModelRequestFields = new Document(), - AdditionalModelResponseFieldPaths = new List() - }; - - return converseRequest; + var executionSettings = AmazonNovaExecutionSettings.FromExecutionSettings(settings); + var schemaVersion = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "schemaVersion") ?? executionSettings.SchemaVersion; + var maxNewTokens = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "max_new_tokens") ?? executionSettings.MaxNewTokens; + var topP = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "top_p") ?? executionSettings.TopP; + var topK = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "top_k") ?? executionSettings.TopK; + var temperature = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "temperature") ?? executionSettings.Temperature; + var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(settings?.ExtensionData, "stopSequences") ?? executionSettings.StopSequences; + + var inferenceConfig = new InferenceConfiguration(); + BedrockModelUtilities.SetPropertyIfNotNull(() => temperature, value => inferenceConfig.Temperature = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => topP, value => inferenceConfig.TopP = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => maxNewTokens, value => inferenceConfig.MaxTokens = value); + BedrockModelUtilities.SetNullablePropertyIfNotNull(() => stopSequences, value => inferenceConfig.StopSequences = value); + + var converseRequest = new ConverseRequest + { + ModelId = modelId, + Messages = messages, + System = systemMessages, + InferenceConfig = inferenceConfig, + AdditionalModelRequestFields = new Document(), + AdditionalModelResponseFieldPaths = new List() + }; + + return converseRequest; + } + + throw new NotSupportedException($"Amazon model '{modelId}' is not supported. Supported models: Titan and Nova."); } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { - var text = chunk["outputText"]?.ToString(); - if (!string.IsNullOrEmpty(text)) + if (modelId.StartsWith("amazon.titan-", StringComparison.OrdinalIgnoreCase)) + { + var text = chunk["outputText"]?.ToString(); + if (!string.IsNullOrEmpty(text)) + { + yield return text!; + } + } + + if (modelId.StartsWith("amazon.nova-", StringComparison.OrdinalIgnoreCase)) { - yield return text!; + var text = chunk["output"]?["message"]?["content"]?["text"]?.ToString(); + if (!string.IsNullOrEmpty(text)) + { + yield return text!; + } } + throw new NotSupportedException($"Amazon model '{modelId}' is not supported. Supported models: Titan and Nova."); } /// @@ -100,28 +201,62 @@ public ConverseStreamRequest GetConverseStreamRequest(string modelId, ChatHistor var messages = BedrockModelUtilities.BuildMessageList(chatHistory); var systemMessages = BedrockModelUtilities.GetSystemMessages(chatHistory); - var executionSettings = AmazonTitanExecutionSettings.FromExecutionSettings(settings); - var temperature = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "temperature") ?? executionSettings.Temperature; - var topP = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "topP") ?? executionSettings.TopP; - var maxTokenCount = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "maxTokenCount") ?? executionSettings.MaxTokenCount; - var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(settings?.ExtensionData, "stopSequences") ?? executionSettings.StopSequences; + if (modelId.StartsWith("amazon.titan-", StringComparison.OrdinalIgnoreCase)) + { + var executionSettings = AmazonTitanExecutionSettings.FromExecutionSettings(settings); + var temperature = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "temperature") ?? executionSettings.Temperature; + var topP = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "topP") ?? executionSettings.TopP; + var maxTokenCount = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "maxTokenCount") ?? executionSettings.MaxTokenCount; + var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(settings?.ExtensionData, "stopSequences") ?? executionSettings.StopSequences; + + var inferenceConfig = new InferenceConfiguration(); + BedrockModelUtilities.SetPropertyIfNotNull(() => temperature, value => inferenceConfig.Temperature = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => topP, value => inferenceConfig.TopP = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => maxTokenCount, value => inferenceConfig.MaxTokens = value); + BedrockModelUtilities.SetNullablePropertyIfNotNull(() => stopSequences, value => inferenceConfig.StopSequences = value); + + var converseRequest = new ConverseStreamRequest() + { + ModelId = modelId, + Messages = messages, + System = systemMessages, + InferenceConfig = inferenceConfig, + AdditionalModelRequestFields = new Document(), + AdditionalModelResponseFieldPaths = new List() + }; - var inferenceConfig = new InferenceConfiguration(); - BedrockModelUtilities.SetPropertyIfNotNull(() => temperature, value => inferenceConfig.Temperature = value); - BedrockModelUtilities.SetPropertyIfNotNull(() => topP, value => inferenceConfig.TopP = value); - BedrockModelUtilities.SetPropertyIfNotNull(() => maxTokenCount, value => inferenceConfig.MaxTokens = value); - BedrockModelUtilities.SetNullablePropertyIfNotNull(() => stopSequences, value => inferenceConfig.StopSequences = value); + return converseRequest; + } - var converseRequest = new ConverseStreamRequest() + if (modelId.StartsWith("amazon.nova-", StringComparison.OrdinalIgnoreCase)) { - ModelId = modelId, - Messages = messages, - System = systemMessages, - InferenceConfig = inferenceConfig, - AdditionalModelRequestFields = new Document(), - AdditionalModelResponseFieldPaths = new List() - }; - - return converseRequest; + var executionSettings = AmazonNovaExecutionSettings.FromExecutionSettings(settings); + var schemaVersion = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "schemaVersion") ?? executionSettings.SchemaVersion; + var maxNewTokens = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "max_new_tokens") ?? executionSettings.MaxNewTokens; + var topP = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "top_p") ?? executionSettings.TopP; + var topK = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "top_k") ?? executionSettings.TopK; + var temperature = BedrockModelUtilities.GetExtensionDataValue(settings?.ExtensionData, "temperature") ?? executionSettings.Temperature; + var stopSequences = BedrockModelUtilities.GetExtensionDataValue?>(settings?.ExtensionData, "stopSequences") ?? executionSettings.StopSequences; + + var inferenceConfig = new InferenceConfiguration(); + BedrockModelUtilities.SetPropertyIfNotNull(() => temperature, value => inferenceConfig.Temperature = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => topP, value => inferenceConfig.TopP = value); + BedrockModelUtilities.SetPropertyIfNotNull(() => maxNewTokens, value => inferenceConfig.MaxTokens = value); + BedrockModelUtilities.SetNullablePropertyIfNotNull(() => stopSequences, value => inferenceConfig.StopSequences = value); + + var converseRequest = new ConverseStreamRequest + { + ModelId = modelId, + Messages = messages, + System = systemMessages, + InferenceConfig = inferenceConfig, + AdditionalModelRequestFields = new Document(), + AdditionalModelResponseFieldPaths = new List() + }; + + return converseRequest; + } + + throw new NotSupportedException($"Amazon model '{modelId}' is not supported. Supported models: Titan and Nova."); } } diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaRequest.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaRequest.cs new file mode 100644 index 000000000000..aee9d7058f71 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaRequest.cs @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.ChatCompletion; + +namespace Microsoft.SemanticKernel.Connectors.Nova.Core; + +internal static class NovaRequest +{ + /// + /// The Nova Text Generation Request object. + /// + internal sealed class NovaTextGenerationRequest + { + /// + /// Schema version for the request, defaulting to "messages-v1". + /// + [JsonPropertyName("schemaVersion")] + public string SchemaVersion { get; set; } = "messages-v1"; + + /// + /// System messages providing context for the generation. + /// + [JsonPropertyName("system")] + public IList? System { get; set; } + + /// + /// User messages for text generation. + /// + [JsonPropertyName("messages")] + public IList? Messages { get; set; } + + /// + /// Text generation configurations as required by Nova request body. + /// + [JsonPropertyName("inferenceConfig")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public NovaTextGenerationConfig? InferenceConfig { get; set; } + } + + /// + /// Represents a system message. + /// + internal sealed class NovaSystemMessage + { + /// + /// The text of the system message. + /// + [JsonPropertyName("text")] + public string? Text { get; set; } + } + + /// + /// Represents a user message. + /// + internal sealed class NovaUserMessage + { + /// + /// The role of the message sender. + /// + [JsonPropertyName("role")] + public string? Role { get; set; } + + /// + /// The content of the user message. + /// + [JsonPropertyName("content")] + public IList? Content { get; set; } = new List(); + } + + /// + /// Represents the content of a user message. + /// + internal sealed class NovaUserMessageContent + { + /// + /// The text of the user message content. + /// + [JsonPropertyName("text")] + public string? Text { get; set; } + } + + /// + /// Nova Text Generation Configurations. + /// + internal sealed class NovaTextGenerationConfig + { + /// + /// Maximum new tokens to generate in the response. + /// + [JsonPropertyName("max_new_tokens")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxNewTokens { get; set; } + + /// + /// Top P controls token choices, based on the probability of the potential choices. The range is 0 to 1. The default is 1. + /// + [JsonPropertyName("top_p")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float? TopP { get; set; } + + /// + /// Top K limits the number of token options considered at each generation step. The default is 20. + /// + [JsonPropertyName("top_k")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? TopK { get; set; } + + /// + /// The Temperature value ranges from 0 to 1, with 0 being the most deterministic and 1 being the most creative. + /// + [JsonPropertyName("temperature")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float? Temperature { get; set; } + } +} diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaResponse.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaResponse.cs new file mode 100644 index 000000000000..e1630a309eb0 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Amazon/NovaResponse.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Amazon.Core; + +internal sealed class NovaMessage +{ + internal sealed class Content + { + public string? Text { get; set; } + } + + [JsonPropertyName("content")] + public List? Contents { get; set; } + + public string? Role { get; set; } +} + +internal sealed class Output +{ + public NovaMessage? Message { get; set; } +} + +internal sealed class Usage +{ + public int InputTokens { get; set; } + + public int OutputTokens { get; set; } + + public int TotalTokens { get; set; } +} + +/// +/// The Amazon Titan Text response object when deserialized from Invoke Model call. +/// +internal sealed class NovaTextResponse +{ + public Output? Output { get; set; } + + public Usage? Usage { get; set; } + + public string? StopReason { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Anthropic/AnthropicService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Anthropic/AnthropicService.cs index d28d86b3cd2c..028604890d4e 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Anthropic/AnthropicService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Anthropic/AnthropicService.cs @@ -33,7 +33,7 @@ public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExe } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { using var reader = new StreamReader(response.Body); var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); @@ -120,7 +120,7 @@ public ConverseRequest GetConverseRequest(string modelId, ChatHistory chatHistor } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { var text = chunk["completion"]?.ToString(); if (!string.IsNullOrEmpty(text)) diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandRService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandRService.cs index 433cdc1417f4..8d709fe3b5e9 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandRService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandRService.cs @@ -57,7 +57,7 @@ public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExe } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { using var reader = new StreamReader(response.Body); var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); @@ -140,7 +140,7 @@ public ConverseRequest GetConverseRequest(string modelId, ChatHistory chatHistor } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { var text = chunk["text"]?.ToString(); if (!string.IsNullOrEmpty(text)) diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandService.cs index d2aa154d416f..c91bf88a595b 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Cohere/CohereCommandService.cs @@ -37,7 +37,7 @@ public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExe } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { using var reader = new StreamReader(response.Body); var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); @@ -53,7 +53,7 @@ public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse resp } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { var generations = chunk["generations"]?.AsArray(); if (generations != null) diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Meta/MetaService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Meta/MetaService.cs index 0b6e0ba6d355..87c77c1240d4 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Meta/MetaService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Meta/MetaService.cs @@ -30,7 +30,7 @@ public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExe } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { using var reader = new StreamReader(response.Body); var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); @@ -75,7 +75,7 @@ public ConverseRequest GetConverseRequest(string modelId, ChatHistory chatHistor } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { var generation = chunk["generation"]?.ToString(); if (!string.IsNullOrEmpty(generation)) diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Mistral/MistralService.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Mistral/MistralService.cs index eee8076b5165..0cee7cdeb077 100644 --- a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Mistral/MistralService.cs +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Core/Models/Mistral/MistralService.cs @@ -40,7 +40,7 @@ public object GetInvokeModelRequestBody(string modelId, string prompt, PromptExe } /// - public IReadOnlyList GetInvokeResponseBody(InvokeModelResponse response) + public IReadOnlyList GetInvokeResponseBody(string modelId, InvokeModelResponse response) { using var reader = new StreamReader(response.Body); var responseBody = JsonSerializer.Deserialize(reader.ReadToEnd()); @@ -82,7 +82,7 @@ public ConverseRequest GetConverseRequest(string modelId, ChatHistory chatHistor } /// - public IEnumerable GetTextStreamOutput(JsonNode chunk) + public IEnumerable GetTextStreamOutput(string modelId, JsonNode chunk) { var outputs = chunk["outputs"]?.AsArray(); if (outputs != null) diff --git a/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Settings/AmazonNovaExecutionSettings.cs b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Settings/AmazonNovaExecutionSettings.cs new file mode 100644 index 000000000000..47fd34c782c1 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Amazon/Bedrock/Settings/AmazonNovaExecutionSettings.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.Connectors.Nova; + +/// +/// Prompt execution settings for Nova Text Generation +/// +[JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] +public class AmazonNovaExecutionSettings : PromptExecutionSettings +{ + private string? _schemaVersion; + private int? _maxNewTokens; + private float? _topP; + private int? _topK; + private float? _temperature; + private List? _stopSequences; + + /// + /// Schema version for the execution settings. + /// + [JsonPropertyName("schemaVersion")] + public string? SchemaVersion + { + get => this._schemaVersion; + set + { + this.ThrowIfFrozen(); + this._schemaVersion = value; + } + } + + /// + /// Maximum new tokens to generate in the response. + /// + [JsonPropertyName("max_new_tokens")] + public int? MaxNewTokens + { + get => this._maxNewTokens; + set + { + this.ThrowIfFrozen(); + this._maxNewTokens = value; + } + } + + /// + /// Top P controls token choices, based on the probability of the potential choices. The range is 0 to 1. The default is 1. + /// + [JsonPropertyName("top_p")] + public float? TopP + { + get => this._topP; + set + { + this.ThrowIfFrozen(); + this._topP = value; + } + } + + /// + /// Top K limits the number of token options considered at each generation step. The default is 20. + /// + [JsonPropertyName("top_k")] + public int? TopK + { + get => this._topK; + set + { + this.ThrowIfFrozen(); + this._topK = value; + } + } + + /// + /// The Temperature value ranges from 0 to 1, with 0 being the most deterministic and 1 being the most creative. + /// + [JsonPropertyName("temperature")] + public float? Temperature + { + get => this._temperature; + set + { + this.ThrowIfFrozen(); + this._temperature = value; + } + } + + /// + /// Use | (pipe) characters (maximum 20 characters). + /// + [JsonPropertyName("stopSequences")] + public List? StopSequences + { + get => this._stopSequences; + set + { + this.ThrowIfFrozen(); + this._stopSequences = value; + } + } + + /// + /// Converts PromptExecutionSettings to NovaExecutionSettings + /// + /// The Kernel standard PromptExecutionSettings. + /// Model specific execution settings + public static AmazonNovaExecutionSettings FromExecutionSettings(PromptExecutionSettings? executionSettings) + { + switch (executionSettings) + { + case null: + return new AmazonNovaExecutionSettings(); + case AmazonNovaExecutionSettings settings: + return settings; + } + + var json = JsonSerializer.Serialize(executionSettings); + return JsonSerializer.Deserialize(json, JsonOptionsCache.ReadPermissive)!; + } +}