From 3b2b97649892379c1e0af92d07122fe04892e701 Mon Sep 17 00:00:00 2001 From: Pavel Zhur Date: Thu, 10 Oct 2024 13:12:34 +0300 Subject: [PATCH] vision --- .../Model/Enums/InteractionType.cs | 2 + .../SongsDatabase.cs | 2 + .../Internal/DialogConstants.cs | 4 +- .../Memory/UserImageMessageMemoryPoint.cs | 6 ++ .../OneShelf.Common.OpenAi.csproj | 2 +- .../ServiceCollectionExtensions.cs | 3 +- .../Services/DialogRunner.cs | 13 +++- .../OneShelf.OneDog.Database/DogDatabase.cs | 2 + .../Model/Enums/InteractionType.cs | 2 + .../PipelineHandlers/AiDialogHandler.cs | 10 +-- .../DragonDatabase.cs | 4 +- .../Model/Enums/InteractionType.cs | 1 + .../PipelineHandlers/AiDialogHandler.cs | 16 ++-- .../IInteractionsRepository.cs | 1 + .../PipelineHandlers/AiDialogHandlerBase.cs | 78 +++++++++++++------ .../PipelineHandlers/AiDialogHandler.cs | 12 +-- 16 files changed, 111 insertions(+), 47 deletions(-) create mode 100644 OneShelf.Common/OneShelf.Common.OpenAi/Models/Memory/UserImageMessageMemoryPoint.cs diff --git a/OneShelf.Common/OneShelf.Common.Database.Songs/Model/Enums/InteractionType.cs b/OneShelf.Common/OneShelf.Common.Database.Songs/Model/Enums/InteractionType.cs index ca678e47..8ffba2d4 100644 --- a/OneShelf.Common/OneShelf.Common.Database.Songs/Model/Enums/InteractionType.cs +++ b/OneShelf.Common/OneShelf.Common.Database.Songs/Model/Enums/InteractionType.cs @@ -37,4 +37,6 @@ public enum InteractionType ImagesLimit, ImagesSuccess, + + OwnChatterImageMessage, } \ No newline at end of file diff --git a/OneShelf.Common/OneShelf.Common.Database.Songs/SongsDatabase.cs b/OneShelf.Common/OneShelf.Common.Database.Songs/SongsDatabase.cs index 50086693..665b83a0 100644 --- a/OneShelf.Common/OneShelf.Common.Database.Songs/SongsDatabase.cs +++ b/OneShelf.Common/OneShelf.Common.Database.Songs/SongsDatabase.cs @@ -225,6 +225,8 @@ async Task IInteractionsRepository.Add(List.OwnChatterMessage => InteractionType.OwnChatterMessage; + InteractionType IInteractionsRepository.OwnChatterImageMessage => InteractionType.OwnChatterImageMessage; + InteractionType IInteractionsRepository.OwnChatterMemoryPoint => InteractionType.OwnChatterMemoryPoint; InteractionType IInteractionsRepository.OwnChatterResetDialog => InteractionType.OwnChatterResetDialog; diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/Internal/DialogConstants.cs b/OneShelf.Common/OneShelf.Common.OpenAi/Internal/DialogConstants.cs index 83f7c6b0..2b382d30 100644 --- a/OneShelf.Common/OneShelf.Common.OpenAi/Internal/DialogConstants.cs +++ b/OneShelf.Common/OneShelf.Common.OpenAi/Internal/DialogConstants.cs @@ -16,7 +16,7 @@ internal static class DialogConstants public const string SystemImageOpportunityMessage = $"If you wish to display the images to the user, call the '{IncludeImageFunctionName}' function."; - private static readonly Tool UserChangedTopicTool = new(new( + private static readonly Tool UserChangedTopicTool = new(new Function( UserChangedTopicFunctionName, "Call this function whenever the next user message has no relation to the previous conversation, i.e. it feels like they start the conversation anew.", new JsonObject @@ -25,7 +25,7 @@ internal static class DialogConstants ["properties"] = new JsonObject(), })); - private static readonly Tool IncludeImageTool = new(new( + private static readonly Tool IncludeImageTool = new(new Function( IncludeImageFunctionName, "Call this function if you wish to display pictures or images to the user.", new JsonObject diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/Models/Memory/UserImageMessageMemoryPoint.cs b/OneShelf.Common/OneShelf.Common.OpenAi/Models/Memory/UserImageMessageMemoryPoint.cs new file mode 100644 index 00000000..0df6f789 --- /dev/null +++ b/OneShelf.Common/OneShelf.Common.OpenAi/Models/Memory/UserImageMessageMemoryPoint.cs @@ -0,0 +1,6 @@ +namespace OneShelf.Common.OpenAi.Models.Memory; + +public class UserImageMessageMemoryPoint(string base64) : MemoryPoint +{ + public string Base64 { get; } = base64; +} \ No newline at end of file diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/OneShelf.Common.OpenAi.csproj b/OneShelf.Common/OneShelf.Common.OpenAi/OneShelf.Common.OpenAi.csproj index 97213444..aa7845a2 100644 --- a/OneShelf.Common/OneShelf.Common.OpenAi/OneShelf.Common.OpenAi.csproj +++ b/OneShelf.Common/OneShelf.Common.OpenAi/OneShelf.Common.OpenAi.csproj @@ -9,7 +9,7 @@ - + diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/ServiceCollectionExtensions.cs b/OneShelf.Common/OneShelf.Common.OpenAi/ServiceCollectionExtensions.cs index a423aa72..61f57ac3 100644 --- a/OneShelf.Common/OneShelf.Common.OpenAi/ServiceCollectionExtensions.cs +++ b/OneShelf.Common/OneShelf.Common.OpenAi/ServiceCollectionExtensions.cs @@ -14,7 +14,8 @@ public static IServiceCollection AddOpenAi(this IServiceCollection services, ICo services .AddScoped() - .AddBillingApiClient(configuration); + .AddBillingApiClient(configuration) + .AddHttpClient(); return services; } diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/Services/DialogRunner.cs b/OneShelf.Common/OneShelf.Common.OpenAi/Services/DialogRunner.cs index 99d465c0..f1194370 100644 --- a/OneShelf.Common/OneShelf.Common.OpenAi/Services/DialogRunner.cs +++ b/OneShelf.Common/OneShelf.Common.OpenAi/Services/DialogRunner.cs @@ -198,7 +198,7 @@ private static List RecreateMessages(IReadOnlyList existin new(Role.System, configuration.SystemMessage), }; - foreach (var memoryPoint in existingMemory) + foreach (var (memoryPoint, i) in existingMemory.WithIndices()) { switch (memoryPoint) { @@ -208,6 +208,17 @@ private static List RecreateMessages(IReadOnlyList existin case ChatBotMemoryPoint chatBotMemoryPoint: messages.AddRange(chatBotMemoryPoint.Messages); break; + case UserImageMessageMemoryPoint userImageMessageMemoryPoint: + var imageDetail = existingMemory.Skip(i).SkipWhile(x => x is not ChatBotMemoryPoint).Any(x => x is UserImageMessageMemoryPoint) + ? ImageDetail.Low + : ImageDetail.High; + + messages.Add(new(Role.User, [ + new ImageUrl( + $"data:image/jpeg;base64,{userImageMessageMemoryPoint.Base64}", + imageDetail) + ])); + break; default: throw new ArgumentOutOfRangeException(nameof(memoryPoint)); } diff --git a/OneShelf.OneDog/OneShelf.OneDog.Database/DogDatabase.cs b/OneShelf.OneDog/OneShelf.OneDog.Database/DogDatabase.cs index e8668962..4813dd42 100644 --- a/OneShelf.OneDog/OneShelf.OneDog.Database/DogDatabase.cs +++ b/OneShelf.OneDog/OneShelf.OneDog.Database/DogDatabase.cs @@ -54,6 +54,8 @@ async Task IInteractionsRepository.Add(List.OwnChatterMessage => InteractionType.OwnChatterMessage; + InteractionType IInteractionsRepository.OwnChatterImageMessage => InteractionType.OwnChatterImageMessage; + InteractionType IInteractionsRepository.OwnChatterMemoryPoint => InteractionType.OwnChatterMemoryPoint; InteractionType IInteractionsRepository.OwnChatterResetDialog => InteractionType.OwnChatterResetDialog; diff --git a/OneShelf.OneDog/OneShelf.OneDog.Database/Model/Enums/InteractionType.cs b/OneShelf.OneDog/OneShelf.OneDog.Database/Model/Enums/InteractionType.cs index 7b1fb10a..bca08b0b 100644 --- a/OneShelf.OneDog/OneShelf.OneDog.Database/Model/Enums/InteractionType.cs +++ b/OneShelf.OneDog/OneShelf.OneDog.Database/Model/Enums/InteractionType.cs @@ -13,4 +13,6 @@ public enum InteractionType ImagesSuccess, ImagesLimit, + + OwnChatterImageMessage, } \ No newline at end of file diff --git a/OneShelf.OneDog/OneShelf.OneDog.Processor/Services/PipelineHandlers/AiDialogHandler.cs b/OneShelf.OneDog/OneShelf.OneDog.Processor/Services/PipelineHandlers/AiDialogHandler.cs index 0cc84707..78ac927c 100644 --- a/OneShelf.OneDog/OneShelf.OneDog.Processor/Services/PipelineHandlers/AiDialogHandler.cs +++ b/OneShelf.OneDog/OneShelf.OneDog.Processor/Services/PipelineHandlers/AiDialogHandler.cs @@ -17,13 +17,13 @@ public class AiDialogHandler : AiDialogHandlerBase private readonly DogContext _dogContext; private readonly DogDatabase _dogDatabase; - public AiDialogHandler( - ILogger logger, + public AiDialogHandler(ILogger logger, DogDatabase dogDatabase, - DialogRunner dialogRunner, + DialogRunner dialogRunner, IScopedAbstractions scopedAbstractions, - DogContext dogContext) - : base(scopedAbstractions, logger, dogDatabase, dialogRunner) + DogContext dogContext, + IHttpClientFactory httpClientFactory) + : base(scopedAbstractions, logger, dogDatabase, dialogRunner, httpClientFactory) { _dogDatabase = dogDatabase; _dogContext = dogContext; diff --git a/OneShelf.OneDragon/OneShelf.OneDragon.Database/DragonDatabase.cs b/OneShelf.OneDragon/OneShelf.OneDragon.Database/DragonDatabase.cs index 8c543c66..f6e81459 100644 --- a/OneShelf.OneDragon/OneShelf.OneDragon.Database/DragonDatabase.cs +++ b/OneShelf.OneDragon/OneShelf.OneDragon.Database/DragonDatabase.cs @@ -57,7 +57,9 @@ async Task IInteractionsRepository.Add(List.OwnChatterMessage => InteractionType.AiMessage; - + + InteractionType IInteractionsRepository.OwnChatterImageMessage => InteractionType.AiImageMessage; + InteractionType IInteractionsRepository.OwnChatterMemoryPoint => InteractionType.AiMemoryPoint; InteractionType IInteractionsRepository.OwnChatterResetDialog => InteractionType.AiResetDialog; diff --git a/OneShelf.OneDragon/OneShelf.OneDragon.Database/Model/Enums/InteractionType.cs b/OneShelf.OneDragon/OneShelf.OneDragon.Database/Model/Enums/InteractionType.cs index 42aa5966..052a31ae 100644 --- a/OneShelf.OneDragon/OneShelf.OneDragon.Database/Model/Enums/InteractionType.cs +++ b/OneShelf.OneDragon/OneShelf.OneDragon.Database/Model/Enums/InteractionType.cs @@ -3,6 +3,7 @@ public enum InteractionType { AiMessage, + AiImageMessage, AiMemoryPoint, AiResetDialog, AiImagesLimit, diff --git a/OneShelf.OneDragon/OneShelf.OneDragon.Processor/PipelineHandlers/AiDialogHandler.cs b/OneShelf.OneDragon/OneShelf.OneDragon.Processor/PipelineHandlers/AiDialogHandler.cs index a7c3ff09..b3e01efa 100644 --- a/OneShelf.OneDragon/OneShelf.OneDragon.Processor/PipelineHandlers/AiDialogHandler.cs +++ b/OneShelf.OneDragon/OneShelf.OneDragon.Processor/PipelineHandlers/AiDialogHandler.cs @@ -22,15 +22,15 @@ public class AiDialogHandler : AiDialogHandlerBase private readonly Availability _availability; private readonly IOptions _options; - public AiDialogHandler( - IScopedAbstractions scopedAbstractions, - ILogger> logger, - DialogRunner dialogRunner, + public AiDialogHandler(IScopedAbstractions scopedAbstractions, + ILogger> logger, + DialogRunner dialogRunner, DragonDatabase dragonDatabase, - DragonScope dragonScope, + DragonScope dragonScope, Availability availability, - IOptions options) - : base(scopedAbstractions, logger, dragonDatabase, dialogRunner) + IOptions options, + IHttpClientFactory httpClientFactory) + : base(scopedAbstractions, logger, dragonDatabase, dialogRunner, httpClientFactory) { _dragonDatabase = dragonDatabase; _dragonScope = dragonScope; @@ -76,7 +76,7 @@ protected override bool CheckRelevant(Update update) var textsSince = Since(limits.Max(x => x.Window)); var texts = (await _dragonDatabase.Interactions .Where(x => x.UserId == _dragonScope.UserId && x.ChatId == _dragonScope.ChatId) - .Where(x => x.InteractionType == InteractionType.AiMessage) + .Where(x => x.InteractionType == InteractionType.AiMessage || x.InteractionType == InteractionType.AiImageMessage) .Where(x => x.CreatedOn >= textsSince) .ToListAsync()) .Select(x => x.CreatedOn) diff --git a/OneShelf.Telegram/OneShelf.Telegram.Ai.Model/IInteractionsRepository.cs b/OneShelf.Telegram/OneShelf.Telegram.Ai.Model/IInteractionsRepository.cs index af664db0..32491bb4 100644 --- a/OneShelf.Telegram/OneShelf.Telegram.Ai.Model/IInteractionsRepository.cs +++ b/OneShelf.Telegram/OneShelf.Telegram.Ai.Model/IInteractionsRepository.cs @@ -7,6 +7,7 @@ public interface IInteractionsRepository Task Add(List> interactions); TInteractionType OwnChatterMessage { get; } + TInteractionType OwnChatterImageMessage { get; } TInteractionType OwnChatterMemoryPoint { get; } TInteractionType OwnChatterResetDialog { get; } TInteractionType ImagesLimit { get; } diff --git a/OneShelf.Telegram/OneShelf.Telegram.Ai/PipelineHandlers/AiDialogHandlerBase.cs b/OneShelf.Telegram/OneShelf.Telegram.Ai/PipelineHandlers/AiDialogHandlerBase.cs index d2616beb..ef144f3f 100644 --- a/OneShelf.Telegram/OneShelf.Telegram.Ai/PipelineHandlers/AiDialogHandlerBase.cs +++ b/OneShelf.Telegram/OneShelf.Telegram.Ai/PipelineHandlers/AiDialogHandlerBase.cs @@ -30,24 +30,55 @@ public abstract class AiDialogHandlerBase : PipelineHandler protected readonly ILogger> _logger; protected readonly IInteractionsRepository _repository; protected readonly DialogRunner _dialogRunner; + protected readonly IHttpClientFactory _httpClientFactory; - protected AiDialogHandlerBase(IScopedAbstractions scopedAbstractions, ILogger> logger, IInteractionsRepository repository, DialogRunner dialogRunner) + protected AiDialogHandlerBase(IScopedAbstractions scopedAbstractions, ILogger> logger, IInteractionsRepository repository, DialogRunner dialogRunner, IHttpClientFactory httpClientFactory) : base(scopedAbstractions) { _logger = logger; _repository = repository; _dialogRunner = dialogRunner; + _httpClientFactory = httpClientFactory; } - protected async Task Log(Update update, TInteractionType interactionType) + private async Task Log(Update update) { - var interaction = CreateInteraction(update); - interaction.CreatedOn = DateTime.Now; - interaction.InteractionType = interactionType; - interaction.UserId = update.Message!.From!.Id; - interaction.ShortInfoSerialized = update.Message.Text; - interaction.Serialized = JsonSerializer.Serialize(update); - await _repository.Add(interaction.Once().ToList()); + var text = update.Message?.Text; + if (string.IsNullOrWhiteSpace(text)) + { + text = update.Message?.Caption; + } + + if (string.IsNullOrWhiteSpace(text) && update.Message?.Photo == null) + return false; + + if (!string.IsNullOrWhiteSpace(text)) + { + var interaction = CreateInteraction(update); + interaction.CreatedOn = DateTime.Now; + interaction.UserId = update.Message!.From!.Id; + interaction.Serialized = JsonSerializer.Serialize(update); + interaction.InteractionType = _repository.OwnChatterMessage; + interaction.ShortInfoSerialized = text; + await _repository.Add(interaction.Once().ToList()); + } + + if (update.Message?.Photo != null) + { + var path = (await GetApi().GetFileAsync(update.Message.Photo.OrderByDescending(x => x.Height).First().FileId)).FilePath; + using var client = _httpClientFactory.CreateClient(); + var bytes = await client.GetByteArrayAsync($"https://api.telegram.org/file/bot{ScopedAbstractions.GetBotToken()}/{path}"); + + var interaction = CreateInteraction(update); + interaction.CreatedOn = DateTime.Now; + interaction.UserId = update.Message!.From!.Id; + interaction.Serialized = JsonSerializer.Serialize(update); + interaction.InteractionType = _repository.OwnChatterImageMessage; + interaction.ShortInfoSerialized = Convert.ToBase64String(bytes); + await _repository.Add(interaction.Once().ToList()); + } + + return true; } protected abstract IInteraction CreateInteraction(Update update); @@ -59,7 +90,7 @@ protected async Task CheckNoUpdates(CancellationTokenSource cancellationTokenSou while (!cancellationToken.IsCancellationRequested) { var last = (await _repository.Get(q => q - .Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage)) + .Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage) || Equals(x.InteractionType, _repository.OwnChatterImageMessage)) .OrderByDescending(x => x.Id) .Take(1))) .Single(); @@ -70,7 +101,7 @@ protected async Task CheckNoUpdates(CancellationTokenSource cancellationTokenSou return; } - await Task.Delay(500, cancellationToken); + await Task.Delay(100, cancellationToken); } } catch (TaskCanceledException) @@ -239,9 +270,8 @@ protected override async Task HandleSync(Update update) return true; } - await Log(update, _repository.OwnChatterMessage); - - if (update.Message?.Text?.Length > 0) + var anyContent = await Log(update); + if (anyContent) { Queued(Respond(update)); return true; @@ -404,13 +434,14 @@ protected virtual void OnInitializing(long userId, long chatId) protected virtual bool TraceImages => false; - protected async Task Respond(Update update) + private async Task Respond(Update update) { var now = DateTime.Now; var since = now.AddDays(-1); var interactions = await _repository.Get(q => q .Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage) || + Equals(x.InteractionType, _repository.OwnChatterImageMessage) || Equals(x.InteractionType, _repository.OwnChatterMemoryPoint) || Equals(x.InteractionType, _repository.OwnChatterResetDialog)) .Where(x => x.ShortInfoSerialized!.Length > 0 || Equals(x.InteractionType, _repository.OwnChatterResetDialog)) @@ -434,20 +465,23 @@ protected async Task Respond(Update update) using var checkingIsStillLast = new CancellationTokenSource(); ValueHolder isPhoto = new(); LongTyping(update.Message!.Chat.Id, update.Message.MessageThreadId, callingApis.Token, isPhoto); - var checking = CheckNoUpdates(checkingIsStillLast, callingApis.Token, interactions.Last(x => Equals(x.InteractionType, _repository.OwnChatterMessage)).Id); + var checking = CheckNoUpdates(checkingIsStillLast, callingApis.Token, interactions.Last(x => Equals(x.InteractionType, _repository.OwnChatterMessage) || Equals(x.InteractionType, _repository.OwnChatterImageMessage)).Id); ChatBotMemoryPointWithTraces newMessagePoint; DialogResult result; try { + await Task.Delay(update.Message.MediaGroupId != null || update.Message.Photo != null ? 500 : 220, checkingIsStillLast.Token); + var existingMemory = interactions.Select(i => - Equals(i.InteractionType, _repository.OwnChatterMessage) - ? - (MemoryPoint)new UserMessageMemoryPoint(i.ShortInfoSerialized!) - : Equals(i.InteractionType, _repository.OwnChatterMemoryPoint) - ? JsonSerializer.Deserialize(i.Serialized)! - : throw new ArgumentOutOfRangeException(nameof(i))).ToList(); + Equals(i.InteractionType, _repository.OwnChatterImageMessage) + ? (MemoryPoint)new UserImageMessageMemoryPoint(i.ShortInfoSerialized!) + : Equals(i.InteractionType, _repository.OwnChatterMessage) + ? new UserMessageMemoryPoint(i.ShortInfoSerialized!) + : Equals(i.InteractionType, _repository.OwnChatterMemoryPoint) + ? JsonSerializer.Deserialize(i.Serialized)! + : throw new ArgumentOutOfRangeException(nameof(i))).ToList(); if (checkingIsStillLast.IsCancellationRequested) { diff --git a/OneShelf.Telegram/OneShelf.Telegram.Processor/Services/PipelineHandlers/AiDialogHandler.cs b/OneShelf.Telegram/OneShelf.Telegram.Processor/Services/PipelineHandlers/AiDialogHandler.cs index 559f72d0..4c8be852 100644 --- a/OneShelf.Telegram/OneShelf.Telegram.Processor/Services/PipelineHandlers/AiDialogHandler.cs +++ b/OneShelf.Telegram/OneShelf.Telegram.Processor/Services/PipelineHandlers/AiDialogHandler.cs @@ -19,13 +19,13 @@ public class AiDialogHandler : AiDialogHandlerBase private readonly SongsDatabase _songsDatabase; private new readonly TelegramOptions _telegramOptions; - public AiDialogHandler( - ILogger logger, + public AiDialogHandler(ILogger logger, IOptions telegramOptions, SongsDatabase songsDatabase, - DialogRunner dialogRunner, - IScopedAbstractions scopedAbstractions) - : base(scopedAbstractions, logger, songsDatabase, dialogRunner) + DialogRunner dialogRunner, + IScopedAbstractions scopedAbstractions, + IHttpClientFactory httpClientFactory) + : base(scopedAbstractions, logger, songsDatabase, dialogRunner, httpClientFactory) { _songsDatabase = songsDatabase; _telegramOptions = telegramOptions.Value; @@ -40,7 +40,7 @@ protected override bool CheckRelevant(Update update) } protected override IInteraction CreateInteraction(Update update) => new Interaction(); - + protected override (string? additionalBillingInfo, int? domainId) GetDialogConfigurationParameters() => default; protected override async Task GetImagesUnavailableUntil(DateTime now) => null;