pavel-zhur · pavel-zhur · Oct 11, 2024 · Oct 10, 2024
diff --git a/OneShelf.Common/OneShelf.Common.Database.Songs/Model/Enums/InteractionType.cs b/OneShelf.Common/OneShelf.Common.Database.Songs/Model/Enums/InteractionType.cs
@@ -37,4 +37,6 @@ public enum InteractionType
 
     ImagesLimit,
     ImagesSuccess,
+
+    OwnChatterImageMessage,
 }
diff --git a/OneShelf.Common/OneShelf.Common.Database.Songs/SongsDatabase.cs b/OneShelf.Common/OneShelf.Common.Database.Songs/SongsDatabase.cs
@@ -225,6 +225,8 @@ async Task IInteractionsRepository<InteractionType>.Add(List<IInteraction<Intera
 
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterMessage => InteractionType.OwnChatterMessage;
 
+    InteractionType IInteractionsRepository<InteractionType>.OwnChatterImageMessage => InteractionType.OwnChatterImageMessage;
+
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterMemoryPoint => InteractionType.OwnChatterMemoryPoint;
 
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterResetDialog => InteractionType.OwnChatterResetDialog;

diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/Internal/DialogConstants.cs b/OneShelf.Common/OneShelf.Common.OpenAi/Internal/DialogConstants.cs
@@ -16,7 +16,7 @@ internal static class DialogConstants
     public const string SystemImageOpportunityMessage =
         $"If you wish to display the images to the user, call the '{IncludeImageFunctionName}' function.";
 
-    private static readonly Tool UserChangedTopicTool = new(new(
+    private static readonly Tool UserChangedTopicTool = new(new Function(
         UserChangedTopicFunctionName,
         "Call this function whenever the next user message has no relation to the previous conversation, i.e. it feels like they start the conversation anew.",
         new JsonObject
@@ -25,7 +25,7 @@ internal static class DialogConstants
             ["properties"] = new JsonObject(),
         }));
 
-    private static readonly Tool IncludeImageTool = new(new(
+    private static readonly Tool IncludeImageTool = new(new Function(
         IncludeImageFunctionName,
         "Call this function if you wish to display pictures or images to the user.",
         new JsonObject

diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/Models/Memory/UserImageMessageMemoryPoint.cs b/OneShelf.Common/OneShelf.Common.OpenAi/Models/Memory/UserImageMessageMemoryPoint.cs
@@ -0,0 +1,6 @@
+namespace OneShelf.Common.OpenAi.Models.Memory;
+
+public class UserImageMessageMemoryPoint(string base64) : MemoryPoint
+{
+    public string Base64 { get; } = base64;
+}
diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/OneShelf.Common.OpenAi.csproj b/OneShelf.Common/OneShelf.Common.OpenAi/OneShelf.Common.OpenAi.csproj
@@ -9,7 +9,7 @@
 
 	<ItemGroup>
 		<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
-		<PackageReference Include="OpenAI-DotNet" Version="7.4.0" />
+		<PackageReference Include="OpenAI-DotNet" Version="8.3.0" />
 	</ItemGroup>
 
 	<ItemGroup>

diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/ServiceCollectionExtensions.cs b/OneShelf.Common/OneShelf.Common.OpenAi/ServiceCollectionExtensions.cs
@@ -14,7 +14,8 @@ public static IServiceCollection AddOpenAi(this IServiceCollection services, ICo
 
         services
             .AddScoped<DialogRunner>()
-            .AddBillingApiClient(configuration);
+            .AddBillingApiClient(configuration)
+            .AddHttpClient();
 
         return services;
     }

diff --git a/OneShelf.Common/OneShelf.Common.OpenAi/Services/DialogRunner.cs b/OneShelf.Common/OneShelf.Common.OpenAi/Services/DialogRunner.cs
@@ -198,7 +198,7 @@ private static List<Message> RecreateMessages(IReadOnlyList<MemoryPoint> existin
             new(Role.System, configuration.SystemMessage),
         };
 
-        foreach (var memoryPoint in existingMemory)
+        foreach (var (memoryPoint, i) in existingMemory.WithIndices())
         {
             switch (memoryPoint)
             {
@@ -208,6 +208,17 @@ private static List<Message> RecreateMessages(IReadOnlyList<MemoryPoint> existin
                 case ChatBotMemoryPoint chatBotMemoryPoint:
                     messages.AddRange(chatBotMemoryPoint.Messages);
                     break;
+                case UserImageMessageMemoryPoint userImageMessageMemoryPoint:
+                    var imageDetail = existingMemory.Skip(i).SkipWhile(x => x is not ChatBotMemoryPoint).Any(x => x is UserImageMessageMemoryPoint)
+                        ? ImageDetail.Low
+                        : ImageDetail.High;
+
+                    messages.Add(new(Role.User, [
+                        new ImageUrl(
+                            $"data:image/jpeg;base64,{userImageMessageMemoryPoint.Base64}",
+                            imageDetail)
+                    ]));
+                    break;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(memoryPoint));
             }

diff --git a/OneShelf.OneDog/OneShelf.OneDog.Database/DogDatabase.cs b/OneShelf.OneDog/OneShelf.OneDog.Database/DogDatabase.cs
@@ -54,6 +54,8 @@ async Task IInteractionsRepository<InteractionType>.Add(List<IInteraction<Intera
 
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterMessage => InteractionType.OwnChatterMessage;
 
+    InteractionType IInteractionsRepository<InteractionType>.OwnChatterImageMessage => InteractionType.OwnChatterImageMessage;
+
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterMemoryPoint => InteractionType.OwnChatterMemoryPoint;
 
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterResetDialog => InteractionType.OwnChatterResetDialog;

diff --git a/OneShelf.OneDog/OneShelf.OneDog.Database/Model/Enums/InteractionType.cs b/OneShelf.OneDog/OneShelf.OneDog.Database/Model/Enums/InteractionType.cs
@@ -13,4 +13,6 @@ public enum InteractionType
     ImagesSuccess,
 
     ImagesLimit,
+
+    OwnChatterImageMessage,
 }
diff --git a/OneShelf.OneDog/OneShelf.OneDog.Processor/Services/PipelineHandlers/AiDialogHandler.cs b/OneShelf.OneDog/OneShelf.OneDog.Processor/Services/PipelineHandlers/AiDialogHandler.cs
@@ -17,13 +17,13 @@
     private readonly DogContext _dogContext;
     private readonly DogDatabase _dogDatabase;
 
-    public AiDialogHandler(
-        ILogger<AiDialogHandler> logger,
+    public AiDialogHandler(ILogger<AiDialogHandler> logger,
         DogDatabase dogDatabase,
-        DialogRunner dialogRunner, 
+        DialogRunner dialogRunner,
         IScopedAbstractions scopedAbstractions,
-        DogContext dogContext)
-        : base(scopedAbstractions, logger, dogDatabase, dialogRunner)
+        DogContext dogContext, 
+        IHttpClientFactory httpClientFactory)
+        : base(scopedAbstractions, logger, dogDatabase, dialogRunner, httpClientFactory)
     {
         _dogDatabase = dogDatabase;
         _dogContext = dogContext;
@@ -78,11 +78,11 @@
        return imagesUnavailableUntil;
    }

    protected override async Task<DateTime?> GetChatUnavailableUntil() => null;

    protected override string UnavailableUntilTemplate => throw new InvalidOperationException();

    protected override async Task<(string? system, string? version, float? frequencyPenalty, float? presencePenalty, int? imagesVersion)> GetAiParameters()
    {
        var system = _dogContext.Domain.SystemMessage;
        var version = _dogContext.Domain.GptVersion;

diff --git a/OneShelf.OneDragon/OneShelf.OneDragon.Database/DragonDatabase.cs b/OneShelf.OneDragon/OneShelf.OneDragon.Database/DragonDatabase.cs
@@ -57,7 +57,9 @@ async Task IInteractionsRepository<InteractionType>.Add(List<IInteraction<Intera
     }
 
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterMessage => InteractionType.AiMessage;
-
+
+    InteractionType IInteractionsRepository<InteractionType>.OwnChatterImageMessage => InteractionType.AiImageMessage;
+
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterMemoryPoint => InteractionType.AiMemoryPoint;
 
     InteractionType IInteractionsRepository<InteractionType>.OwnChatterResetDialog => InteractionType.AiResetDialog;

diff --git a/OneShelf.OneDragon/OneShelf.OneDragon.Database/Model/Enums/InteractionType.cs b/OneShelf.OneDragon/OneShelf.OneDragon.Database/Model/Enums/InteractionType.cs
@@ -3,6 +3,7 @@
 public enum InteractionType
 {
     AiMessage,
+    AiImageMessage,
     AiMemoryPoint,
     AiResetDialog,
     AiImagesLimit,

diff --git a/OneShelf.OneDragon/OneShelf.OneDragon.Processor/PipelineHandlers/AiDialogHandler.cs b/OneShelf.OneDragon/OneShelf.OneDragon.Processor/PipelineHandlers/AiDialogHandler.cs
@@ -22,15 +22,15 @@ public class AiDialogHandler : AiDialogHandlerBase<InteractionType>
     private readonly Availability _availability;
     private readonly IOptions<TelegramOptions> _options;
 
-    public AiDialogHandler(
-        IScopedAbstractions scopedAbstractions,
-        ILogger<AiDialogHandlerBase<InteractionType>> logger, 
-        DialogRunner dialogRunner, 
+    public AiDialogHandler(IScopedAbstractions scopedAbstractions,
+        ILogger<AiDialogHandlerBase<InteractionType>> logger,
+        DialogRunner dialogRunner,
         DragonDatabase dragonDatabase,
-        DragonScope dragonScope, 
+        DragonScope dragonScope,
         Availability availability,
-        IOptions<TelegramOptions> options)
-        : base(scopedAbstractions, logger, dragonDatabase, dialogRunner)
+        IOptions<TelegramOptions> options, 
+        IHttpClientFactory httpClientFactory)
+        : base(scopedAbstractions, logger, dragonDatabase, dialogRunner, httpClientFactory)
     {
         _dragonDatabase = dragonDatabase;
         _dragonScope = dragonScope;
@@ -76,7 +76,7 @@ protected override bool CheckRelevant(Update update)
         var textsSince = Since(limits.Max(x => x.Window));
         var texts = (await _dragonDatabase.Interactions
                 .Where(x => x.UserId == _dragonScope.UserId && x.ChatId == _dragonScope.ChatId)
-                .Where(x => x.InteractionType == InteractionType.AiMessage)
+                .Where(x => x.InteractionType == InteractionType.AiMessage || x.InteractionType == InteractionType.AiImageMessage)
                 .Where(x => x.CreatedOn >= textsSince)
                 .ToListAsync())
             .Select(x => x.CreatedOn)

diff --git a/OneShelf.Telegram/OneShelf.Telegram.Ai.Model/IInteractionsRepository.cs b/OneShelf.Telegram/OneShelf.Telegram.Ai.Model/IInteractionsRepository.cs
@@ -7,6 +7,7 @@ public interface IInteractionsRepository<TInteractionType>
     Task Add(List<IInteraction<TInteractionType>> interactions);
 
     TInteractionType OwnChatterMessage { get; }
+    TInteractionType OwnChatterImageMessage { get; }
     TInteractionType OwnChatterMemoryPoint { get; }
     TInteractionType OwnChatterResetDialog { get; }
     TInteractionType ImagesLimit { get; }

diff --git a/OneShelf.Telegram/OneShelf.Telegram.Ai/PipelineHandlers/AiDialogHandlerBase.cs b/OneShelf.Telegram/OneShelf.Telegram.Ai/PipelineHandlers/AiDialogHandlerBase.cs
@@ -30,24 +30,55 @@ public abstract class AiDialogHandlerBase<TInteractionType> : PipelineHandler
     protected readonly ILogger<AiDialogHandlerBase<TInteractionType>> _logger;
     protected readonly IInteractionsRepository<TInteractionType> _repository;
     protected readonly DialogRunner _dialogRunner;
+    protected readonly IHttpClientFactory _httpClientFactory;
 
-    protected AiDialogHandlerBase(IScopedAbstractions scopedAbstractions, ILogger<AiDialogHandlerBase<TInteractionType>> logger, IInteractionsRepository<TInteractionType> repository, DialogRunner dialogRunner) 
+    protected AiDialogHandlerBase(IScopedAbstractions scopedAbstractions, ILogger<AiDialogHandlerBase<TInteractionType>> logger, IInteractionsRepository<TInteractionType> repository, DialogRunner dialogRunner, IHttpClientFactory httpClientFactory) 
         : base(scopedAbstractions)
     {
         _logger = logger;
         _repository = repository;
         _dialogRunner = dialogRunner;
+        _httpClientFactory = httpClientFactory;
     }
 
-    protected async Task Log(Update update, TInteractionType interactionType)
+    private async Task<bool> Log(Update update)
     {
-        var interaction = CreateInteraction(update);
-        interaction.CreatedOn = DateTime.Now;
-        interaction.InteractionType = interactionType;
-        interaction.UserId = update.Message!.From!.Id;
-        interaction.ShortInfoSerialized = update.Message.Text;
-        interaction.Serialized = JsonSerializer.Serialize(update);
-        await _repository.Add(interaction.Once().ToList());
+        var text = update.Message?.Text;
+        if (string.IsNullOrWhiteSpace(text))
+        {
+            text = update.Message?.Caption;
+        }
+
+        if (string.IsNullOrWhiteSpace(text) && update.Message?.Photo == null)
+            return false;
+
+        if (!string.IsNullOrWhiteSpace(text))
+        {
+            var interaction = CreateInteraction(update);
+            interaction.CreatedOn = DateTime.Now;
+            interaction.UserId = update.Message!.From!.Id;
+            interaction.Serialized = JsonSerializer.Serialize(update);
+            interaction.InteractionType = _repository.OwnChatterMessage;
+            interaction.ShortInfoSerialized = text;
+            await _repository.Add(interaction.Once().ToList());
+        }
+
+        if (update.Message?.Photo != null)
+        {
+            var path = (await GetApi().GetFileAsync(update.Message.Photo.OrderByDescending(x => x.Height).First().FileId)).FilePath;
+            using var client = _httpClientFactory.CreateClient();
+            var bytes = await client.GetByteArrayAsync($"https://api.telegram.org/file/bot{ScopedAbstractions.GetBotToken()}/{path}");
+
+            var interaction = CreateInteraction(update);
+            interaction.CreatedOn = DateTime.Now;
+            interaction.UserId = update.Message!.From!.Id;
+            interaction.Serialized = JsonSerializer.Serialize(update);
+            interaction.InteractionType = _repository.OwnChatterImageMessage;
+            interaction.ShortInfoSerialized = Convert.ToBase64String(bytes);
+            await _repository.Add(interaction.Once().ToList());
+        }
+
+        return true;
     }
 
     protected abstract IInteraction<TInteractionType> CreateInteraction(Update update);
@@ -59,7 +90,7 @@ protected async Task CheckNoUpdates(CancellationTokenSource cancellationTokenSou
             while (!cancellationToken.IsCancellationRequested)
             {
                 var last = (await _repository.Get(q => q
-                        .Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage))
+                        .Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage) || Equals(x.InteractionType, _repository.OwnChatterImageMessage))
                         .OrderByDescending(x => x.Id)
                         .Take(1)))
                     .Single();
@@ -70,7 +101,7 @@ protected async Task CheckNoUpdates(CancellationTokenSource cancellationTokenSou
                     return;
                 }
 
-                await Task.Delay(500, cancellationToken);
+                await Task.Delay(100, cancellationToken);
             }
         }
         catch (TaskCanceledException)
@@ -239,9 +270,8 @@ protected override async Task<bool> HandleSync(Update update)
             return true;
         }
 
-        await Log(update, _repository.OwnChatterMessage);
-
-        if (update.Message?.Text?.Length > 0)
+        var anyContent = await Log(update);
+        if (anyContent)
         {
             Queued(Respond(update));
             return true;
@@ -404,13 +434,14 @@ protected virtual void OnInitializing(long userId, long chatId)
 
     protected virtual bool TraceImages => false;
 
-    protected async Task Respond(Update update)
+    private async Task Respond(Update update)
     {
         var now = DateTime.Now;
         var since = now.AddDays(-1);
 
         var interactions = await _repository.Get(q => q
             .Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage) ||
+                        Equals(x.InteractionType, _repository.OwnChatterImageMessage) ||
                         Equals(x.InteractionType, _repository.OwnChatterMemoryPoint) ||
                         Equals(x.InteractionType, _repository.OwnChatterResetDialog))
             .Where(x => x.ShortInfoSerialized!.Length > 0 || Equals(x.InteractionType, _repository.OwnChatterResetDialog))
@@ -434,20 +465,23 @@ protected async Task Respond(Update update)
         using var checkingIsStillLast = new CancellationTokenSource();
         ValueHolder<bool> isPhoto = new();
         LongTyping(update.Message!.Chat.Id, update.Message.MessageThreadId, callingApis.Token, isPhoto);
-        var checking = CheckNoUpdates(checkingIsStillLast, callingApis.Token, interactions.Last(x => Equals(x.InteractionType, _repository.OwnChatterMessage)).Id);
+        var checking = CheckNoUpdates(checkingIsStillLast, callingApis.Token, interactions.Last(x => Equals(x.InteractionType, _repository.OwnChatterMessage) || Equals(x.InteractionType, _repository.OwnChatterImageMessage)).Id);
 
         ChatBotMemoryPointWithTraces newMessagePoint;
         DialogResult result;
 
         try
         {
+            await Task.Delay(update.Message.MediaGroupId != null || update.Message.Photo != null ? 500 : 220, checkingIsStillLast.Token);
+
             var existingMemory = interactions.Select(i =>
-                Equals(i.InteractionType, _repository.OwnChatterMessage)
-                    ?
-                    (MemoryPoint)new UserMessageMemoryPoint(i.ShortInfoSerialized!)
-                    : Equals(i.InteractionType, _repository.OwnChatterMemoryPoint)
-                        ? JsonSerializer.Deserialize<ChatBotMemoryPoint>(i.Serialized)!
-                        : throw new ArgumentOutOfRangeException(nameof(i))).ToList();
+                Equals(i.InteractionType, _repository.OwnChatterImageMessage)
+                    ? (MemoryPoint)new UserImageMessageMemoryPoint(i.ShortInfoSerialized!)
+                    : Equals(i.InteractionType, _repository.OwnChatterMessage)
+                        ? new UserMessageMemoryPoint(i.ShortInfoSerialized!)
+                        : Equals(i.InteractionType, _repository.OwnChatterMemoryPoint)
+                            ? JsonSerializer.Deserialize<ChatBotMemoryPoint>(i.Serialized)!
+                            : throw new ArgumentOutOfRangeException(nameof(i))).ToList();
 
             if (checkingIsStillLast.IsCancellationRequested)
             {

diff --git a/OneShelf.Telegram/OneShelf.Telegram.Processor/Services/PipelineHandlers/AiDialogHandler.cs b/OneShelf.Telegram/OneShelf.Telegram.Processor/Services/PipelineHandlers/AiDialogHandler.cs
@@ -19,13 +19,13 @@ public class AiDialogHandler : AiDialogHandlerBase<InteractionType>
     private readonly SongsDatabase _songsDatabase;
     private new readonly TelegramOptions _telegramOptions;
 
-    public AiDialogHandler(
-        ILogger<AiDialogHandler> logger,
+    public AiDialogHandler(ILogger<AiDialogHandler> logger,
         IOptions<TelegramOptions> telegramOptions,
         SongsDatabase songsDatabase,
-        DialogRunner dialogRunner, 
-        IScopedAbstractions scopedAbstractions)
-        : base(scopedAbstractions, logger, songsDatabase, dialogRunner)
+        DialogRunner dialogRunner,
+        IScopedAbstractions scopedAbstractions, 
+        IHttpClientFactory httpClientFactory)
+        : base(scopedAbstractions, logger, songsDatabase, dialogRunner, httpClientFactory)
     {
         _songsDatabase = songsDatabase;
         _telegramOptions = telegramOptions.Value;
@@ -40,7 +40,7 @@ protected override bool CheckRelevant(Update update)
     }
 
     protected override IInteraction<InteractionType> CreateInteraction(Update update) => new Interaction();
-
+    
     protected override (string? additionalBillingInfo, int? domainId) GetDialogConfigurationParameters() => default;
 
     protected override async Task<DateTime?> GetImagesUnavailableUntil(DateTime now) => null;