Skip to content

Commit

Permalink
vision
Browse files Browse the repository at this point in the history
  • Loading branch information
pavel-zhur committed Oct 10, 2024
1 parent 1e72ae5 commit 3b2b976
Show file tree
Hide file tree
Showing 16 changed files with 111 additions and 47 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,6 @@ public enum InteractionType

ImagesLimit,
ImagesSuccess,

OwnChatterImageMessage,
}
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,8 @@ async Task IInteractionsRepository<InteractionType>.Add(List<IInteraction<Intera

InteractionType IInteractionsRepository<InteractionType>.OwnChatterMessage => InteractionType.OwnChatterMessage;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterImageMessage => InteractionType.OwnChatterImageMessage;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterMemoryPoint => InteractionType.OwnChatterMemoryPoint;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterResetDialog => InteractionType.OwnChatterResetDialog;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ internal static class DialogConstants
public const string SystemImageOpportunityMessage =
$"If you wish to display the images to the user, call the '{IncludeImageFunctionName}' function.";

private static readonly Tool UserChangedTopicTool = new(new(
private static readonly Tool UserChangedTopicTool = new(new Function(
UserChangedTopicFunctionName,
"Call this function whenever the next user message has no relation to the previous conversation, i.e. it feels like they start the conversation anew.",
new JsonObject
Expand All @@ -25,7 +25,7 @@ internal static class DialogConstants
["properties"] = new JsonObject(),
}));

private static readonly Tool IncludeImageTool = new(new(
private static readonly Tool IncludeImageTool = new(new Function(
IncludeImageFunctionName,
"Call this function if you wish to display pictures or images to the user.",
new JsonObject
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
namespace OneShelf.Common.OpenAi.Models.Memory;

public class UserImageMessageMemoryPoint(string base64) : MemoryPoint
{
public string Base64 { get; } = base64;
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="OpenAI-DotNet" Version="7.4.0" />
<PackageReference Include="OpenAI-DotNet" Version="8.3.0" />
</ItemGroup>

<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ public static IServiceCollection AddOpenAi(this IServiceCollection services, ICo

services
.AddScoped<DialogRunner>()
.AddBillingApiClient(configuration);
.AddBillingApiClient(configuration)
.AddHttpClient();

return services;
}
Expand Down
13 changes: 12 additions & 1 deletion OneShelf.Common/OneShelf.Common.OpenAi/Services/DialogRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ private static List<Message> RecreateMessages(IReadOnlyList<MemoryPoint> existin
new(Role.System, configuration.SystemMessage),
};

foreach (var memoryPoint in existingMemory)
foreach (var (memoryPoint, i) in existingMemory.WithIndices())
{
switch (memoryPoint)
{
Expand All @@ -208,6 +208,17 @@ private static List<Message> RecreateMessages(IReadOnlyList<MemoryPoint> existin
case ChatBotMemoryPoint chatBotMemoryPoint:
messages.AddRange(chatBotMemoryPoint.Messages);
break;
case UserImageMessageMemoryPoint userImageMessageMemoryPoint:
var imageDetail = existingMemory.Skip(i).SkipWhile(x => x is not ChatBotMemoryPoint).Any(x => x is UserImageMessageMemoryPoint)
? ImageDetail.Low
: ImageDetail.High;

messages.Add(new(Role.User, [
new ImageUrl(
$"data:image/jpeg;base64,{userImageMessageMemoryPoint.Base64}",
imageDetail)
]));
break;
default:
throw new ArgumentOutOfRangeException(nameof(memoryPoint));
}
Expand Down
2 changes: 2 additions & 0 deletions OneShelf.OneDog/OneShelf.OneDog.Database/DogDatabase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ async Task IInteractionsRepository<InteractionType>.Add(List<IInteraction<Intera

InteractionType IInteractionsRepository<InteractionType>.OwnChatterMessage => InteractionType.OwnChatterMessage;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterImageMessage => InteractionType.OwnChatterImageMessage;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterMemoryPoint => InteractionType.OwnChatterMemoryPoint;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterResetDialog => InteractionType.OwnChatterResetDialog;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,6 @@ public enum InteractionType
ImagesSuccess,

ImagesLimit,

OwnChatterImageMessage,
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ public class AiDialogHandler : AiDialogHandlerBase<InteractionType>
private readonly DogContext _dogContext;
private readonly DogDatabase _dogDatabase;

public AiDialogHandler(
ILogger<AiDialogHandler> logger,
public AiDialogHandler(ILogger<AiDialogHandler> logger,
DogDatabase dogDatabase,
DialogRunner dialogRunner,
DialogRunner dialogRunner,
IScopedAbstractions scopedAbstractions,
DogContext dogContext)
: base(scopedAbstractions, logger, dogDatabase, dialogRunner)
DogContext dogContext,
IHttpClientFactory httpClientFactory)
: base(scopedAbstractions, logger, dogDatabase, dialogRunner, httpClientFactory)
{
_dogDatabase = dogDatabase;
_dogContext = dogContext;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ async Task IInteractionsRepository<InteractionType>.Add(List<IInteraction<Intera
}

InteractionType IInteractionsRepository<InteractionType>.OwnChatterMessage => InteractionType.AiMessage;


InteractionType IInteractionsRepository<InteractionType>.OwnChatterImageMessage => InteractionType.AiImageMessage;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterMemoryPoint => InteractionType.AiMemoryPoint;

InteractionType IInteractionsRepository<InteractionType>.OwnChatterResetDialog => InteractionType.AiResetDialog;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
public enum InteractionType
{
AiMessage,
AiImageMessage,
AiMemoryPoint,
AiResetDialog,
AiImagesLimit,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ public class AiDialogHandler : AiDialogHandlerBase<InteractionType>
private readonly Availability _availability;
private readonly IOptions<TelegramOptions> _options;

public AiDialogHandler(
IScopedAbstractions scopedAbstractions,
ILogger<AiDialogHandlerBase<InteractionType>> logger,
DialogRunner dialogRunner,
public AiDialogHandler(IScopedAbstractions scopedAbstractions,
ILogger<AiDialogHandlerBase<InteractionType>> logger,
DialogRunner dialogRunner,
DragonDatabase dragonDatabase,
DragonScope dragonScope,
DragonScope dragonScope,
Availability availability,
IOptions<TelegramOptions> options)
: base(scopedAbstractions, logger, dragonDatabase, dialogRunner)
IOptions<TelegramOptions> options,
IHttpClientFactory httpClientFactory)
: base(scopedAbstractions, logger, dragonDatabase, dialogRunner, httpClientFactory)
{
_dragonDatabase = dragonDatabase;
_dragonScope = dragonScope;
Expand Down Expand Up @@ -76,7 +76,7 @@ protected override bool CheckRelevant(Update update)
var textsSince = Since(limits.Max(x => x.Window));
var texts = (await _dragonDatabase.Interactions
.Where(x => x.UserId == _dragonScope.UserId && x.ChatId == _dragonScope.ChatId)
.Where(x => x.InteractionType == InteractionType.AiMessage)
.Where(x => x.InteractionType == InteractionType.AiMessage || x.InteractionType == InteractionType.AiImageMessage)
.Where(x => x.CreatedOn >= textsSince)
.ToListAsync())
.Select(x => x.CreatedOn)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ public interface IInteractionsRepository<TInteractionType>
Task Add(List<IInteraction<TInteractionType>> interactions);

TInteractionType OwnChatterMessage { get; }
TInteractionType OwnChatterImageMessage { get; }
TInteractionType OwnChatterMemoryPoint { get; }
TInteractionType OwnChatterResetDialog { get; }
TInteractionType ImagesLimit { get; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,55 @@ public abstract class AiDialogHandlerBase<TInteractionType> : PipelineHandler
protected readonly ILogger<AiDialogHandlerBase<TInteractionType>> _logger;
protected readonly IInteractionsRepository<TInteractionType> _repository;
protected readonly DialogRunner _dialogRunner;
protected readonly IHttpClientFactory _httpClientFactory;

protected AiDialogHandlerBase(IScopedAbstractions scopedAbstractions, ILogger<AiDialogHandlerBase<TInteractionType>> logger, IInteractionsRepository<TInteractionType> repository, DialogRunner dialogRunner)
protected AiDialogHandlerBase(IScopedAbstractions scopedAbstractions, ILogger<AiDialogHandlerBase<TInteractionType>> logger, IInteractionsRepository<TInteractionType> repository, DialogRunner dialogRunner, IHttpClientFactory httpClientFactory)
: base(scopedAbstractions)
{
_logger = logger;
_repository = repository;
_dialogRunner = dialogRunner;
_httpClientFactory = httpClientFactory;
}

protected async Task Log(Update update, TInteractionType interactionType)
private async Task<bool> Log(Update update)
{
var interaction = CreateInteraction(update);
interaction.CreatedOn = DateTime.Now;
interaction.InteractionType = interactionType;
interaction.UserId = update.Message!.From!.Id;
interaction.ShortInfoSerialized = update.Message.Text;
interaction.Serialized = JsonSerializer.Serialize(update);
await _repository.Add(interaction.Once().ToList());
var text = update.Message?.Text;
if (string.IsNullOrWhiteSpace(text))
{
text = update.Message?.Caption;
}

if (string.IsNullOrWhiteSpace(text) && update.Message?.Photo == null)
return false;

if (!string.IsNullOrWhiteSpace(text))
{
var interaction = CreateInteraction(update);
interaction.CreatedOn = DateTime.Now;
interaction.UserId = update.Message!.From!.Id;
interaction.Serialized = JsonSerializer.Serialize(update);
interaction.InteractionType = _repository.OwnChatterMessage;
interaction.ShortInfoSerialized = text;
await _repository.Add(interaction.Once().ToList());
}

if (update.Message?.Photo != null)
{
var path = (await GetApi().GetFileAsync(update.Message.Photo.OrderByDescending(x => x.Height).First().FileId)).FilePath;
using var client = _httpClientFactory.CreateClient();
var bytes = await client.GetByteArrayAsync($"https://api.telegram.org/file/bot{ScopedAbstractions.GetBotToken()}/{path}");

var interaction = CreateInteraction(update);
interaction.CreatedOn = DateTime.Now;
interaction.UserId = update.Message!.From!.Id;
interaction.Serialized = JsonSerializer.Serialize(update);
interaction.InteractionType = _repository.OwnChatterImageMessage;
interaction.ShortInfoSerialized = Convert.ToBase64String(bytes);
await _repository.Add(interaction.Once().ToList());
}

return true;
}

protected abstract IInteraction<TInteractionType> CreateInteraction(Update update);
Expand All @@ -59,7 +90,7 @@ protected async Task CheckNoUpdates(CancellationTokenSource cancellationTokenSou
while (!cancellationToken.IsCancellationRequested)
{
var last = (await _repository.Get(q => q
.Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage))
.Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage) || Equals(x.InteractionType, _repository.OwnChatterImageMessage))
.OrderByDescending(x => x.Id)
.Take(1)))
.Single();
Expand All @@ -70,7 +101,7 @@ protected async Task CheckNoUpdates(CancellationTokenSource cancellationTokenSou
return;
}

await Task.Delay(500, cancellationToken);
await Task.Delay(100, cancellationToken);
}
}
catch (TaskCanceledException)
Expand Down Expand Up @@ -239,9 +270,8 @@ protected override async Task<bool> HandleSync(Update update)
return true;
}

await Log(update, _repository.OwnChatterMessage);

if (update.Message?.Text?.Length > 0)
var anyContent = await Log(update);
if (anyContent)
{
Queued(Respond(update));
return true;
Expand Down Expand Up @@ -404,13 +434,14 @@ protected virtual void OnInitializing(long userId, long chatId)

protected virtual bool TraceImages => false;

protected async Task Respond(Update update)
private async Task Respond(Update update)
{
var now = DateTime.Now;
var since = now.AddDays(-1);

var interactions = await _repository.Get(q => q
.Where(x => Equals(x.InteractionType, _repository.OwnChatterMessage) ||
Equals(x.InteractionType, _repository.OwnChatterImageMessage) ||
Equals(x.InteractionType, _repository.OwnChatterMemoryPoint) ||
Equals(x.InteractionType, _repository.OwnChatterResetDialog))
.Where(x => x.ShortInfoSerialized!.Length > 0 || Equals(x.InteractionType, _repository.OwnChatterResetDialog))
Expand All @@ -434,20 +465,23 @@ protected async Task Respond(Update update)
using var checkingIsStillLast = new CancellationTokenSource();
ValueHolder<bool> isPhoto = new();
LongTyping(update.Message!.Chat.Id, update.Message.MessageThreadId, callingApis.Token, isPhoto);
var checking = CheckNoUpdates(checkingIsStillLast, callingApis.Token, interactions.Last(x => Equals(x.InteractionType, _repository.OwnChatterMessage)).Id);
var checking = CheckNoUpdates(checkingIsStillLast, callingApis.Token, interactions.Last(x => Equals(x.InteractionType, _repository.OwnChatterMessage) || Equals(x.InteractionType, _repository.OwnChatterImageMessage)).Id);

ChatBotMemoryPointWithTraces newMessagePoint;
DialogResult result;

try
{
await Task.Delay(update.Message.MediaGroupId != null || update.Message.Photo != null ? 500 : 220, checkingIsStillLast.Token);

var existingMemory = interactions.Select(i =>
Equals(i.InteractionType, _repository.OwnChatterMessage)
?
(MemoryPoint)new UserMessageMemoryPoint(i.ShortInfoSerialized!)
: Equals(i.InteractionType, _repository.OwnChatterMemoryPoint)
? JsonSerializer.Deserialize<ChatBotMemoryPoint>(i.Serialized)!
: throw new ArgumentOutOfRangeException(nameof(i))).ToList();
Equals(i.InteractionType, _repository.OwnChatterImageMessage)
? (MemoryPoint)new UserImageMessageMemoryPoint(i.ShortInfoSerialized!)
: Equals(i.InteractionType, _repository.OwnChatterMessage)
? new UserMessageMemoryPoint(i.ShortInfoSerialized!)
: Equals(i.InteractionType, _repository.OwnChatterMemoryPoint)
? JsonSerializer.Deserialize<ChatBotMemoryPoint>(i.Serialized)!
: throw new ArgumentOutOfRangeException(nameof(i))).ToList();

if (checkingIsStillLast.IsCancellationRequested)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ public class AiDialogHandler : AiDialogHandlerBase<InteractionType>
private readonly SongsDatabase _songsDatabase;
private new readonly TelegramOptions _telegramOptions;

public AiDialogHandler(
ILogger<AiDialogHandler> logger,
public AiDialogHandler(ILogger<AiDialogHandler> logger,
IOptions<TelegramOptions> telegramOptions,
SongsDatabase songsDatabase,
DialogRunner dialogRunner,
IScopedAbstractions scopedAbstractions)
: base(scopedAbstractions, logger, songsDatabase, dialogRunner)
DialogRunner dialogRunner,
IScopedAbstractions scopedAbstractions,
IHttpClientFactory httpClientFactory)
: base(scopedAbstractions, logger, songsDatabase, dialogRunner, httpClientFactory)
{
_songsDatabase = songsDatabase;
_telegramOptions = telegramOptions.Value;
Expand All @@ -40,7 +40,7 @@ protected override bool CheckRelevant(Update update)
}

protected override IInteraction<InteractionType> CreateInteraction(Update update) => new Interaction();

protected override (string? additionalBillingInfo, int? domainId) GetDialogConfigurationParameters() => default;

protected override async Task<DateTime?> GetImagesUnavailableUntil(DateTime now) => null;
Expand Down

0 comments on commit 3b2b976

Please sign in to comment.