diff --git a/src/Benchmark/Benchmark.ConsoleApp/Benchmark.ConsoleApp.csproj b/src/Benchmark/Benchmark.ConsoleApp/Benchmark.ConsoleApp.csproj index d9940d1..ebd2e38 100644 --- a/src/Benchmark/Benchmark.ConsoleApp/Benchmark.ConsoleApp.csproj +++ b/src/Benchmark/Benchmark.ConsoleApp/Benchmark.ConsoleApp.csproj @@ -8,11 +8,11 @@ - - - - - + + + + + diff --git a/src/FrameworkTest/FrameworkTest/App.config b/src/FrameworkTest/FrameworkTest/App.config index 193aecc..91126fc 100644 --- a/src/FrameworkTest/FrameworkTest/App.config +++ b/src/FrameworkTest/FrameworkTest/App.config @@ -1,6 +1,14 @@ - + + + + + + + + + \ No newline at end of file diff --git a/src/FrameworkTest/FrameworkTest/FrameworkTest.csproj b/src/FrameworkTest/FrameworkTest/FrameworkTest.csproj index 6f783dd..dac143b 100644 --- a/src/FrameworkTest/FrameworkTest/FrameworkTest.csproj +++ b/src/FrameworkTest/FrameworkTest/FrameworkTest.csproj @@ -36,14 +36,14 @@ FrameworkTest.Program - - ..\..\TokenEvaluator.Net\packages\Microsoft.Bcl.AsyncInterfaces.7.0.0\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\TokenEvaluator.Net\packages\Microsoft.Bcl.AsyncInterfaces.8.0.0\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll - - ..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.7.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.dll + + ..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.8.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.dll - - ..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.Abstractions.7.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.Abstractions.dll + + ..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.Abstractions.8.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.Abstractions.dll ..\..\TokenEvaluator.Net\packages\Newtonsoft.Json.13.0.3\lib\net45\Newtonsoft.Json.dll @@ -55,8 +55,8 @@ True True - - ..\..\TokenEvaluator.Net\packages\System.Runtime.CompilerServices.Unsafe.4.5.3\lib\net461\System.Runtime.CompilerServices.Unsafe.dll + + ..\..\TokenEvaluator.Net\packages\System.Runtime.CompilerServices.Unsafe.6.0.0\lib\net461\System.Runtime.CompilerServices.Unsafe.dll ..\..\TokenEvaluator.Net\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll @@ -67,8 +67,8 @@ - - ..\..\TokenEvaluator.Net\packages\TokenEvaluator.Net.1.0.5\lib\netstandard2.0\TokenEvaluator.Net.dll + + ..\..\TokenEvaluator.Net\packages\TokenEvaluator.Net.1.0.7\lib\netstandard2.0\TokenEvaluator.Net.dll diff --git a/src/FrameworkTest/FrameworkTest/packages.config b/src/FrameworkTest/FrameworkTest/packages.config index 669083b..d20753a 100644 --- a/src/FrameworkTest/FrameworkTest/packages.config +++ b/src/FrameworkTest/FrameworkTest/packages.config @@ -1,11 +1,11 @@  - - - + + + - + - + \ No newline at end of file diff --git a/src/TokenEvaluator.Net/ITokenEvaluatorClient.cs b/src/TokenEvaluator.Net/ITokenEvaluatorClient.cs index ff49200..2ce1929 100644 --- a/src/TokenEvaluator.Net/ITokenEvaluatorClient.cs +++ b/src/TokenEvaluator.Net/ITokenEvaluatorClient.cs @@ -12,6 +12,6 @@ public interface ITokenEvaluatorClient void OverridePairedByteEncodingDirectory(string directoryPath); void SetDefaultTokenEncodingForModel(ModelType modelType); void SetDefaultTokenEncoding(EncodingType encodingType); - int VisionTokenCount(int width, int height, DetailLevel detail); + double VisionTokenCount(int width, int height, DetailLevel detail); } } \ No newline at end of file diff --git a/src/TokenEvaluator.Net/TokenEvaluator.Net.csproj b/src/TokenEvaluator.Net/TokenEvaluator.Net.csproj index a3e9a9c..d809e10 100644 --- a/src/TokenEvaluator.Net/TokenEvaluator.Net.csproj +++ b/src/TokenEvaluator.Net/TokenEvaluator.Net.csproj @@ -16,7 +16,7 @@ https://github.com/Dev-In-A-Box-Solutions/TokenEvaluator.Net git Image-To-Token-Count; Token; Text-To-Token-Count; Token-Count; - 1.0.7 + 1.0.8 en README.md OpenAI Vision Preview Model Image Token Count diff --git a/src/TokenEvaluator.Net/TokenEvaluatorClient.cs b/src/TokenEvaluator.Net/TokenEvaluatorClient.cs index 8eb1db0..7925770 100644 --- a/src/TokenEvaluator.Net/TokenEvaluatorClient.cs +++ b/src/TokenEvaluator.Net/TokenEvaluatorClient.cs @@ -246,8 +246,12 @@ private static Regex SpecialTokenRegex(HashSet tokens) /// Height of the image in pixels. /// Detail level of the image, can be either 'Low' or 'High'. /// Token Count for Image - public int VisionTokenCount(int width, int height, DetailLevel detail) + public double VisionTokenCount(int width, int height, DetailLevel detail) { + // move from int to doubles for the purpose of the calculations + double widthDouble = width; + double heightDouble = height; + if (detail == DetailLevel.Low) { return 85; @@ -255,30 +259,30 @@ public int VisionTokenCount(int width, int height, DetailLevel detail) else if (detail == DetailLevel.High) { // Scale the image to fit within a 2048 x 2048 square - if (width > 2048 || height > 2048) + if (widthDouble > 2048 || heightDouble > 2048) { - float aspectRatio = (float)width / height; - if (width > height) + double aspectRatio = widthDouble / heightDouble; + if (widthDouble > heightDouble) { - width = 2048; - height = (int)(width / aspectRatio); + widthDouble = 2048; + heightDouble = widthDouble / aspectRatio; } else { - height = 2048; - width = (int)(height * aspectRatio); + heightDouble = 2048; + widthDouble = heightDouble * aspectRatio; } } // Scale the image such that the shortest side is 768px long - float scaleRatio = width < height ? 768f / width : 768f / height; - width = (int)(width * scaleRatio); - height = (int)(height * scaleRatio); + double scaleRatio = widthDouble < heightDouble ? 768.0 / widthDouble : 768.0 / heightDouble; + widthDouble *= scaleRatio; + heightDouble *= scaleRatio; // Count how many 512px squares the image consists of - int squareWidthCount = (int)Math.Ceiling(width / 512.0); - int squareHeightCount = (int)Math.Ceiling(height / 512.0); - int totalSquares = squareWidthCount * squareHeightCount; + double squareWidthCount = Math.Ceiling(widthDouble / 512.0); + double squareHeightCount = Math.Ceiling(heightDouble / 512.0); + double totalSquares = squareWidthCount * squareHeightCount; // Each of those squares costs 170 tokens - int tokenCost = totalSquares * 170; + double tokenCost = totalSquares * 170; // Add 85 tokens to the final total return tokenCost + 85; } diff --git a/src/TokenEvaluator.Tests/TokenEvaluator.Tests.csproj b/src/TokenEvaluator.Tests/TokenEvaluator.Tests.csproj index a635e94..282cef4 100644 --- a/src/TokenEvaluator.Tests/TokenEvaluator.Tests.csproj +++ b/src/TokenEvaluator.Tests/TokenEvaluator.Tests.csproj @@ -11,10 +11,10 @@ - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/TokenEvaluator.Tests/VisionTokenTests.cs b/src/TokenEvaluator.Tests/VisionTokenTests.cs new file mode 100644 index 0000000..53f0cb7 --- /dev/null +++ b/src/TokenEvaluator.Tests/VisionTokenTests.cs @@ -0,0 +1,102 @@ +using TokenEvaluator.Net; + + +namespace TokenEvaluator.Tests +{ + [TestClass] + public class VisionTokenTests + { + internal readonly IServiceCollection services = new ServiceCollection(); + internal ServiceProvider? serviceProvider; + internal ITokenEvaluatorClient? tokenClient; + + [TestInitialize] + public void Init() + { + // init a service collection, run the extension method to add the library services, and build the service provider + services.AddTextTokenizationEvaluatorServices(); + services.AddSingleton(); + serviceProvider = services.BuildServiceProvider(); + + // get the token client. + tokenClient = serviceProvider.GetService(); + tokenClient?.OverridePairedByteEncodingDirectory(Path.Combine(Environment.CurrentDirectory, "TestDataFolder")); + } + + [TestMethod] + public void HighDetailSquareImageTokenCount() + { + if (serviceProvider == null) + { + Assert.Fail("Service Provider Null"); + } + + if (tokenClient != null) + { + var tokenCount = tokenClient.VisionTokenCount(1040, 1040, DetailLevel.High); + Assert.AreEqual(tokenCount, 765); + } + else + { + Assert.Fail("Token Client Null"); + } + } + + [TestMethod] + public void HighDetailLandscapeImageTokenCount() + { + if (serviceProvider == null) + { + Assert.Fail("Service Provider Null"); + } + + if (tokenClient != null) + { + var tokenCount = tokenClient.VisionTokenCount(2080, 1040, DetailLevel.High); + Assert.AreEqual(tokenCount, 1105); + } + else + { + Assert.Fail("Token Client Null"); + } + } + + [TestMethod] + public void HighDetailPortraitImageTokenCount() + { + if (serviceProvider == null) + { + Assert.Fail("Service Provider Null"); + } + + if (tokenClient != null) + { + var tokenCount = tokenClient.VisionTokenCount(2080, 1040, DetailLevel.High); + Assert.AreEqual(tokenCount, 1105); + } + else + { + Assert.Fail("Token Client Null"); + } + } + + [TestMethod] + public void LowDetailImageTokenCount() + { + if (serviceProvider == null) + { + Assert.Fail("Service Provider Null"); + } + + if (tokenClient != null) + { + var tokenCount = tokenClient.VisionTokenCount(2080, 1040, DetailLevel.Low); + Assert.AreEqual(tokenCount, 85); + } + else + { + Assert.Fail("Token Client Null"); + } + } + } +}