Dev-In-A-Box-Solutions · JoeTomkinson · Dec 5, 2023 · Dec 5, 2023 · Dec 5, 2023
diff --git a/src/Benchmark/Benchmark.ConsoleApp/Benchmark.ConsoleApp.csproj b/src/Benchmark/Benchmark.ConsoleApp/Benchmark.ConsoleApp.csproj
@@ -8,11 +8,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.13.6" />
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" />
-    <PackageReference Include="SharpToken" Version="1.2.2" />
-    <PackageReference Include="Tiktoken" Version="1.1.0" />
-    <PackageReference Include="TiktokenSharp" Version="1.0.6" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.10" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
+    <PackageReference Include="SharpToken" Version="1.2.12" />
+    <PackageReference Include="Tiktoken" Version="1.1.3" />
+    <PackageReference Include="TiktokenSharp" Version="1.0.7" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/FrameworkTest/FrameworkTest/App.config b/src/FrameworkTest/FrameworkTest/App.config
@@ -1,6 +1,14 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
     <startup> 
         <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8" />
     </startup>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="System.Runtime.CompilerServices.Unsafe" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
 </configuration>
diff --git a/src/FrameworkTest/FrameworkTest/FrameworkTest.csproj b/src/FrameworkTest/FrameworkTest/FrameworkTest.csproj
@@ -36,14 +36,14 @@
     <StartupObject>FrameworkTest.Program</StartupObject>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Microsoft.Bcl.AsyncInterfaces, Version=7.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\TokenEvaluator.Net\packages\Microsoft.Bcl.AsyncInterfaces.7.0.0\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll</HintPath>
+    <Reference Include="Microsoft.Bcl.AsyncInterfaces, Version=8.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\..\TokenEvaluator.Net\packages\Microsoft.Bcl.AsyncInterfaces.8.0.0\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.Extensions.DependencyInjection, Version=7.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.7.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.dll</HintPath>
+    <Reference Include="Microsoft.Extensions.DependencyInjection, Version=8.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
+      <HintPath>..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.8.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.Extensions.DependencyInjection.Abstractions, Version=7.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.Abstractions.7.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.Abstractions.dll</HintPath>
+    <Reference Include="Microsoft.Extensions.DependencyInjection.Abstractions, Version=8.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
+      <HintPath>..\..\TokenEvaluator.Net\packages\Microsoft.Extensions.DependencyInjection.Abstractions.8.0.0\lib\net462\Microsoft.Extensions.DependencyInjection.Abstractions.dll</HintPath>
     </Reference>
     <Reference Include="Newtonsoft.Json, Version=13.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\..\TokenEvaluator.Net\packages\Newtonsoft.Json.13.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
@@ -55,8 +55,8 @@
       <Private>True</Private>
       <Private>True</Private>
     </Reference>
-    <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\..\TokenEvaluator.Net\packages\System.Runtime.CompilerServices.Unsafe.4.5.3\lib\net461\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
+    <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\..\TokenEvaluator.Net\packages\System.Runtime.CompilerServices.Unsafe.6.0.0\lib\net461\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
     </Reference>
     <Reference Include="System.Threading.Tasks.Extensions, Version=4.2.0.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
       <HintPath>..\..\TokenEvaluator.Net\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll</HintPath>
@@ -67,8 +67,8 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Xml" />
-    <Reference Include="TokenEvaluator.Net, Version=1.0.5.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\..\TokenEvaluator.Net\packages\TokenEvaluator.Net.1.0.5\lib\netstandard2.0\TokenEvaluator.Net.dll</HintPath>
+    <Reference Include="TokenEvaluator.Net, Version=1.0.7.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\..\TokenEvaluator.Net\packages\TokenEvaluator.Net.1.0.7\lib\netstandard2.0\TokenEvaluator.Net.dll</HintPath>
     </Reference>
   </ItemGroup>
   <ItemGroup>

diff --git a/src/FrameworkTest/FrameworkTest/packages.config b/src/FrameworkTest/FrameworkTest/packages.config
@@ -1,11 +1,11 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Bcl.AsyncInterfaces" version="7.0.0" targetFramework="net48" />
-  <package id="Microsoft.Extensions.DependencyInjection" version="7.0.0" targetFramework="net48" />
-  <package id="Microsoft.Extensions.DependencyInjection.Abstractions" version="7.0.0" targetFramework="net48" />
+  <package id="Microsoft.Bcl.AsyncInterfaces" version="8.0.0" targetFramework="net48" />
+  <package id="Microsoft.Extensions.DependencyInjection" version="8.0.0" targetFramework="net48" />
+  <package id="Microsoft.Extensions.DependencyInjection.Abstractions" version="8.0.0" targetFramework="net48" />
   <package id="Newtonsoft.Json" version="13.0.3" targetFramework="net48" />
   <package id="System.Linq" version="4.3.0" targetFramework="net48" />
-  <package id="System.Runtime.CompilerServices.Unsafe" version="4.5.3" targetFramework="net48" />
+  <package id="System.Runtime.CompilerServices.Unsafe" version="6.0.0" targetFramework="net48" />
   <package id="System.Threading.Tasks.Extensions" version="4.5.4" targetFramework="net48" />
-  <package id="TokenEvaluator.Net" version="1.0.5" targetFramework="net48" />
+  <package id="TokenEvaluator.Net" version="1.0.7" targetFramework="net48" />
 </packages>
diff --git a/src/TokenEvaluator.Net/ITokenEvaluatorClient.cs b/src/TokenEvaluator.Net/ITokenEvaluatorClient.cs
@@ -12,6 +12,6 @@ public interface ITokenEvaluatorClient
         void OverridePairedByteEncodingDirectory(string directoryPath);
         void SetDefaultTokenEncodingForModel(ModelType modelType);
         void SetDefaultTokenEncoding(EncodingType encodingType);
-        int VisionTokenCount(int width, int height, DetailLevel detail);
+        double VisionTokenCount(int width, int height, DetailLevel detail);
     }
 }
diff --git a/src/TokenEvaluator.Net/TokenEvaluator.Net.csproj b/src/TokenEvaluator.Net/TokenEvaluator.Net.csproj
@@ -16,7 +16,7 @@
     <RepositoryUrl>https://github.com/Dev-In-A-Box-Solutions/TokenEvaluator.Net</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
     <PackageTags>Image-To-Token-Count; Token; Text-To-Token-Count; Token-Count;</PackageTags>
-    <AssemblyVersion>1.0.7</AssemblyVersion>
+    <AssemblyVersion>1.0.8</AssemblyVersion>
     <NeutralLanguage>en</NeutralLanguage>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <PackageReleaseNotes>OpenAI Vision Preview Model Image Token Count</PackageReleaseNotes>

diff --git a/src/TokenEvaluator.Net/TokenEvaluatorClient.cs b/src/TokenEvaluator.Net/TokenEvaluatorClient.cs
@@ -37,7 +37,7 @@
            }
        }

        public TokenEvaluatorClient(IEncodingService encodingService)
        {
            _encodingService = encodingService;

@@ -246,39 +246,43 @@
         /// <param name="height">Height of the image in pixels.</param>
         /// <param name="detail">Detail level of the image, can be either 'Low' or 'High'.</param>
         /// <returns>Token Count for Image</returns>
-        public int VisionTokenCount(int width, int height, DetailLevel detail)
+        public double VisionTokenCount(int width, int height, DetailLevel detail)
         {
+            // move from int to doubles for the purpose of the calculations
+            double widthDouble = width;
+            double heightDouble = height;
+
             if (detail == DetailLevel.Low)
             {
                 return 85;
             }
             else if (detail == DetailLevel.High)
             {
                 // Scale the image to fit within a 2048 x 2048 square
-                if (width > 2048 || height > 2048)
+                if (widthDouble > 2048 || heightDouble > 2048)
                 {
-                    float aspectRatio = (float)width / height;
-                    if (width > height)
+                    double aspectRatio = widthDouble / heightDouble;
+                    if (widthDouble > heightDouble)
                     {
-                        width = 2048;
-                        height = (int)(width / aspectRatio);
+                        widthDouble = 2048;
+                        heightDouble = widthDouble / aspectRatio;
                     }
                     else
                     {
-                        height = 2048;
-                        width = (int)(height * aspectRatio);
+                        heightDouble = 2048;
+                        widthDouble = heightDouble * aspectRatio;
                     }
                 }
                 // Scale the image such that the shortest side is 768px long
-                float scaleRatio = width < height ? 768f / width : 768f / height;
-                width = (int)(width * scaleRatio);
-                height = (int)(height * scaleRatio);
+                double scaleRatio = widthDouble < heightDouble ? 768.0 / widthDouble : 768.0 / heightDouble;
+                widthDouble *= scaleRatio;
+                heightDouble *= scaleRatio;
                 // Count how many 512px squares the image consists of
-                int squareWidthCount = (int)Math.Ceiling(width / 512.0);
-                int squareHeightCount = (int)Math.Ceiling(height / 512.0);
-                int totalSquares = squareWidthCount * squareHeightCount;
+                double squareWidthCount = Math.Ceiling(widthDouble / 512.0);
+                double squareHeightCount = Math.Ceiling(heightDouble / 512.0);
+                double totalSquares = squareWidthCount * squareHeightCount;
                 // Each of those squares costs 170 tokens
-                int tokenCost = totalSquares * 170;
+                double tokenCost = totalSquares * 170;
                 // Add 85 tokens to the final total
                 return tokenCost + 85;
             }

diff --git a/src/TokenEvaluator.Tests/TokenEvaluator.Tests.csproj b/src/TokenEvaluator.Tests/TokenEvaluator.Tests.csproj
@@ -11,10 +11,10 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
-    <PackageReference Include="MSTest.TestAdapter" Version="3.0.4" />
-    <PackageReference Include="MSTest.TestFramework" Version="3.0.4" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
     <PackageReference Include="coverlet.collector" Version="6.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

diff --git a/src/TokenEvaluator.Tests/VisionTokenTests.cs b/src/TokenEvaluator.Tests/VisionTokenTests.cs
@@ -0,0 +1,102 @@
+using TokenEvaluator.Net;
+
+
+namespace TokenEvaluator.Tests
+{
+    [TestClass]
+    public class VisionTokenTests
+    {
+        internal readonly IServiceCollection services = new ServiceCollection();
+        internal ServiceProvider? serviceProvider;
+        internal ITokenEvaluatorClient? tokenClient;
+
+        [TestInitialize]
+        public void Init()
+        {
+            // init a service collection, run the extension method to add the library services, and build the service provider
+            services.AddTextTokenizationEvaluatorServices();
+            services.AddSingleton<ITokenEvaluatorClient, TokenEvaluatorClient>();
+            serviceProvider = services.BuildServiceProvider();
+
+            // get the token client.
+            tokenClient = serviceProvider.GetService<ITokenEvaluatorClient>();
+            tokenClient?.OverridePairedByteEncodingDirectory(Path.Combine(Environment.CurrentDirectory, "TestDataFolder"));
+        }
+
+        [TestMethod]
+        public void HighDetailSquareImageTokenCount()
+        {
+            if (serviceProvider == null)
+            {
+                Assert.Fail("Service Provider Null");
+            }
+
+            if (tokenClient != null)
+            {
+                var tokenCount = tokenClient.VisionTokenCount(1040, 1040, DetailLevel.High);
+                Assert.AreEqual(tokenCount, 765);
+            }
+            else
+            {
+                Assert.Fail("Token Client Null");
+            }
+        }
+
+        [TestMethod]
+        public void HighDetailLandscapeImageTokenCount()
+        {
+            if (serviceProvider == null)
+            {
+                Assert.Fail("Service Provider Null");
+            }
+
+            if (tokenClient != null)
+            {
+                var tokenCount = tokenClient.VisionTokenCount(2080, 1040, DetailLevel.High);
+                Assert.AreEqual(tokenCount, 1105);
+            }
+            else
+            {
+                Assert.Fail("Token Client Null");
+            }
+        }
+
+        [TestMethod]
+        public void HighDetailPortraitImageTokenCount()
+        {
+            if (serviceProvider == null)
+            {
+                Assert.Fail("Service Provider Null");
+            }
+
+            if (tokenClient != null)
+            {
+                var tokenCount = tokenClient.VisionTokenCount(2080, 1040, DetailLevel.High);
+                Assert.AreEqual(tokenCount, 1105);
+            }
+            else
+            {
+                Assert.Fail("Token Client Null");
+            }
+        }
+
+        [TestMethod]
+        public void LowDetailImageTokenCount()
+        {
+            if (serviceProvider == null)
+            {
+                Assert.Fail("Service Provider Null");
+            }
+
+            if (tokenClient != null)
+            {
+                var tokenCount = tokenClient.VisionTokenCount(2080, 1040, DetailLevel.Low);
+                Assert.AreEqual(tokenCount, 85);
+            }
+            else
+            {
+                Assert.Fail("Token Client Null");
+            }
+        }
+    }
+}