From ab0720740cbc2186c635c2f56dbe8ba40120c74e Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 29 Apr 2023 13:56:35 -0400 Subject: [PATCH 01/21] Update packages --- .../Hangfire.Storage.SQLite.csproj | 10 +++++----- src/samples/ConsoleSample/ConsoleSample.csproj | 2 +- src/samples/WebSample/WebSample.csproj | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj index 33bc8b2..7096a08 100644 --- a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj +++ b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj @@ -7,7 +7,7 @@ netstandard2.0;net48 - 0.3.4 + 0.4.0 RaisedApp RaisedApp Copyright © 2019 - Present @@ -20,8 +20,8 @@ Hangfire Storage SQLite An Alternative SQLite Storage for Hangfire - 0.3.4 - - Bug Fix (GetTimelineStats and GetHourlyTimelineStats return wrongly matched results), thanks to @MaxTheWhale + 0.4.0 + - @@ -31,9 +31,9 @@ - + - + \ No newline at end of file diff --git a/src/samples/ConsoleSample/ConsoleSample.csproj b/src/samples/ConsoleSample/ConsoleSample.csproj index 1eb47c7..b1249a1 100644 --- a/src/samples/ConsoleSample/ConsoleSample.csproj +++ b/src/samples/ConsoleSample/ConsoleSample.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp2.2 + net6.0 false RaisedApp RaisedApp diff --git a/src/samples/WebSample/WebSample.csproj b/src/samples/WebSample/WebSample.csproj index 10b3626..f00d350 100644 --- a/src/samples/WebSample/WebSample.csproj +++ b/src/samples/WebSample/WebSample.csproj @@ -1,7 +1,7 @@  - netcoreapp2.2 + net6.0 InProcess false RaisedApp @@ -14,9 +14,9 @@ - - - + + + From 1b67bfc0ae33c350455413a938d3666e9fc72887 Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 29 Apr 2023 14:57:14 -0400 Subject: [PATCH 02/21] Fix references & new samples --- Hangfire.Storage.SQLite.sln | 24 ++++---- .../ConsoleSample/ConsoleSample.csproj | 21 ++----- src/samples/ConsoleSample/Program.cs | 49 +++++------------ src/samples/WebSample/Program.cs | 44 ++++++++++----- src/samples/WebSample/Startup.cs | 55 ------------------- src/samples/WebSample/TaskSample.cs | 19 +++++++ src/samples/WebSample/WebSample.csproj | 22 +------- .../WebSample/appsettings.Development.json | 5 +- src/samples/WebSample/appsettings.json | 3 +- 9 files changed, 86 insertions(+), 156 deletions(-) delete mode 100644 src/samples/WebSample/Startup.cs create mode 100644 src/samples/WebSample/TaskSample.cs diff --git a/Hangfire.Storage.SQLite.sln b/Hangfire.Storage.SQLite.sln index 1a2cf8d..33c61d1 100644 --- a/Hangfire.Storage.SQLite.sln +++ b/Hangfire.Storage.SQLite.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29424.173 +# Visual Studio Version 17 +VisualStudioVersion = 17.2.32602.215 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Hangfire.Storage.SQLite", "src\main\Hangfire.Storage.SQLite\Hangfire.Storage.SQLite.csproj", "{27A56CA6-6B11-4D43-8BF8-F9AF5907556C}" EndProject @@ -14,9 +14,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .github\workflows\dotnetcore.yml = .github\workflows\dotnetcore.yml EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleSample", "src\samples\ConsoleSample\ConsoleSample.csproj", "{453B460B-6643-4EBC-A046-3BDA69EFF2CE}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebSample", "src\samples\WebSample\WebSample.csproj", "{7170DD33-D6CA-40A5-AA03-D96DE74BA353}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebSample", "src\samples\WebSample\WebSample.csproj", "{1ABCB458-08C4-4132-8295-A5A605F6B75F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleSample", "src\samples\ConsoleSample\ConsoleSample.csproj", "{C2137C38-9AE6-4200-9F97-E3DA5A2DC741}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -32,14 +32,14 @@ Global {279C04FC-22EB-438C-9BF7-2CBA306025F7}.Debug|Any CPU.Build.0 = Debug|Any CPU {279C04FC-22EB-438C-9BF7-2CBA306025F7}.Release|Any CPU.ActiveCfg = Release|Any CPU {279C04FC-22EB-438C-9BF7-2CBA306025F7}.Release|Any CPU.Build.0 = Release|Any CPU - {453B460B-6643-4EBC-A046-3BDA69EFF2CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {453B460B-6643-4EBC-A046-3BDA69EFF2CE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {453B460B-6643-4EBC-A046-3BDA69EFF2CE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {453B460B-6643-4EBC-A046-3BDA69EFF2CE}.Release|Any CPU.Build.0 = Release|Any CPU - {1ABCB458-08C4-4132-8295-A5A605F6B75F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1ABCB458-08C4-4132-8295-A5A605F6B75F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1ABCB458-08C4-4132-8295-A5A605F6B75F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1ABCB458-08C4-4132-8295-A5A605F6B75F}.Release|Any CPU.Build.0 = Release|Any CPU + {7170DD33-D6CA-40A5-AA03-D96DE74BA353}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7170DD33-D6CA-40A5-AA03-D96DE74BA353}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7170DD33-D6CA-40A5-AA03-D96DE74BA353}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7170DD33-D6CA-40A5-AA03-D96DE74BA353}.Release|Any CPU.Build.0 = Release|Any CPU + {C2137C38-9AE6-4200-9F97-E3DA5A2DC741}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C2137C38-9AE6-4200-9F97-E3DA5A2DC741}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C2137C38-9AE6-4200-9F97-E3DA5A2DC741}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C2137C38-9AE6-4200-9F97-E3DA5A2DC741}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/samples/ConsoleSample/ConsoleSample.csproj b/src/samples/ConsoleSample/ConsoleSample.csproj index b1249a1..4c37de1 100644 --- a/src/samples/ConsoleSample/ConsoleSample.csproj +++ b/src/samples/ConsoleSample/ConsoleSample.csproj @@ -1,27 +1,14 @@ - + Exe net6.0 - false - RaisedApp - RaisedApp - Copyright © 2019 - Present - LICENSE - https://github.com/raisedapp/Hangfire.Storage.SQLite - https://github.com/raisedapp/Hangfire.Storage.SQLite - git + enable + enable - - - True - - - - - + diff --git a/src/samples/ConsoleSample/Program.cs b/src/samples/ConsoleSample/Program.cs index 9e4fc2e..4d8de36 100644 --- a/src/samples/ConsoleSample/Program.cs +++ b/src/samples/ConsoleSample/Program.cs @@ -1,42 +1,23 @@ using Hangfire; using Hangfire.Storage.SQLite; -using System; -namespace ConsoleSample +try { - public class Program - { - public static void Main(string[] args) - { - try - { - // you can use SQLite Storage and specify the connection string name - GlobalConfiguration.Configuration - .UseColouredConsoleLogProvider() - .UseSQLiteStorage("Hangfire.db", new SQLiteStorageOptions() { AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL }); - - //you have to create an instance of background job server at least once for background jobs to run - using (new BackgroundJobServer()) - { - BackgroundJob.Enqueue(() => Console.WriteLine("Background Job: Hello, world!")); - - RecurringJob.AddOrUpdate(() => TaskMethod(), Cron.Minutely); + GlobalConfiguration.Configuration. + UseColouredConsoleLogProvider(). + UseSQLiteStorage("Hangfire.db", new SQLiteStorageOptions() { AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL }); - Console.WriteLine("Press Enter to exit..."); - Console.ReadLine(); - } - } - catch (Exception ex) - { - Console.WriteLine($"Error in execution. Detail: {ex}"); - } - - Console.ReadLine(); - } + using (new BackgroundJobServer()) + { + BackgroundJob.Enqueue(() => Console.WriteLine("ConsoleSample Start :)")); + RecurringJob.AddOrUpdate("Console.WriteLine()", () => Console.WriteLine("Background Job: Hello, world!"), Cron.Minutely); - public static void TaskMethod() - { - Console.WriteLine("Testing Web Sample!!!"); - } + Console.WriteLine("Press Enter to exit..."); + Console.ReadLine(); } } +catch (Exception ex) +{ + Console.WriteLine($"Error in execution. Detail: {ex}"); + Console.ReadLine(); +} \ No newline at end of file diff --git a/src/samples/WebSample/Program.cs b/src/samples/WebSample/Program.cs index 98eed05..66f7eb1 100644 --- a/src/samples/WebSample/Program.cs +++ b/src/samples/WebSample/Program.cs @@ -1,17 +1,31 @@ -using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; +using Hangfire; +using Hangfire.Heartbeat; +using Hangfire.Heartbeat.Server; +using Hangfire.JobsLogger; +using Hangfire.Server; +using Hangfire.Storage.SQLite; +using WebSample; -namespace WebSample -{ - public class Program - { - public static void Main(string[] args) - { - CreateWebHostBuilder(args).Build().Run(); - } +var builder = WebApplication.CreateBuilder(args); +var services = builder.Services; - public static IWebHostBuilder CreateWebHostBuilder(string[] args) => - WebHost.CreateDefaultBuilder(args) - .UseStartup(); - } -} +services.AddTransient(); +services.AddTransient(x => new ProcessMonitor(checkInterval: TimeSpan.FromSeconds(10))); +services.AddHangfire(configuration => configuration + .SetDataCompatibilityLevel(CompatibilityLevel.Version_180) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseSQLiteStorage("Hangfire.db", + new SQLiteStorageOptions() { AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL, JobExpirationCheckInterval = TimeSpan.FromSeconds(30) }) + .UseHeartbeatPage(checkInterval: TimeSpan.FromSeconds(10)) + .UseJobsLogger()); +services.AddHangfireServer(); + +var app = builder.Build(); + +app.UseHangfireDashboard(string.Empty); + +RecurringJob.AddOrUpdate("TaskMethod()", (TaskSample t) => t.TaskMethod(), Cron.Minutely); +RecurringJob.AddOrUpdate("TaskMethod2()", (TaskSample t) => t.TaskMethod2(null), Cron.Minutely); + +app.Run(); \ No newline at end of file diff --git a/src/samples/WebSample/Startup.cs b/src/samples/WebSample/Startup.cs deleted file mode 100644 index 1db36b0..0000000 --- a/src/samples/WebSample/Startup.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Hangfire; -using Hangfire.Heartbeat; -using Hangfire.Heartbeat.Server; -using Hangfire.JobsLogger; -using Hangfire.Server; -using Hangfire.Storage.SQLite; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.DependencyInjection; -using System; - -namespace WebSample -{ - public class Startup - { - // This method gets called by the runtime. Use this method to add services to the container. - // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 - public void ConfigureServices(IServiceCollection services) - { - services.AddHangfire(configuration => configuration - .SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseSQLiteStorage("Hangfire.db", new SQLiteStorageOptions() { AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL, JobExpirationCheckInterval = TimeSpan.FromSeconds(30) }) - .UseHeartbeatPage(checkInterval: TimeSpan.FromSeconds(30)) - .UseJobsLogger()); - } - - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IHostingEnvironment env) - { - if (env.IsDevelopment()) - { - app.UseDeveloperExceptionPage(); - } - - app.UseHangfireServer(additionalProcesses: new[] { new ProcessMonitor(checkInterval: TimeSpan.FromSeconds(30)) }); - app.UseHangfireDashboard(string.Empty); - - RecurringJob.AddOrUpdate(() => TaskMethod(), Cron.Minutely); - RecurringJob.AddOrUpdate(() => TaskMethod2(null), Cron.Minutely); - } - - public void TaskMethod() - { - Console.WriteLine("Testing Web Sample!!!"); - } - - public void TaskMethod2(PerformContext pfContext) - { - pfContext.LogInformation("Testing Web Sample!!!"); - Console.WriteLine("Testing Web Sample!!!"); - } - } -} diff --git a/src/samples/WebSample/TaskSample.cs b/src/samples/WebSample/TaskSample.cs new file mode 100644 index 0000000..da9f356 --- /dev/null +++ b/src/samples/WebSample/TaskSample.cs @@ -0,0 +1,19 @@ +using Hangfire.JobsLogger; +using Hangfire.Server; + +namespace WebSample +{ + public class TaskSample + { + public void TaskMethod() + { + Console.WriteLine("Testing Web Sample!!!"); + } + + public void TaskMethod2(PerformContext? pfContext) + { + pfContext.LogInformation("Testing Web Sample!!!"); + Console.WriteLine("Testing Web Sample!!!"); + } + } +} \ No newline at end of file diff --git a/src/samples/WebSample/WebSample.csproj b/src/samples/WebSample/WebSample.csproj index f00d350..f3fd659 100644 --- a/src/samples/WebSample/WebSample.csproj +++ b/src/samples/WebSample/WebSample.csproj @@ -1,31 +1,15 @@ - + net6.0 - InProcess - false - RaisedApp - RaisedApp - Copyright © 2019 - Present - LICENSE - https://github.com/raisedapp/Hangfire.Storage.SQLite - https://github.com/raisedapp/Hangfire.Storage.SQLite - git + enable + enable - - - - - - - True - - diff --git a/src/samples/WebSample/appsettings.Development.json b/src/samples/WebSample/appsettings.Development.json index e203e94..0c208ae 100644 --- a/src/samples/WebSample/appsettings.Development.json +++ b/src/samples/WebSample/appsettings.Development.json @@ -1,9 +1,8 @@ { "Logging": { "LogLevel": { - "Default": "Debug", - "System": "Information", - "Microsoft": "Information" + "Default": "Information", + "Microsoft.AspNetCore": "Warning" } } } diff --git a/src/samples/WebSample/appsettings.json b/src/samples/WebSample/appsettings.json index def9159..10f68b8 100644 --- a/src/samples/WebSample/appsettings.json +++ b/src/samples/WebSample/appsettings.json @@ -1,7 +1,8 @@ { "Logging": { "LogLevel": { - "Default": "Warning" + "Default": "Information", + "Microsoft.AspNetCore": "Warning" } }, "AllowedHosts": "*" From d9ffa1797eb37b8297992a248d8a49934f64a797 Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 29 Apr 2023 15:02:39 -0400 Subject: [PATCH 03/21] Fix... --- .github/workflows/dotnetcore.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dotnetcore.yml b/.github/workflows/dotnetcore.yml index 7aa3fdf..4d9939f 100644 --- a/.github/workflows/dotnetcore.yml +++ b/.github/workflows/dotnetcore.yml @@ -21,11 +21,11 @@ jobs: runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v3 - name: Setup .NET Core - uses: actions/setup-dotnet@v1.7.2 + uses: actions/setup-dotnet@v3 with: - dotnet-version: 2.2.108 + dotnet-version: 6.0.x - name: Build with dotnet run: dotnet build --configuration Release - name: Unit Tests From d4005c1ce1d8716ff82f300a1d3ce1df9abc9302 Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 29 Apr 2023 15:08:47 -0400 Subject: [PATCH 04/21] Upgrade Hangfire.Storage.SQLite.Test to Dotnet 6 --- .../Hangfire.Storage.SQLite.Test.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/Hangfire.Storage.SQLite.Test/Hangfire.Storage.SQLite.Test.csproj b/src/test/Hangfire.Storage.SQLite.Test/Hangfire.Storage.SQLite.Test.csproj index 963c779..62d7c60 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/Hangfire.Storage.SQLite.Test.csproj +++ b/src/test/Hangfire.Storage.SQLite.Test/Hangfire.Storage.SQLite.Test.csproj @@ -1,7 +1,7 @@  - netcoreapp2.1 + net6.0 false RaisedApp RaisedApp From 3f082045762c8ee37985b6408e86ee30c5a2f547 Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Sun, 25 Jun 2023 16:33:57 +0200 Subject: [PATCH 05/21] Allow to pass in SQLiteConnection from outside, removed static-accessor for HangfireDbContext.Instance --- .../HangfireDbContext.cs | 47 ++++++----------- .../ObjectExtensions.cs | 2 +- .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 27 +++++++--- .../SQLiteStorageExtensions.cs | 22 ++++++++ .../SQLiteStorageOptions.cs | 12 +++++ .../Utils/CleanDatabaseAttribute.cs | 50 ------------------- 6 files changed, 71 insertions(+), 89 deletions(-) delete mode 100644 src/test/Hangfire.Storage.SQLite.Test/Utils/CleanDatabaseAttribute.cs diff --git a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs index 06dd231..abd278a 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs @@ -23,9 +23,6 @@ public class HangfireDbContext /// public SQLiteStorageOptions StorageOptions { get; private set; } - private static readonly object Locker = new object(); - private static volatile HangfireDbContext _instance; - /// /// SQLite database connection identifier /// @@ -35,9 +32,10 @@ public class HangfireDbContext /// /// Starts SQLite database using a connection string for file system database /// - /// the database path + /// the database path + /// /// Table prefix - private HangfireDbContext(string databasePath, string prefix = "hangfire") + internal HangfireDbContext(SQLiteConnection connection, string prefix = "hangfire") { //UTC - Internal JSON GlobalConfiguration.Configuration @@ -48,33 +46,11 @@ private HangfireDbContext(string databasePath, string prefix = "hangfire") DateFormatString = "yyyy-MM-dd HH:mm:ss.fff" }); - Database = new SQLiteConnection(databasePath, SQLiteOpenFlags.ReadWrite | - SQLiteOpenFlags.Create | SQLiteOpenFlags.FullMutex, storeDateTimeAsTicks: true); + Database = connection; ConnectionId = Guid.NewGuid().ToString(); } - - /// - /// - /// - /// - /// - /// - internal static HangfireDbContext Instance(string databasePath, string prefix = "hangfire") - { - if (_instance != null) return _instance; - lock (Locker) - { - if (_instance == null) - { - _instance = new HangfireDbContext(databasePath, prefix); - } - } - - return _instance; - } - /// /// Initializes initial tables schema for Hangfire /// @@ -82,7 +58,7 @@ public void Init(SQLiteStorageOptions storageOptions) { StorageOptions = storageOptions; - AutoClean(storageOptions); + InitializePragmas(storageOptions); Database.CreateTable(); Database.CreateTable(); @@ -109,11 +85,20 @@ public void Init(SQLiteStorageOptions storageOptions) DistributedLockRepository = Database.Table(); } - private void AutoClean(SQLiteStorageOptions storageOptions) + private void InitializePragmas(SQLiteStorageOptions storageOptions) { try { - Database.Execute($@"PRAGMA auto_vacuum = '{(int)storageOptions.AutoVacuumSelected}'"); + Database.ExecuteScalar($"PRAGMA journal_mode = {storageOptions.JournalMode}", Array.Empty()); + } + catch (Exception ex) + { + Logger.Log(LogLevel.Error, () => $"Error set journal mode. Details: {ex}"); + } + + try + { + Database.ExecuteScalar($"PRAGMA auto_vacuum = '{(int)storageOptions.AutoVacuumSelected}'", Array.Empty()); } catch (Exception ex) { diff --git a/src/main/Hangfire.Storage.SQLite/ObjectExtensions.cs b/src/main/Hangfire.Storage.SQLite/ObjectExtensions.cs index 8ccfae7..cdab391 100644 --- a/src/main/Hangfire.Storage.SQLite/ObjectExtensions.cs +++ b/src/main/Hangfire.Storage.SQLite/ObjectExtensions.cs @@ -2,7 +2,7 @@ namespace Hangfire.Storage.SQLite { - public static class ObjectExtensions + internal static class ObjectExtensions { /// /// diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index afec872..ff3b510 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -1,6 +1,8 @@ using Hangfire.Server; +using SQLite; using System; using System.Collections.Generic; +using Hangfire.Logging; namespace Hangfire.Storage.SQLite { @@ -35,16 +37,29 @@ public SQLiteStorage(string databasePath) /// SQLite connection string /// Storage options public SQLiteStorage(string databasePath, SQLiteStorageOptions storageOptions) + : this(new SQLiteConnection( + string.IsNullOrWhiteSpace(databasePath) ? throw new ArgumentNullException(nameof(databasePath)) : databasePath, + SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.FullMutex, + storeDateTimeAsTicks: true + ), storageOptions) { - if (string.IsNullOrWhiteSpace(databasePath)) + } + + /// + /// Constructs Job Storage by database connection string and options + /// + /// SQLite connection + /// Storage options + public SQLiteStorage(SQLiteConnection dbConnection, SQLiteStorageOptions storageOptions) + { + if (dbConnection == null) { - throw new ArgumentNullException(nameof(databasePath)); + throw new ArgumentNullException(nameof(dbConnection)); } - _connectionString = databasePath; _storageOptions = storageOptions ?? throw new ArgumentNullException(nameof(storageOptions)); - Connection = HangfireDbContext.Instance(databasePath, storageOptions.Prefix); + Connection = new HangfireDbContext(dbConnection, storageOptions.Prefix); Connection.Init(_storageOptions); var defaultQueueProvider = new SQLiteJobQueueProvider(_storageOptions); @@ -67,9 +82,7 @@ public override IMonitoringApi GetMonitoringApi() /// Database context public HangfireDbContext CreateAndOpenConnection() { - return _connectionString != null - ? HangfireDbContext.Instance(_connectionString, _storageOptions.Prefix) - : null; + return Connection; } /// diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs index 70d3149..69296c3 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs @@ -47,5 +47,27 @@ public static IGlobalConfiguration UseSQLiteStorage( return configuration.UseStorage(storage); } + + /// + /// + /// + /// + /// Existing connection to use + /// + /// + /// + public static IGlobalConfiguration UseSQLiteStorage( + [NotNull] this IGlobalConfiguration configuration, + [NotNull] global::SQLite.SQLiteConnection connection, + SQLiteStorageOptions options = null) + { + if (configuration == null) throw new ArgumentNullException(nameof(configuration)); + if (connection == null) throw new ArgumentNullException(nameof(connection)); + if (options == null) options = new SQLiteStorageOptions(); + + var storage = new SQLiteStorage(connection, options); + + return configuration.UseStorage(storage); + } } } diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs index 612ce18..378b14f 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs @@ -105,5 +105,17 @@ public enum AutoVacuum FULL = 1, INCREMENTAL = 2 } + + public JournalModes JournalMode { get; set; } = JournalModes.DELETE; + + public enum JournalModes + { + DELETE, + TRUNCATE, + PERSIST, + MEMORY, + WAL, + OFF + } } } diff --git a/src/test/Hangfire.Storage.SQLite.Test/Utils/CleanDatabaseAttribute.cs b/src/test/Hangfire.Storage.SQLite.Test/Utils/CleanDatabaseAttribute.cs deleted file mode 100644 index 303c54c..0000000 --- a/src/test/Hangfire.Storage.SQLite.Test/Utils/CleanDatabaseAttribute.cs +++ /dev/null @@ -1,50 +0,0 @@ -using Hangfire.Storage.SQLite.Entities; -using System; -using System.Reflection; -using System.Threading; -using Xunit.Sdk; - -namespace Hangfire.Storage.SQLite.Test.Utils -{ - public class CleanDatabaseAttribute : BeforeAfterTestAttribute - { - private static readonly object GlobalLock = new object(); - - public override void Before(MethodInfo methodUnderTest) - { - Monitor.Enter(GlobalLock); - - RecreateDatabaseAndInstallObjects(); - } - - public override void After(MethodInfo methodUnderTest) - { - Monitor.Exit(GlobalLock); - } - - private static void RecreateDatabaseAndInstallObjects() - { - var context = ConnectionUtils.CreateConnection(); - try - { - context.Init(new SQLiteStorageOptions()); - - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - context.Database.DeleteAll(); - } - catch (Exception ex) - { - throw new InvalidOperationException("Unable to cleanup database.", ex); - } - } - } -} \ No newline at end of file From 533ab052a3e41763604587226863deaed9277bb9 Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Sun, 25 Jun 2023 16:34:38 +0200 Subject: [PATCH 06/21] UnitTests use Unique in-memory database instance each. allowing much improved unittest speeds --- .../CountersAggregatorFacts.cs | 3 +- .../ExpirationManagerFacts.cs | 37 ++-- .../SQLiteConnectionFacts.cs | 165 +++++++++--------- .../SQLiteDistributedLockFacts.cs | 17 +- .../SQLiteFetchedJobFacts.cs | 9 +- .../SQLiteJobQueueFacts.cs | 21 ++- .../SQLiteMonitoringApiFacts.cs | 27 ++- .../SQLiteStorageOptionsFacts.cs | 1 - .../SQLiteWriteOnlyTransactionFacts.cs | 87 +++++---- .../Utils/ConnectionUtils.cs | 28 ++- 10 files changed, 198 insertions(+), 197 deletions(-) diff --git a/src/test/Hangfire.Storage.SQLite.Test/CountersAggregatorFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/CountersAggregatorFacts.cs index b5f01c6..e7a5d51 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/CountersAggregatorFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/CountersAggregatorFacts.cs @@ -6,10 +6,9 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class CountersAggregatorFacts { - [Fact, CleanDatabase] + [Fact] public void CountersAggregatorExecutesProperly() { var storage = ConnectionUtils.CreateStorage(); diff --git a/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs index 3fdc484..6b746f9 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs @@ -7,7 +7,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class ExpirationManagerFacts { private readonly SQLiteStorage _storage; @@ -29,40 +28,40 @@ public void Ctor_ThrowsAnException_WhenStorageIsNull() Assert.Throws(() => new ExpirationManager(null)); } - [Fact, CleanDatabase] + [Fact] public void Execute_RemovesOutdatedRecords() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; CreateExpirationEntries(connection, DateTime.UtcNow.AddMonths(-1)); var manager = CreateManager(); manager.Execute(_token); Assert.True(IsEntryExpired(connection)); } - [Fact, CleanDatabase] + [Fact] public void Execute_DoesNotRemoveEntries_WithNoExpirationTimeSet() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; CreateExpirationEntries(connection, null); var manager = CreateManager(); manager.Execute(_token); Assert.False(IsEntryExpired(connection)); } - [Fact, CleanDatabase] + [Fact] public void Execute_DoesNotRemoveEntries_WithFreshExpirationTime() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; CreateExpirationEntries(connection, DateTime.UtcNow.AddMonths(1)); var manager = CreateManager(); manager.Execute(_token); Assert.False(IsEntryExpired(connection)); } - [Fact, CleanDatabase] + [Fact] public void Execute_Processes_CounterTable() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; connection.Database.Insert(new Counter { Id = Guid.NewGuid().ToString(), @@ -76,10 +75,10 @@ public void Execute_Processes_CounterTable() Assert.Equal(0, count); } - [Fact, CleanDatabase] + [Fact] public void Execute_Processes_JobTable() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; connection.Database.Insert(new HangfireJob() { InvocationData = "", @@ -93,10 +92,10 @@ public void Execute_Processes_JobTable() Assert.Equal(0, count); } - [Fact, CleanDatabase] + [Fact] public void Execute_Processes_ListTable() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; connection.Database.Insert(new HangfireList() { Key = "key", @@ -110,10 +109,10 @@ public void Execute_Processes_ListTable() Assert.Equal(0, count); } - [Fact, CleanDatabase] + [Fact] public void Execute_Processes_SetTable() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; connection.Database.Insert(new Set { Key = "key", @@ -129,10 +128,10 @@ public void Execute_Processes_SetTable() Assert.Equal(0, count); } - [Fact, CleanDatabase] + [Fact] public void Execute_Processes_HashTable() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; connection.Database.Insert(new Hash() { Key = "key", @@ -149,10 +148,10 @@ public void Execute_Processes_HashTable() } - [Fact, CleanDatabase] + [Fact] public void Execute_Processes_AggregatedCounterTable() { - var connection = ConnectionUtils.CreateConnection(); + var connection = _storage.Connection; connection.Database.Insert(new AggregatedCounter { Key = "key", diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs index 72dbbeb..797f522 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs @@ -13,7 +13,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class HangfireSQLiteConnectionFacts { private readonly Mock _queue; @@ -38,7 +37,7 @@ public void Ctor_ThrowsAnException_WhenConnectionIsNull() Assert.Equal("database", exception.ParamName); } - [Fact, CleanDatabase] + [Fact] public void Ctor_ThrowsAnException_WhenProvidersCollectionIsNull() { var exception = Assert.Throws( @@ -47,7 +46,7 @@ public void Ctor_ThrowsAnException_WhenProvidersCollectionIsNull() Assert.Equal("queueProviders", exception.ParamName); } - [Fact, CleanDatabase] + [Fact] public void FetchNextJob_DelegatesItsExecution_ToTheQueue() { UseConnection((database, connection) => @@ -61,7 +60,7 @@ public void FetchNextJob_DelegatesItsExecution_ToTheQueue() }); } - [Fact, CleanDatabase] + [Fact] public void FetchNextJob_Throws_IfMultipleProvidersResolved() { UseConnection((database, connection) => @@ -75,7 +74,7 @@ public void FetchNextJob_Throws_IfMultipleProvidersResolved() }); } - [Fact, CleanDatabase] + [Fact] public void CreateWriteTransaction_ReturnsNonNullInstance() { UseConnection((database, connection) => @@ -85,7 +84,7 @@ public void CreateWriteTransaction_ReturnsNonNullInstance() }); } - [Fact, CleanDatabase] + [Fact] public void AcquireLock_ReturnsNonNullInstance() { UseConnection((database, connection) => @@ -95,7 +94,7 @@ public void AcquireLock_ReturnsNonNullInstance() }); } - [Fact, CleanDatabase] + [Fact] public void CreateExpiredJob_ThrowsAnException_WhenJobIsNull() { UseConnection((database, connection) => @@ -111,7 +110,7 @@ public void CreateExpiredJob_ThrowsAnException_WhenJobIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void CreateExpiredJob_ThrowsAnException_WhenParametersCollectionIsNull() { UseConnection((database, connection) => @@ -127,7 +126,7 @@ public void CreateExpiredJob_ThrowsAnException_WhenParametersCollectionIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void CreateExpiredJob_CreatesAJobInTheStorage_AndSetsItsParameters() { UseConnection((database, connection) => @@ -172,14 +171,14 @@ public void CreateExpiredJob_CreatesAJobInTheStorage_AndSetsItsParameters() }); } - [Fact, CleanDatabase] + [Fact] public void GetJobData_ThrowsAnException_WhenJobIdIsNull() { UseConnection((database, connection) => Assert.Throws( () => connection.GetJobData(null))); } - [Fact, CleanDatabase] + [Fact] public void GetJobData_ReturnsNull_WhenThereIsNoSuchJob() { UseConnection((database, connection) => @@ -189,7 +188,7 @@ public void GetJobData_ReturnsNull_WhenThereIsNoSuchJob() }); } - [Fact, CleanDatabase] + [Fact] public void GetJobData_ReturnsResult_WhenJobExists() { UseConnection((database, connection) => @@ -218,7 +217,7 @@ public void GetJobData_ReturnsResult_WhenJobExists() }); } - [Fact, CleanDatabase] + [Fact] public void GetStateData_ThrowsAnException_WhenJobIdIsNull() { UseConnection( @@ -226,7 +225,7 @@ public void GetStateData_ThrowsAnException_WhenJobIdIsNull() () => connection.GetStateData(null))); } - [Fact, CleanDatabase] + [Fact] public void GetStateData_ReturnsNull_IfThereIsNoSuchState() { UseConnection((database, connection) => @@ -236,7 +235,7 @@ public void GetStateData_ReturnsNull_IfThereIsNoSuchState() }); } - [Fact, CleanDatabase] + [Fact] public void GetStateData_ReturnsCorrectData() { UseConnection((database, connection) => @@ -285,7 +284,7 @@ public void GetStateData_ReturnsCorrectData() }); } - [Fact, CleanDatabase] + [Fact] public void GetJobData_ReturnsJobLoadException_IfThereWasADeserializationException() { UseConnection((database, connection) => @@ -307,7 +306,7 @@ public void GetJobData_ReturnsJobLoadException_IfThereWasADeserializationExcepti }); } - [Fact, CleanDatabase] + [Fact] public void SetParameter_ThrowsAnException_WhenJobIdIsNull() { UseConnection((database, connection) => @@ -319,7 +318,7 @@ public void SetParameter_ThrowsAnException_WhenJobIdIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void SetParameter_ThrowsAnException_WhenNameIsNull() { UseConnection((database, connection) => @@ -331,7 +330,7 @@ public void SetParameter_ThrowsAnException_WhenNameIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void SetParameters_CreatesNewParameter_WhenParameterWithTheGivenNameDoesNotExists() { UseConnection((database, connection) => @@ -360,7 +359,7 @@ public void SetParameters_CreatesNewParameter_WhenParameterWithTheGivenNameDoesN }); } - [Fact, CleanDatabase] + [Fact] public void SetParameter_UpdatesValue_WhenParameterWithTheGivenName_AlreadyExists() { UseConnection((database, connection) => @@ -389,7 +388,7 @@ public void SetParameter_UpdatesValue_WhenParameterWithTheGivenName_AlreadyExist }); } - [Fact, CleanDatabase] + [Fact] public void SetParameter_CanAcceptNulls_AsValues() { UseConnection((database, connection) => @@ -416,7 +415,7 @@ public void SetParameter_CanAcceptNulls_AsValues() }); } - [Fact, CleanDatabase] + [Fact] public void GetParameter_ThrowsAnException_WhenJobIdIsNull() { UseConnection((database, connection) => @@ -428,7 +427,7 @@ public void GetParameter_ThrowsAnException_WhenJobIdIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetParameter_ThrowsAnException_WhenNameIsNull() { UseConnection((database, connection) => @@ -440,7 +439,7 @@ public void GetParameter_ThrowsAnException_WhenNameIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetParameter_ReturnsNull_WhenParameterDoesNotExists() { UseConnection((database, connection) => @@ -450,7 +449,7 @@ public void GetParameter_ReturnsNull_WhenParameterDoesNotExists() }); } - [Fact, CleanDatabase] + [Fact] public void GetParameter_ReturnsParameterValue_WhenJobExists() { UseConnection((database, connection) => @@ -472,7 +471,7 @@ public void GetParameter_ReturnsParameterValue_WhenJobExists() }); } - [Fact, CleanDatabase] + [Fact] public void GetFirstByLowestScoreFromSet_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -484,14 +483,14 @@ public void GetFirstByLowestScoreFromSet_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetFirstByLowestScoreFromSet_ThrowsAnException_ToScoreIsLowerThanFromScore() { UseConnection((database, connection) => Assert.Throws( () => connection.GetFirstByLowestScoreFromSet("key", 0, -1))); } - [Fact, CleanDatabase] + [Fact] public void GetFirstByLowestScoreFromSet_ReturnsNull_WhenTheKeyDoesNotExist() { UseConnection((database, connection) => @@ -503,7 +502,7 @@ public void GetFirstByLowestScoreFromSet_ReturnsNull_WhenTheKeyDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetFirstByLowestScoreFromSet_ReturnsTheValueWithTheLowestScore() { UseConnection((database, connection) => @@ -539,7 +538,7 @@ public void GetFirstByLowestScoreFromSet_ReturnsTheValueWithTheLowestScore() }); } - [Fact, CleanDatabase] + [Fact] public void AnnounceServer_ThrowsAnException_WhenServerIdIsNull() { UseConnection((database, connection) => @@ -551,7 +550,7 @@ public void AnnounceServer_ThrowsAnException_WhenServerIdIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void AnnounceServer_ThrowsAnException_WhenContextIsNull() { UseConnection((database, connection) => @@ -563,7 +562,7 @@ public void AnnounceServer_ThrowsAnException_WhenContextIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void AnnounceServer_CreatesOrUpdatesARecord() { UseConnection((database, connection) => @@ -594,14 +593,14 @@ public void AnnounceServer_CreatesOrUpdatesARecord() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveServer_ThrowsAnException_WhenServerIdIsNull() { UseConnection((database, connection) => Assert.Throws( () => connection.RemoveServer(null))); } - [Fact, CleanDatabase] + [Fact] public void RemoveServer_RemovesAServerRecord() { UseConnection((database, connection) => @@ -626,21 +625,21 @@ public void RemoveServer_RemovesAServerRecord() }); } - [Fact, CleanDatabase] + [Fact] public void Heartbeat_ThrowsBackgroundServerGoneException_WhenGivenServerDoesNotExist() { UseConnection((database, connection) => Assert.Throws( () => connection.Heartbeat(Guid.NewGuid().ToString()))); } - [Fact, CleanDatabase] + [Fact] public void Heartbeat_ThrowsAnException_WhenServerIdIsNull() { UseConnection((database, connection) => Assert.Throws( () => connection.Heartbeat(null))); } - [Fact, CleanDatabase] + [Fact] public void Heartbeat_UpdatesLastHeartbeat_OfTheServerWithGivenId() { UseConnection((database, connection) => @@ -668,14 +667,14 @@ public void Heartbeat_UpdatesLastHeartbeat_OfTheServerWithGivenId() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveTimedOutServers_ThrowsAnException_WhenTimeOutIsNegative() { UseConnection((database, connection) => Assert.Throws( () => connection.RemoveTimedOutServers(TimeSpan.FromMinutes(-5)))); } - [Fact, CleanDatabase] + [Fact] public void RemoveTimedOutServers_DoItsWorkPerfectly() { UseConnection((database, connection) => @@ -700,14 +699,14 @@ public void RemoveTimedOutServers_DoItsWorkPerfectly() }); } - [Fact, CleanDatabase] + [Fact] public void GetAllItemsFromSet_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => Assert.Throws(() => connection.GetAllItemsFromSet(null))); } - [Fact, CleanDatabase] + [Fact] public void GetAllItemsFromSet_ReturnsEmptyCollection_WhenKeyDoesNotExist() { UseConnection((database, connection) => @@ -719,7 +718,7 @@ public void GetAllItemsFromSet_ReturnsEmptyCollection_WhenKeyDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetAllItemsFromSet_ReturnsAllItems_InCorrectOrder() { UseConnection((database, connection) => @@ -772,7 +771,7 @@ public void GetAllItemsFromSet_ReturnsAllItems_InCorrectOrder() }); } - [Fact, CleanDatabase] + [Fact] public void SetRangeInHash_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -784,7 +783,7 @@ public void SetRangeInHash_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void SetRangeInHash_ThrowsAnException_WhenKeyValuePairsArgumentIsNull() { UseConnection((database, connection) => @@ -796,7 +795,7 @@ public void SetRangeInHash_ThrowsAnException_WhenKeyValuePairsArgumentIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void SetRangeInHash_MergesAllRecords() { UseConnection((database, connection) => @@ -815,14 +814,14 @@ public void SetRangeInHash_MergesAllRecords() }); } - [Fact, CleanDatabase] + [Fact] public void GetAllEntriesFromHash_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => Assert.Throws(() => connection.GetAllEntriesFromHash(null))); } - [Fact, CleanDatabase] + [Fact] public void GetAllEntriesFromHash_ReturnsNull_IfHashDoesNotExist() { UseConnection((database, connection) => @@ -832,7 +831,7 @@ public void GetAllEntriesFromHash_ReturnsNull_IfHashDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetAllEntriesFromHash_ReturnsAllKeysAndTheirValues() { UseConnection((database, connection) => @@ -868,7 +867,7 @@ public void GetAllEntriesFromHash_ReturnsAllKeysAndTheirValues() }); } - [Fact, CleanDatabase] + [Fact] public void GetSetCount_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -878,7 +877,7 @@ public void GetSetCount_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetSetCount_ReturnsZero_WhenSetDoesNotExist() { UseConnection((database, connection) => @@ -888,7 +887,7 @@ public void GetSetCount_ReturnsZero_WhenSetDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetSetCount_ReturnsNumberOfElements_InASet() { UseConnection((database, connection) => @@ -915,7 +914,7 @@ public void GetSetCount_ReturnsNumberOfElements_InASet() }); } - [Fact, CleanDatabase] + [Fact] public void GetRangeFromSet_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -924,7 +923,7 @@ public void GetRangeFromSet_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetRangeFromSet_ReturnsPagedElementsInCorrectOrder() { UseConnection((database, connection) => @@ -977,7 +976,7 @@ public void GetRangeFromSet_ReturnsPagedElementsInCorrectOrder() }); } - [Fact, CleanDatabase] + [Fact] public void GetSetTtl_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -986,7 +985,7 @@ public void GetSetTtl_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetSetTtl_ReturnsNegativeValue_WhenSetDoesNotExist() { UseConnection((database, connection) => @@ -996,7 +995,7 @@ public void GetSetTtl_ReturnsNegativeValue_WhenSetDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetSetTtl_ReturnsExpirationTime_OfAGivenSet() { UseConnection((database, connection) => @@ -1027,7 +1026,7 @@ public void GetSetTtl_ReturnsExpirationTime_OfAGivenSet() }); } - [Fact, CleanDatabase] + [Fact] public void GetCounter_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1037,7 +1036,7 @@ public void GetCounter_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetCounter_ReturnsZero_WhenKeyDoesNotExist() { UseConnection((database, connection) => @@ -1047,7 +1046,7 @@ public void GetCounter_ReturnsZero_WhenKeyDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetCounter_ReturnsSumOfValues_InCounterTable() { UseConnection((database, connection) => @@ -1077,7 +1076,7 @@ public void GetCounter_ReturnsSumOfValues_InCounterTable() }); } - [Fact, CleanDatabase] + [Fact] public void GetCounter_IncludesValues_FromCounterAggregateTable() { UseConnection((database, connection) => @@ -1101,7 +1100,7 @@ public void GetCounter_IncludesValues_FromCounterAggregateTable() }); } - [Fact, CleanDatabase] + [Fact] public void GetHashCount_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1110,7 +1109,7 @@ public void GetHashCount_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetHashCount_ReturnsZero_WhenKeyDoesNotExist() { UseConnection((database, connection) => @@ -1120,7 +1119,7 @@ public void GetHashCount_ReturnsZero_WhenKeyDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetHashCount_ReturnsNumber_OfHashFields() { UseConnection((database, connection) => @@ -1150,7 +1149,7 @@ public void GetHashCount_ReturnsNumber_OfHashFields() }); } - [Fact, CleanDatabase] + [Fact] public void GetHashTtl_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1160,7 +1159,7 @@ public void GetHashTtl_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetHashTtl_ReturnsNegativeValue_WhenHashDoesNotExist() { UseConnection((database, connection) => @@ -1170,7 +1169,7 @@ public void GetHashTtl_ReturnsNegativeValue_WhenHashDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetHashTtl_ReturnsExpirationTimeForHash() { UseConnection((database, connection) => @@ -1198,7 +1197,7 @@ public void GetHashTtl_ReturnsExpirationTimeForHash() }); } - [Fact, CleanDatabase] + [Fact] public void GetValueFromHash_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1210,7 +1209,7 @@ public void GetValueFromHash_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetValueFromHash_ThrowsAnException_WhenNameIsNull() { UseConnection((database, connection) => @@ -1222,7 +1221,7 @@ public void GetValueFromHash_ThrowsAnException_WhenNameIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetValueFromHash_ReturnsNull_WhenHashDoesNotExist() { UseConnection((database, connection) => @@ -1232,7 +1231,7 @@ public void GetValueFromHash_ReturnsNull_WhenHashDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetValueFromHash_ReturnsValue_OfAGivenField() { UseConnection((database, connection) => @@ -1265,7 +1264,7 @@ public void GetValueFromHash_ReturnsValue_OfAGivenField() }); } - [Fact, CleanDatabase] + [Fact] public void GetListCount_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1275,7 +1274,7 @@ public void GetListCount_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetListCount_ReturnsZero_WhenListDoesNotExist() { UseConnection((database, connection) => @@ -1285,7 +1284,7 @@ public void GetListCount_ReturnsZero_WhenListDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetListCount_ReturnsTheNumberOfListElements() { UseConnection((database, connection) => @@ -1312,7 +1311,7 @@ public void GetListCount_ReturnsTheNumberOfListElements() }); } - [Fact, CleanDatabase] + [Fact] public void GetListTtl_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1322,7 +1321,7 @@ public void GetListTtl_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetListTtl_ReturnsNegativeValue_WhenListDoesNotExist() { UseConnection((database, connection) => @@ -1332,7 +1331,7 @@ public void GetListTtl_ReturnsNegativeValue_WhenListDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetListTtl_ReturnsExpirationTimeForList() { UseConnection((database, connection) => @@ -1358,7 +1357,7 @@ public void GetListTtl_ReturnsExpirationTimeForList() }); } - [Fact, CleanDatabase] + [Fact] public void GetRangeFromList_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1370,7 +1369,7 @@ public void GetRangeFromList_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetRangeFromList_ReturnsAnEmptyList_WhenListDoesNotExist() { UseConnection((database, connection) => @@ -1380,7 +1379,7 @@ public void GetRangeFromList_ReturnsAnEmptyList_WhenListDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetRangeFromList_ReturnsAllEntries_WithinGivenBounds() { UseConnection((database, connection) => @@ -1420,7 +1419,7 @@ public void GetRangeFromList_ReturnsAllEntries_WithinGivenBounds() }); } - [Fact, CleanDatabase] + [Fact] public void GetRangeFromList_ReturnsAllEntriesInCorrectOrder() { UseConnection((database, connection) => @@ -1465,7 +1464,7 @@ public void GetRangeFromList_ReturnsAllEntriesInCorrectOrder() }); } - [Fact, CleanDatabase] + [Fact] public void GetAllItemsFromList_ThrowsAnException_WhenKeyIsNull() { UseConnection((database, connection) => @@ -1475,7 +1474,7 @@ public void GetAllItemsFromList_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void GetAllItemsFromList_ReturnsAnEmptyList_WhenListDoesNotExist() { UseConnection((database, connection) => @@ -1485,7 +1484,7 @@ public void GetAllItemsFromList_ReturnsAnEmptyList_WhenListDoesNotExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetAllItemsFromList_ReturnsAllItemsFromAGivenList_InCorrectOrder() { UseConnection((database, connection) => diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs index 6b901ff..6f718ba 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs @@ -7,7 +7,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class SQLiteDistributedLockFacts { [Fact] @@ -31,7 +30,7 @@ public void Ctor_ThrowsAnException_WhenConnectionIsNull() Assert.Equal("database", exception.ParamName); } - [Fact, CleanDatabase] + [Fact] public void Ctor_SetLock_WhenResourceIsNotLocked() { UseConnection(database => @@ -46,7 +45,7 @@ public void Ctor_SetLock_WhenResourceIsNotLocked() }); } - [Fact, CleanDatabase] + [Fact] public void Ctor_SetReleaseLock_WhenResourceIsNotLocked() { UseConnection(database => @@ -62,7 +61,7 @@ public void Ctor_SetReleaseLock_WhenResourceIsNotLocked() }); } - [Fact, CleanDatabase] + [Fact] public void Ctor_AcquireLockWithinSameThread_WhenResourceIsLocked() { UseConnection(database => @@ -81,7 +80,7 @@ public void Ctor_AcquireLockWithinSameThread_WhenResourceIsLocked() }); } - [Fact, CleanDatabase] + [Fact] public void Ctor_ThrowsAnException_WhenResourceIsLocked() { UseConnection(database => @@ -102,7 +101,7 @@ public void Ctor_ThrowsAnException_WhenResourceIsLocked() }); } - [Fact, CleanDatabase] + [Fact] public void Ctor_WaitForLock_SignaledAtLockRelease() { UseConnection(database => @@ -128,7 +127,7 @@ public void Ctor_WaitForLock_SignaledAtLockRelease() }); } - [Fact, CleanDatabase] + [Fact] public void Ctor_WaitForLock_OnlySingleLockCanBeAcquired() { var connection = ConnectionUtils.CreateConnection(); @@ -184,7 +183,7 @@ public void Ctor_ThrowsAnException_WhenOptionsIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void Ctor_SetLockExpireAtWorks_WhenResourceIsNotLocked() { UseConnection(database => @@ -202,7 +201,7 @@ public void Ctor_SetLockExpireAtWorks_WhenResourceIsNotLocked() } // see https://github.com/raisedapp/Hangfire.Storage.SQLite/issues/38 - [Fact, CleanDatabase] + [Fact] public void Ctor_SetLockExpireAtWorks_WhenResourceIsLockedAndExpiring() { UseConnection(database => diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs index 4b4b4fd..f5ca728 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs @@ -6,7 +6,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class SQLiteFetchedJobFacts { private const int JobId = 0; @@ -59,7 +58,7 @@ public void Ctor_CorrectlySets_AllInstanceProperties() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromQueue_ReallyDeletesTheJobFromTheQueue() { UseConnection(connection => @@ -79,7 +78,7 @@ public void RemoveFromQueue_ReallyDeletesTheJobFromTheQueue() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromQueue_DoesNotDelete_UnrelatedJobs() { UseConnection(connection => @@ -100,7 +99,7 @@ public void RemoveFromQueue_DoesNotDelete_UnrelatedJobs() }); } - [Fact, CleanDatabase] + [Fact] public void Requeue_SetsFetchedAtValueToNull() { UseConnection(connection => @@ -120,7 +119,7 @@ public void Requeue_SetsFetchedAtValueToNull() }); } - [Fact, CleanDatabase] + [Fact] public void Dispose_SetsFetchedAtValueToNull_IfThereWereNoCallsToComplete() { UseConnection(connection => diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs index b19ad78..6931334 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs @@ -7,7 +7,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class SQLiteJobQueueFacts { private static readonly string[] DefaultQueues = { "default" }; @@ -33,7 +32,7 @@ public void Ctor_ThrowsAnException_WhenOptionsValueIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldThrowAnException_WhenQueuesCollectionIsNull() { UseConnection(connection => @@ -47,7 +46,7 @@ public void Dequeue_ShouldThrowAnException_WhenQueuesCollectionIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldThrowAnException_WhenQueuesCollectionIsEmpty() { UseConnection(connection => @@ -75,7 +74,7 @@ public void Dequeue_ThrowsOperationCanceled_WhenCancellationTokenIsSetAtTheBegin }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldWaitIndefinitely_WhenThereAreNoJobs() { UseConnection(connection => @@ -88,7 +87,7 @@ public void Dequeue_ShouldWaitIndefinitely_WhenThereAreNoJobs() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldFetchAJob_FromTheSpecifiedQueue() { // Arrange @@ -113,7 +112,7 @@ public void Dequeue_ShouldFetchAJob_FromTheSpecifiedQueue() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldLeaveJobInTheQueue_ButSetItsFetchedAtValue() { // Arrange @@ -149,7 +148,7 @@ public void Dequeue_ShouldLeaveJobInTheQueue_ButSetItsFetchedAtValue() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldFetchATimedOutJobs_FromTheSpecifiedQueue() { // Arrange @@ -181,7 +180,7 @@ public void Dequeue_ShouldFetchATimedOutJobs_FromTheSpecifiedQueue() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldSetFetchedAt_OnlyForTheFetchedJob() { // Arrange @@ -228,7 +227,7 @@ public void Dequeue_ShouldSetFetchedAt_OnlyForTheFetchedJob() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldFetchJobs_OnlyFromSpecifiedQueues() { UseConnection(connection => @@ -254,7 +253,7 @@ public void Dequeue_ShouldFetchJobs_OnlyFromSpecifiedQueues() }); } - [Fact, CleanDatabase] + [Fact] public void Dequeue_ShouldFetchJobs_FromMultipleQueuesBasedOnQueuePriority() { UseConnection(connection => @@ -305,7 +304,7 @@ public void Dequeue_ShouldFetchJobs_FromMultipleQueuesBasedOnQueuePriority() }); } - [Fact, CleanDatabase] + [Fact] public void Enqueue_AddsAJobToTheQueue() { UseConnection(connection => diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs index 9d22ac6..7caaf0e 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs @@ -10,7 +10,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class SQLiteMonitoringApiFacts { private const string DefaultQueue = "default"; @@ -33,7 +32,7 @@ public SQLiteMonitoringApiFacts() _providers = new PersistentJobQueueProviderCollection(provider.Object); } - [Fact, CleanDatabase] + [Fact] public void GetStatistics_ReturnsZero_WhenNoJobsExist() { UseMonitoringApi((database, monitoringApi) => @@ -46,7 +45,7 @@ public void GetStatistics_ReturnsZero_WhenNoJobsExist() }); } - [Fact, CleanDatabase] + [Fact] public void GetStatistics_ReturnsExpectedCounts_WhenJobsExist() { UseMonitoringApi((database, monitoringApi) => @@ -66,7 +65,7 @@ public void GetStatistics_ReturnsExpectedCounts_WhenJobsExist() }); } - [Fact, CleanDatabase] + [Fact] public void JobDetails_ReturnsNull_WhenThereIsNoSuchJob() { UseMonitoringApi((database, monitoringApi) => @@ -76,7 +75,7 @@ public void JobDetails_ReturnsNull_WhenThereIsNoSuchJob() }); } - [Fact, CleanDatabase] + [Fact] public void JobDetails_ReturnsResult_WhenJobExists() { UseMonitoringApi((database, monitoringApi) => @@ -93,7 +92,7 @@ public void JobDetails_ReturnsResult_WhenJobExists() }); } - [Fact, CleanDatabase] + [Fact] public void EnqueuedJobs_ReturnsEmpty_WhenThereIsNoJobs() { UseMonitoringApi((database, monitoringApi) => @@ -110,7 +109,7 @@ public void EnqueuedJobs_ReturnsEmpty_WhenThereIsNoJobs() }); } - [Fact, CleanDatabase] + [Fact] public void EnqueuedJobs_ReturnsSingleJob_WhenOneJobExistsThatIsNotFetched() { UseMonitoringApi((database, monitoringApi) => @@ -128,7 +127,7 @@ public void EnqueuedJobs_ReturnsSingleJob_WhenOneJobExistsThatIsNotFetched() }); } - [Fact, CleanDatabase] + [Fact] public void EnqueuedJobs_ReturnsEmpty_WhenOneJobExistsThatIsFetched() { UseMonitoringApi((database, monitoringApi) => @@ -146,7 +145,7 @@ public void EnqueuedJobs_ReturnsEmpty_WhenOneJobExistsThatIsFetched() }); } - [Fact, CleanDatabase] + [Fact] public void EnqueuedJobs_ReturnsUnfetchedJobsOnly_WhenMultipleJobsExistsInFetchedAndUnfetchedStates() { UseMonitoringApi((database, monitoringApi) => @@ -166,7 +165,7 @@ public void EnqueuedJobs_ReturnsUnfetchedJobsOnly_WhenMultipleJobsExistsInFetche }); } - [Fact, CleanDatabase] + [Fact] public void FetchedJobs_ReturnsEmpty_WhenThereIsNoJobs() { UseMonitoringApi((database, monitoringApi) => @@ -183,7 +182,7 @@ public void FetchedJobs_ReturnsEmpty_WhenThereIsNoJobs() }); } - [Fact, CleanDatabase] + [Fact] public void FetchedJobs_ReturnsSingleJob_WhenOneJobExistsThatIsFetched() { UseMonitoringApi((database, monitoringApi) => @@ -201,7 +200,7 @@ public void FetchedJobs_ReturnsSingleJob_WhenOneJobExistsThatIsFetched() }); } - [Fact, CleanDatabase] + [Fact] public void FetchedJobs_ReturnsEmpty_WhenOneJobExistsThatIsNotFetched() { UseMonitoringApi((database, monitoringApi) => @@ -219,7 +218,7 @@ public void FetchedJobs_ReturnsEmpty_WhenOneJobExistsThatIsNotFetched() }); } - [Fact, CleanDatabase] + [Fact] public void FetchedJobs_ReturnsFetchedJobsOnly_WhenMultipleJobsExistsInFetchedAndUnfetchedStates() { UseMonitoringApi((database, monitoringApi) => @@ -239,7 +238,7 @@ public void FetchedJobs_ReturnsFetchedJobsOnly_WhenMultipleJobsExistsInFetchedAn }); } - [Fact, CleanDatabase] + [Fact] public void ProcessingJobs_ReturnsProcessingJobsOnly_WhenMultipleJobsExistsInProcessingSucceededAndEnqueuedState() { UseMonitoringApi((database, monitoringApi) => diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteStorageOptionsFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteStorageOptionsFacts.cs index 41aabf7..f555762 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteStorageOptionsFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteStorageOptionsFacts.cs @@ -5,7 +5,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class SQLiteStorageOptionsFacts { [Fact] diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs index e6403b9..d24f525 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs @@ -10,7 +10,6 @@ namespace Hangfire.Storage.SQLite.Test { - [Collection("Database")] public class SQLiteWriteOnlyTransactionFacts { private readonly PersistentJobQueueProviderCollection _queueProviders; @@ -32,7 +31,7 @@ public void Ctor_ThrowsAnException_IfConnectionIsNull() Assert.Equal("connection", exception.ParamName); } - [Fact, CleanDatabase] + [Fact] public void Ctor_ThrowsAnException_IfProvidersCollectionIsNull() { var exception = Assert.Throws(() => new SQLiteWriteOnlyTransaction(ConnectionUtils.CreateConnection(), null)); @@ -41,7 +40,7 @@ public void Ctor_ThrowsAnException_IfProvidersCollectionIsNull() } - [Fact, CleanDatabase] + [Fact] public void ExpireJob_SetsJobExpirationData() { UseConnection(database => @@ -77,7 +76,7 @@ public void ExpireJob_SetsJobExpirationData() }); } - [Fact, CleanDatabase] + [Fact] public void PersistJob_ClearsTheJobExpirationData() { UseConnection(database => @@ -113,7 +112,7 @@ public void PersistJob_ClearsTheJobExpirationData() }); } - [Fact, CleanDatabase] + [Fact] public void SetJobState_AppendsAStateAndSetItToTheJob() { UseConnection(database => @@ -163,7 +162,7 @@ public void SetJobState_AppendsAStateAndSetItToTheJob() }); } - [Fact, CleanDatabase] + [Fact] public void AddJobState_JustAddsANewRecordInATable() { UseConnection(database => @@ -200,7 +199,7 @@ public void AddJobState_JustAddsANewRecordInATable() }); } - [Fact, CleanDatabase] + [Fact] public void AddToQueue_CallsEnqueue_OnTargetPersistentQueue() { UseConnection(database => @@ -218,7 +217,7 @@ public void AddToQueue_CallsEnqueue_OnTargetPersistentQueue() }); } - [Fact, CleanDatabase] + [Fact] public void IncrementCounter_AddsRecordToCounterTable_WithPositiveValue() { UseConnection(database => @@ -233,7 +232,7 @@ public void IncrementCounter_AddsRecordToCounterTable_WithPositiveValue() }); } - [Fact, CleanDatabase] + [Fact] public void IncrementCounter_WithExpiry_AddsARecord_WithExpirationTimeSet() { UseConnection(database => @@ -253,7 +252,7 @@ public void IncrementCounter_WithExpiry_AddsARecord_WithExpirationTimeSet() }); } - [Fact, CleanDatabase] + [Fact] public void IncrementCounter_WithExistingKey_AddsAnotherRecord() { UseConnection(database => @@ -270,7 +269,7 @@ public void IncrementCounter_WithExistingKey_AddsAnotherRecord() }); } - [Fact, CleanDatabase] + [Fact] public void DecrementCounter_AddsRecordToCounterTable_WithNegativeValue() { UseConnection(database => @@ -285,7 +284,7 @@ public void DecrementCounter_AddsRecordToCounterTable_WithNegativeValue() }); } - [Fact, CleanDatabase] + [Fact] public void DecrementCounter_WithExpiry_AddsARecord_WithExpirationTimeSet() { UseConnection(database => @@ -305,7 +304,7 @@ public void DecrementCounter_WithExpiry_AddsARecord_WithExpirationTimeSet() }); } - [Fact, CleanDatabase] + [Fact] public void DecrementCounter_WithExistingKey_AddsAnotherRecord() { UseConnection(database => @@ -322,7 +321,7 @@ public void DecrementCounter_WithExistingKey_AddsAnotherRecord() }); } - [Fact, CleanDatabase] + [Fact] public void AddToSet_AddsARecord_IfThereIsNo_SuchKeyAndValue() { UseConnection(database => @@ -337,7 +336,7 @@ public void AddToSet_AddsARecord_IfThereIsNo_SuchKeyAndValue() }); } - [Fact, CleanDatabase] + [Fact] public void AddToSet_AddsARecord_WhenKeyIsExists_ButValuesAreDifferent() { UseConnection(database => @@ -354,7 +353,7 @@ public void AddToSet_AddsARecord_WhenKeyIsExists_ButValuesAreDifferent() }); } - [Fact, CleanDatabase] + [Fact] public void AddToSet_DoesNotAddARecord_WhenBothKeyAndValueAreExist() { UseConnection(database => @@ -371,7 +370,7 @@ public void AddToSet_DoesNotAddARecord_WhenBothKeyAndValueAreExist() }); } - [Fact, CleanDatabase] + [Fact] public void AddToSet_WithScore_AddsARecordWithScore_WhenBothKeyAndValueAreNotExist() { UseConnection(database => @@ -386,7 +385,7 @@ public void AddToSet_WithScore_AddsARecordWithScore_WhenBothKeyAndValueAreNotExi }); } - [Fact, CleanDatabase] + [Fact] public void AddToSet_WithScore_UpdatesAScore_WhenBothKeyAndValueAreExist() { UseConnection(database => @@ -403,7 +402,7 @@ public void AddToSet_WithScore_UpdatesAScore_WhenBothKeyAndValueAreExist() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromSet_RemovesARecord_WithGivenKeyAndValue() { UseConnection(database => @@ -423,7 +422,7 @@ public void RemoveFromSet_RemovesARecord_WithGivenKeyAndValue() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromSet_DoesNotRemoveRecord_WithSameKey_AndDifferentValue() { UseConnection(database => @@ -440,7 +439,7 @@ public void RemoveFromSet_DoesNotRemoveRecord_WithSameKey_AndDifferentValue() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromSet_DoesNotRemoveRecord_WithSameValue_AndDifferentKey() { UseConnection(database => @@ -457,7 +456,7 @@ public void RemoveFromSet_DoesNotRemoveRecord_WithSameValue_AndDifferentKey() }); } - [Fact, CleanDatabase] + [Fact] public void InsertToList_AddsARecord_WithGivenValues() { UseConnection(database => @@ -471,7 +470,7 @@ public void InsertToList_AddsARecord_WithGivenValues() }); } - [Fact, CleanDatabase] + [Fact] public void InsertToList_AddsAnotherRecord_WhenBothKeyAndValueAreExist() { UseConnection(database => @@ -488,7 +487,7 @@ public void InsertToList_AddsAnotherRecord_WhenBothKeyAndValueAreExist() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromList_RemovesAllRecords_WithGivenKeyAndValue() { UseConnection(database => @@ -506,7 +505,7 @@ public void RemoveFromList_RemovesAllRecords_WithGivenKeyAndValue() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromList_DoesNotRemoveRecords_WithSameKey_ButDifferentValue() { UseConnection(database => @@ -523,7 +522,7 @@ public void RemoveFromList_DoesNotRemoveRecords_WithSameKey_ButDifferentValue() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveFromList_DoesNotRemoveRecords_WithSameValue_ButDifferentKey() { UseConnection(database => @@ -540,7 +539,7 @@ public void RemoveFromList_DoesNotRemoveRecords_WithSameValue_ButDifferentKey() }); } - [Fact, CleanDatabase] + [Fact] public void TrimList_TrimsAList_ToASpecifiedRange() { UseConnection(database => @@ -562,7 +561,7 @@ public void TrimList_TrimsAList_ToASpecifiedRange() }); } - [Fact, CleanDatabase] + [Fact] public void TrimList_RemovesRecordsToEnd_IfKeepAndingAt_GreaterThanMaxElementIndex() { UseConnection(database => @@ -581,7 +580,7 @@ public void TrimList_RemovesRecordsToEnd_IfKeepAndingAt_GreaterThanMaxElementInd }); } - [Fact, CleanDatabase] + [Fact] public void TrimList_RemovesAllRecords_WhenStartingFromValue_GreaterThanMaxElementIndex() { UseConnection(database => @@ -598,7 +597,7 @@ public void TrimList_RemovesAllRecords_WhenStartingFromValue_GreaterThanMaxEleme }); } - [Fact, CleanDatabase] + [Fact] public void TrimList_RemovesAllRecords_IfStartFromGreaterThanEndingAt() { UseConnection(database => @@ -615,7 +614,7 @@ public void TrimList_RemovesAllRecords_IfStartFromGreaterThanEndingAt() }); } - [Fact, CleanDatabase] + [Fact] public void TrimList_RemovesRecords_OnlyOfAGivenKey() { UseConnection(database => @@ -632,7 +631,7 @@ public void TrimList_RemovesRecords_OnlyOfAGivenKey() }); } - [Fact, CleanDatabase] + [Fact] public void SetRangeInHash_ThrowsAnException_WhenKeyIsNull() { UseConnection(database => @@ -644,7 +643,7 @@ public void SetRangeInHash_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void SetRangeInHash_ThrowsAnException_WhenKeyValuePairsArgumentIsNull() { UseConnection(database => @@ -656,7 +655,7 @@ public void SetRangeInHash_ThrowsAnException_WhenKeyValuePairsArgumentIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void SetRangeInHash_MergesAllRecords() { UseConnection(database => @@ -675,7 +674,7 @@ public void SetRangeInHash_MergesAllRecords() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveHash_ThrowsAnException_WhenKeyIsNull() { UseConnection(database => @@ -685,7 +684,7 @@ public void RemoveHash_ThrowsAnException_WhenKeyIsNull() }); } - [Fact, CleanDatabase] + [Fact] public void RemoveHash_RemovesAllHashRecords() { UseConnection(database => @@ -706,7 +705,7 @@ public void RemoveHash_RemovesAllHashRecords() }); } - [Fact, CleanDatabase] + [Fact] public void ExpireSet_SetsSetExpirationData() { UseConnection(database => @@ -728,7 +727,7 @@ public void ExpireSet_SetsSetExpirationData() }); } - [Fact, CleanDatabase] + [Fact] public void ExpireList_SetsListExpirationData() { UseConnection(database => @@ -749,7 +748,7 @@ public void ExpireList_SetsListExpirationData() }); } - [Fact, CleanDatabase] + [Fact] public void ExpireHash_SetsHashExpirationData() { UseConnection(database => @@ -771,7 +770,7 @@ public void ExpireHash_SetsHashExpirationData() } - [Fact, CleanDatabase] + [Fact] public void PersistSet_ClearsTheSetExpirationData() { UseConnection(database => @@ -792,7 +791,7 @@ public void PersistSet_ClearsTheSetExpirationData() }); } - [Fact, CleanDatabase] + [Fact] public void PersistList_ClearsTheListExpirationData() { UseConnection(database => @@ -813,7 +812,7 @@ public void PersistList_ClearsTheListExpirationData() }); } - [Fact, CleanDatabase] + [Fact] public void PersistHash_ClearsTheHashExpirationData() { UseConnection(database => @@ -834,7 +833,7 @@ public void PersistHash_ClearsTheHashExpirationData() }); } - [Fact, CleanDatabase] + [Fact] public void AddRangeToSet_AddToExistingSetData() { UseConnection(database => @@ -866,7 +865,7 @@ public void AddRangeToSet_AddToExistingSetData() } - [Fact, CleanDatabase] + [Fact] public void RemoveSet_ClearsTheSetData() { UseConnection(database => diff --git a/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs b/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs index 0d4f5de..1e8dca2 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs @@ -1,14 +1,10 @@ +using System; +using SQLite; + namespace Hangfire.Storage.SQLite.Test.Utils { public static class ConnectionUtils { - private const string Ext = "db"; - - private static string GetConnectionString() - { - return $"Hangfire-Tests.{Ext}"; - } - public static SQLiteStorage CreateStorage() { var storageOptions = new SQLiteStorageOptions(); @@ -18,8 +14,22 @@ public static SQLiteStorage CreateStorage() public static SQLiteStorage CreateStorage(SQLiteStorageOptions storageOptions) { - var connectionString = GetConnectionString(); - return new SQLiteStorage(connectionString, storageOptions); + // See SQLite Docs: https://www.sqlite.org/c3ref/c_open_autoproxy.html + const SQLiteOpenFlags SQLITE_OPEN_MEMORY = (SQLiteOpenFlags) 0x00000080; + + const SQLiteOpenFlags flags = // open the database in memory + SQLITE_OPEN_MEMORY | + // open the database in read/write mode + SQLiteOpenFlags.ReadWrite | + // create the database if it doesn't exist + SQLiteOpenFlags.Create | + // enable multi-threaded database access + SQLiteOpenFlags.SharedCache; + + var connection = new SQLiteConnection($"hangfire_{Guid.NewGuid():n}.db", + flags, + storeDateTimeAsTicks: true); + return new SQLiteStorage(connection, storageOptions); } public static HangfireDbContext CreateConnection() From fa36c04a02ce2799c02868737e394ac35ece3fd3 Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 20:00:15 +0200 Subject: [PATCH 07/21] Add PooledHangfireDbContext, HangfireSQLiteConnection now disposes open connections, ... --- .../ExpirationManager.cs | 24 ++-- .../HangfireDbContext.cs | 124 ++++++++++++------ .../HangfireSQLiteConnection.cs | 6 + .../SQLiteDistributedLock.cs | 6 +- .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 70 ++++++---- .../SQLiteStorageExtensions.cs | 8 +- .../SQLiteWriteOnlyTransaction.cs | 2 +- .../ExpirationManagerFacts.cs | 18 +-- .../SQLiteConnectionFacts.cs | 8 +- .../SQLiteDistributedLockFacts.cs | 27 ++-- .../SQLiteFetchedJobFacts.cs | 2 +- .../SQLiteJobQueueFacts.cs | 2 +- .../SQLiteMonitoringApiFacts.cs | 2 +- .../SQLiteWriteOnlyTransactionFacts.cs | 2 +- .../Utils/ConnectionUtils.cs | 28 ++-- 15 files changed, 212 insertions(+), 117 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/ExpirationManager.cs b/src/main/Hangfire.Storage.SQLite/ExpirationManager.cs index 87cae4f..e3cb2bd 100644 --- a/src/main/Hangfire.Storage.SQLite/ExpirationManager.cs +++ b/src/main/Hangfire.Storage.SQLite/ExpirationManager.cs @@ -76,20 +76,22 @@ public void Execute([NotNull] BackgroundProcessContext context) /// Cancellation token public void Execute(CancellationToken cancellationToken) { - HangfireDbContext connection = _storage.CreateAndOpenConnection(); - var storageConnection = connection.StorageOptions; - - foreach (var table in ProcessedTables) + using (HangfireDbContext connection = _storage.CreateAndOpenConnection()) { - Logger.Info($"Removing outdated records from the '{table}' table..."); + var storageConnection = connection.StorageOptions; - int affected; - do + foreach (var table in ProcessedTables) { - affected = RemoveExpireRows(connection, table); - } while (affected == NumberOfRecordsInSinglePass); + Logger.Info($"Removing outdated records from the '{table}' table..."); + + int affected; + do + { + affected = RemoveExpireRows(connection, table); + } while (affected == NumberOfRecordsInSinglePass); - Logger.Info($"Outdated records removed from the '{table}' table..."); + Logger.Info($"Outdated records removed from the '{table}' table..."); + } } cancellationToken.WaitHandle.WaitOne(_checkInterval); @@ -128,4 +130,4 @@ private int RemoveExpireRows(HangfireDbContext db, string table) return rowsAffected; } } -} +} \ No newline at end of file diff --git a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs index abd278a..c51ea27 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs @@ -3,20 +3,43 @@ using Newtonsoft.Json; using SQLite; using System; +using System.Threading; namespace Hangfire.Storage.SQLite { + internal class PooledHangfireDbContext : HangfireDbContext + { + private readonly Action _onDispose; + public bool PhaseOut { get; set; } + + internal PooledHangfireDbContext(SQLiteConnection connection, Action onDispose, string prefix = "hangfire") + : base(connection, prefix) + { + _onDispose = onDispose; + } + + protected override void Dispose(bool disposing) + { + if (PhaseOut) + { + base.Dispose(disposing); + return; + } + _onDispose(this); + } + } + /// /// Represents SQLite database context for Hangfire /// - public class HangfireDbContext + public class HangfireDbContext : IDisposable { private readonly ILog Logger = LogProvider.For(); /// /// /// - public SQLiteConnection Database { get; } + public SQLiteConnection Database { get; private set; } /// /// @@ -50,7 +73,7 @@ internal HangfireDbContext(SQLiteConnection connection, string prefix = "hangfir ConnectionId = Guid.NewGuid().ToString(); } - + /// /// Initializes initial tables schema for Hangfire /// @@ -58,33 +81,41 @@ public void Init(SQLiteStorageOptions storageOptions) { StorageOptions = storageOptions; - InitializePragmas(storageOptions); - - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - Database.CreateTable(); - - AggregatedCounterRepository = Database.Table(); - CounterRepository = Database.Table(); - HangfireJobRepository = Database.Table(); - HangfireListRepository = Database.Table(); - HashRepository = Database.Table(); - JobParameterRepository = Database.Table(); - JobQueueRepository = Database.Table(); - HangfireServerRepository = Database.Table(); - SetRepository = Database.Table(); - StateRepository = Database.Table(); - DistributedLockRepository = Database.Table(); + TryFewTimesDueToConcurrency(() => InitializePragmas(storageOptions)); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + TryFewTimesDueToConcurrency(() => Database.CreateTable()); + + void TryFewTimesDueToConcurrency(Action action, int times = 10) + { + var current = 0; + while (current < times) + { + try + { + action(); + return; + } + catch (SQLiteException e) when (e.Result == SQLite3.Result.Locked) + { + // This can happen if too many connections are opened + // at the same time, trying to create tables + Thread.Sleep(10); + } + current++; + } + } } + private void InitializePragmas(SQLiteStorageOptions storageOptions) { try @@ -106,26 +137,41 @@ private void InitializePragmas(SQLiteStorageOptions storageOptions) } } - public TableQuery AggregatedCounterRepository { get; private set; } + public TableQuery AggregatedCounterRepository => Database.Table(); + + public TableQuery CounterRepository => Database.Table(); - public TableQuery CounterRepository { get; private set; } + public TableQuery HangfireJobRepository => Database.Table(); - public TableQuery HangfireJobRepository { get; private set; } + public TableQuery HangfireListRepository => Database.Table(); - public TableQuery HangfireListRepository { get; private set; } + public TableQuery HashRepository => Database.Table(); - public TableQuery HashRepository { get; private set; } + public TableQuery JobParameterRepository => Database.Table(); - public TableQuery JobParameterRepository { get; private set; } + public TableQuery JobQueueRepository => Database.Table(); - public TableQuery JobQueueRepository { get; private set; } + public TableQuery HangfireServerRepository => Database.Table(); - public TableQuery HangfireServerRepository { get; private set; } + public TableQuery SetRepository => Database.Table(); - public TableQuery SetRepository { get; private set; } + public TableQuery StateRepository => Database.Table(); - public TableQuery StateRepository { get; private set; } + public TableQuery DistributedLockRepository => Database.Table(); - public TableQuery DistributedLockRepository { get; private set; } + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + Database?.Dispose(); + Database = null; + } + GC.SuppressFinalize(this); + } + + public void Dispose() + { + Dispose(true); + } } } diff --git a/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs b/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs index e9a733f..ca546aa 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs @@ -38,6 +38,12 @@ public HangfireSQLiteConnection( _queueProviders = queueProviders ?? throw new ArgumentNullException(nameof(queueProviders)); } + public override void Dispose() + { + DbContext.Dispose(); + base.Dispose(); + } + public override IDisposable AcquireDistributedLock(string resource, TimeSpan timeout) { return new SQLiteDistributedLock($"HangFire:{resource}", timeout, DbContext, _storageOptions); diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs index 01c3150..11cbd3b 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs @@ -162,11 +162,11 @@ private void Acquire(TimeSpan timeout) } catch (DistributedLockTimeoutException ex) { - throw ex; + throw; } catch (Exception ex) { - throw ex; + throw; } } @@ -185,7 +185,7 @@ private void Release() } catch (Exception ex) { - throw ex; + throw; } } diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index ff3b510..2fe3f85 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -1,22 +1,36 @@ using Hangfire.Server; using SQLite; using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Threading; using Hangfire.Logging; +using Hangfire.Storage.SQLite.Entities; namespace Hangfire.Storage.SQLite { + public class SQLiteDbConnectionFactory + { + private readonly Func _getConnection; + + public SQLiteDbConnectionFactory(Func getConnection) + { + _getConnection = getConnection; + } + + public SQLiteConnection Create() + { + return _getConnection(); + } + } + public class SQLiteStorage : JobStorage { private readonly string _connectionString; + private readonly SQLiteDbConnectionFactory _dbConnectionFactory; private readonly SQLiteStorageOptions _storageOptions; - /// - /// Database context - /// - public HangfireDbContext Connection { get; } - /// /// Queue providers collection /// @@ -37,52 +51,62 @@ public SQLiteStorage(string databasePath) /// SQLite connection string /// Storage options public SQLiteStorage(string databasePath, SQLiteStorageOptions storageOptions) - : this(new SQLiteConnection( - string.IsNullOrWhiteSpace(databasePath) ? throw new ArgumentNullException(nameof(databasePath)) : databasePath, - SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.FullMutex, - storeDateTimeAsTicks: true - ), storageOptions) + : this(new SQLiteDbConnectionFactory(() => new SQLiteConnection( + string.IsNullOrWhiteSpace(databasePath) ? throw new ArgumentNullException(nameof(databasePath)) : databasePath, + SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.FullMutex, + storeDateTimeAsTicks: true + ) {BusyTimeout = TimeSpan.FromSeconds(10)}), storageOptions) { } /// /// Constructs Job Storage by database connection string and options /// - /// SQLite connection + /// Factory that creates SQLite connections /// Storage options - public SQLiteStorage(SQLiteConnection dbConnection, SQLiteStorageOptions storageOptions) + public SQLiteStorage(SQLiteDbConnectionFactory dbConnectionFactory, SQLiteStorageOptions storageOptions) { - if (dbConnection == null) - { - throw new ArgumentNullException(nameof(dbConnection)); - } - + _dbConnectionFactory = dbConnectionFactory ?? throw new ArgumentNullException(nameof(dbConnectionFactory)); _storageOptions = storageOptions ?? throw new ArgumentNullException(nameof(storageOptions)); - Connection = new HangfireDbContext(dbConnection, storageOptions.Prefix); - Connection.Init(_storageOptions); - var defaultQueueProvider = new SQLiteJobQueueProvider(_storageOptions); QueueProviders = new PersistentJobQueueProviderCollection(defaultQueueProvider); + + using (var dbContext = CreateAndOpenConnection()) + { + // Use this to initialize the database as soon as possible + // in case of error, the user will immediately get an exception at startup + } } public override IStorageConnection GetConnection() { - return new HangfireSQLiteConnection(Connection, _storageOptions, QueueProviders); + var dbContext = CreateAndOpenConnection(); + return new HangfireSQLiteConnection(dbContext, _storageOptions, QueueProviders); } public override IMonitoringApi GetMonitoringApi() { - return new SQLiteMonitoringApi(Connection, QueueProviders); + var dbContext = CreateAndOpenConnection(); + return new SQLiteMonitoringApi(dbContext, QueueProviders); } + private readonly ConcurrentQueue _dbContextPool = new ConcurrentQueue(); + /// /// Opens connection to database /// /// Database context public HangfireDbContext CreateAndOpenConnection() { - return Connection; + if (_dbContextPool.TryDequeue(out var dbContext)) + { + return dbContext; + } + + dbContext = new PooledHangfireDbContext(_dbConnectionFactory.Create(), ctx => _dbContextPool.Enqueue(ctx), _storageOptions.Prefix); + dbContext.Init(_storageOptions); + return dbContext; } /// diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs index 69296c3..7579d0d 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorageExtensions.cs @@ -52,20 +52,20 @@ public static IGlobalConfiguration UseSQLiteStorage( /// /// /// - /// Existing connection to use + /// connection factory to use /// /// /// public static IGlobalConfiguration UseSQLiteStorage( [NotNull] this IGlobalConfiguration configuration, - [NotNull] global::SQLite.SQLiteConnection connection, + [NotNull] SQLiteDbConnectionFactory connectionFactory, SQLiteStorageOptions options = null) { if (configuration == null) throw new ArgumentNullException(nameof(configuration)); - if (connection == null) throw new ArgumentNullException(nameof(connection)); + if (connectionFactory == null) throw new ArgumentNullException(nameof(connectionFactory)); if (options == null) options = new SQLiteStorageOptions(); - var storage = new SQLiteStorage(connection, options); + var storage = new SQLiteStorage(connectionFactory, options); return configuration.UseStorage(storage); } diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs index f97069c..322da3d 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs @@ -330,7 +330,7 @@ public override void SetJobState(string jobId, IState state) { _.Database.Rollback(); - throw ex; + throw; } } }); diff --git a/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs index 6b746f9..772ca94 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/ExpirationManagerFacts.cs @@ -31,7 +31,7 @@ public void Ctor_ThrowsAnException_WhenStorageIsNull() [Fact] public void Execute_RemovesOutdatedRecords() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); CreateExpirationEntries(connection, DateTime.UtcNow.AddMonths(-1)); var manager = CreateManager(); manager.Execute(_token); @@ -41,7 +41,7 @@ public void Execute_RemovesOutdatedRecords() [Fact] public void Execute_DoesNotRemoveEntries_WithNoExpirationTimeSet() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); CreateExpirationEntries(connection, null); var manager = CreateManager(); manager.Execute(_token); @@ -51,7 +51,7 @@ public void Execute_DoesNotRemoveEntries_WithNoExpirationTimeSet() [Fact] public void Execute_DoesNotRemoveEntries_WithFreshExpirationTime() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); CreateExpirationEntries(connection, DateTime.UtcNow.AddMonths(1)); var manager = CreateManager(); manager.Execute(_token); @@ -61,7 +61,7 @@ public void Execute_DoesNotRemoveEntries_WithFreshExpirationTime() [Fact] public void Execute_Processes_CounterTable() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); connection.Database.Insert(new Counter { Id = Guid.NewGuid().ToString(), @@ -78,7 +78,7 @@ public void Execute_Processes_CounterTable() [Fact] public void Execute_Processes_JobTable() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); connection.Database.Insert(new HangfireJob() { InvocationData = "", @@ -95,7 +95,7 @@ public void Execute_Processes_JobTable() [Fact] public void Execute_Processes_ListTable() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); connection.Database.Insert(new HangfireList() { Key = "key", @@ -112,7 +112,7 @@ public void Execute_Processes_ListTable() [Fact] public void Execute_Processes_SetTable() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); connection.Database.Insert(new Set { Key = "key", @@ -131,7 +131,7 @@ public void Execute_Processes_SetTable() [Fact] public void Execute_Processes_HashTable() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); connection.Database.Insert(new Hash() { Key = "key", @@ -151,7 +151,7 @@ public void Execute_Processes_HashTable() [Fact] public void Execute_Processes_AggregatedCounterTable() { - var connection = _storage.Connection; + using var connection = _storage.CreateAndOpenConnection(); connection.Database.Insert(new AggregatedCounter { Key = "key", diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs index 797f522..03d301f 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteConnectionFacts.cs @@ -1525,11 +1525,9 @@ public void GetAllItemsFromList_ReturnsAllItemsFromAGivenList_InCorrectOrder() } private void UseConnection(Action action) { - var database = ConnectionUtils.CreateConnection(); - using (var connection = new HangfireSQLiteConnection(database, _providers)) - { - action(database, connection); - } + using var database = ConnectionUtils.CreateConnection(); + using var connection = new HangfireSQLiteConnection(database, _providers); + action(database, connection); } public static void SampleMethod(string arg) diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs index 6f718ba..62b6e8c 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteDistributedLockFacts.cs @@ -104,19 +104,25 @@ public void Ctor_ThrowsAnException_WhenResourceIsLocked() [Fact] public void Ctor_WaitForLock_SignaledAtLockRelease() { - UseConnection(database => + var storage = ConnectionUtils.CreateStorage(); + using var mre = new ManualResetEventSlim(); + var t = new Thread(() => { - var t = new Thread(() => + UseConnection(database => { using (new SQLiteDistributedLock("resource1", TimeSpan.Zero, database, new SQLiteStorageOptions())) { + mre.Set(); Thread.Sleep(TimeSpan.FromSeconds(3)); } - }); + }, storage); + }); + UseConnection(database => + { t.Start(); // Wait just a bit to make sure the above lock is acuired - Thread.Sleep(TimeSpan.FromSeconds(1)); + mre.Wait(TimeSpan.FromSeconds(5)); // Record when we try to aquire the lock var startTime = DateTime.UtcNow; @@ -124,21 +130,22 @@ public void Ctor_WaitForLock_SignaledAtLockRelease() { Assert.InRange(DateTime.UtcNow - startTime, TimeSpan.Zero, TimeSpan.FromSeconds(5)); } - }); + }, storage); } [Fact] public void Ctor_WaitForLock_OnlySingleLockCanBeAcquired() { - var connection = ConnectionUtils.CreateConnection(); var numThreads = 10; long concurrencyCounter = 0; - var manualResetEvent = new ManualResetEventSlim(); + using var manualResetEvent = new ManualResetEventSlim(); var success = new bool[numThreads]; - + var storage = ConnectionUtils.CreateStorage(); + // Spawn multiple threads to race each other. var threads = Enumerable.Range(0, numThreads).Select(i => new Thread(() => { + using var connection = storage.CreateAndOpenConnection(); // Wait for the start signal. manualResetEvent.Wait(); @@ -224,9 +231,9 @@ public void Ctor_SetLockExpireAtWorks_WhenResourceIsLockedAndExpiring() }); } - private static void UseConnection(Action action) + private static void UseConnection(Action action, SQLiteStorage storage = null) { - var connection = ConnectionUtils.CreateConnection(); + using var connection = storage?.CreateAndOpenConnection() ?? ConnectionUtils.CreateConnection(); action(connection); } } diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs index f5ca728..ee90fd3 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteFetchedJobFacts.cs @@ -155,7 +155,7 @@ private static int CreateJobQueueRecord(HangfireDbContext connection, int jobId, private static void UseConnection(Action action) { - var connection = ConnectionUtils.CreateConnection(); + using var connection = ConnectionUtils.CreateConnection(); action(connection); } } diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs index 6931334..4b0986c 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteJobQueueFacts.cs @@ -333,7 +333,7 @@ private static SQLiteJobQueue CreateJobQueue(HangfireDbContext connection) private static void UseConnection(Action action) { - var connection = ConnectionUtils.CreateConnection(); + using var connection = ConnectionUtils.CreateConnection(); action(connection); } } diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs index 7caaf0e..acf551f 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs @@ -282,7 +282,7 @@ public void ProcessingJobs_ReturnsProcessingJobsOnly_WhenMultipleJobsExistsInPro private void UseMonitoringApi(Action action) { - var database = ConnectionUtils.CreateConnection(); + using var database = ConnectionUtils.CreateConnection(); var connection = new SQLiteMonitoringApi(database, _providers); action(database, connection); } diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs index d24f525..f74ea12 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteWriteOnlyTransactionFacts.cs @@ -911,7 +911,7 @@ private static dynamic GetTestHash(HangfireDbContext database, string key) private void UseConnection(Action action) { - HangfireDbContext connection = ConnectionUtils.CreateConnection(); + using var connection = ConnectionUtils.CreateConnection(); action(connection); } diff --git a/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs b/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs index 1e8dca2..6afd0fb 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs @@ -16,25 +16,37 @@ public static SQLiteStorage CreateStorage(SQLiteStorageOptions storageOptions) { // See SQLite Docs: https://www.sqlite.org/c3ref/c_open_autoproxy.html const SQLiteOpenFlags SQLITE_OPEN_MEMORY = (SQLiteOpenFlags) 0x00000080; - + const SQLiteOpenFlags SQLITE_OPEN_URI = (SQLiteOpenFlags) 0x00000040; const SQLiteOpenFlags flags = // open the database in memory - SQLITE_OPEN_MEMORY | + // SQLITE_OPEN_MEMORY | + // for whatever reason, if we don't use URI-mode, shared in-memory databases dont work. + SQLITE_OPEN_URI | // open the database in read/write mode SQLiteOpenFlags.ReadWrite | // create the database if it doesn't exist SQLiteOpenFlags.Create | // enable multi-threaded database access - SQLiteOpenFlags.SharedCache; + SQLiteOpenFlags.NoMutex; - var connection = new SQLiteConnection($"hangfire_{Guid.NewGuid():n}.db", - flags, - storeDateTimeAsTicks: true); - return new SQLiteStorage(connection, storageOptions); + var dbId = $"file:hangfire_{Guid.NewGuid():n}.db?mode=memory&cache=shared"; + return new SQLiteStorage(new SQLiteDbConnectionFactory(() => + new SQLiteConnection(dbId, + flags, + storeDateTimeAsTicks: true) + { + BusyTimeout = TimeSpan.FromSeconds(10), + }), + storageOptions); } + /// + /// Only use this if you have a single thread. + /// For multi-threaded tests, use directly and + /// then call per Thread. + /// public static HangfireDbContext CreateConnection() { - return CreateStorage().Connection; + return CreateStorage().CreateAndOpenConnection(); } } } \ No newline at end of file From fbed9715d306089ec1c16e1ae94a4ec17c2ec7b5 Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 20:19:33 +0200 Subject: [PATCH 08/21] switch to NoMutex, as we now have scoped connections --- src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index 2fe3f85..df950f9 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -53,7 +53,7 @@ public SQLiteStorage(string databasePath) public SQLiteStorage(string databasePath, SQLiteStorageOptions storageOptions) : this(new SQLiteDbConnectionFactory(() => new SQLiteConnection( string.IsNullOrWhiteSpace(databasePath) ? throw new ArgumentNullException(nameof(databasePath)) : databasePath, - SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.FullMutex, + SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.NoMutex, storeDateTimeAsTicks: true ) {BusyTimeout = TimeSpan.FromSeconds(10)}), storageOptions) { From 8348eed8295ea3a77d5eaa0924a1e0839cd35e1c Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 20:20:45 +0200 Subject: [PATCH 09/21] introduce dbContextPool Limit of 10 --- .../HangfireDbContext.cs | 22 ------------ .../PooledHangfireDbContext.cs | 26 ++++++++++++++ .../SQLiteDbConnectionFactory.cs | 20 +++++++++++ .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 35 ++++++++----------- 4 files changed, 60 insertions(+), 43 deletions(-) create mode 100644 src/main/Hangfire.Storage.SQLite/PooledHangfireDbContext.cs create mode 100644 src/main/Hangfire.Storage.SQLite/SQLiteDbConnectionFactory.cs diff --git a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs index c51ea27..cd36bf6 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs @@ -7,28 +7,6 @@ namespace Hangfire.Storage.SQLite { - internal class PooledHangfireDbContext : HangfireDbContext - { - private readonly Action _onDispose; - public bool PhaseOut { get; set; } - - internal PooledHangfireDbContext(SQLiteConnection connection, Action onDispose, string prefix = "hangfire") - : base(connection, prefix) - { - _onDispose = onDispose; - } - - protected override void Dispose(bool disposing) - { - if (PhaseOut) - { - base.Dispose(disposing); - return; - } - _onDispose(this); - } - } - /// /// Represents SQLite database context for Hangfire /// diff --git a/src/main/Hangfire.Storage.SQLite/PooledHangfireDbContext.cs b/src/main/Hangfire.Storage.SQLite/PooledHangfireDbContext.cs new file mode 100644 index 0000000..d874996 --- /dev/null +++ b/src/main/Hangfire.Storage.SQLite/PooledHangfireDbContext.cs @@ -0,0 +1,26 @@ +using System; +using SQLite; + +namespace Hangfire.Storage.SQLite +{ + internal class PooledHangfireDbContext : HangfireDbContext + { + private readonly Action _onDispose; + public bool PhaseOut { get; set; } + + internal PooledHangfireDbContext(SQLiteConnection connection, Action onDispose, string prefix = "hangfire") + : base(connection, prefix) + { + _onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); + } + + protected override void Dispose(bool disposing) + { + _onDispose(this); + if (PhaseOut) + { + base.Dispose(disposing); + } + } + } +} \ No newline at end of file diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteDbConnectionFactory.cs b/src/main/Hangfire.Storage.SQLite/SQLiteDbConnectionFactory.cs new file mode 100644 index 0000000..9c91fe5 --- /dev/null +++ b/src/main/Hangfire.Storage.SQLite/SQLiteDbConnectionFactory.cs @@ -0,0 +1,20 @@ +using System; +using SQLite; + +namespace Hangfire.Storage.SQLite +{ + public class SQLiteDbConnectionFactory + { + private readonly Func _getConnection; + + public SQLiteDbConnectionFactory(Func getConnection) + { + _getConnection = getConnection; + } + + public SQLiteConnection Create() + { + return _getConnection(); + } + } +} \ No newline at end of file diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index df950f9..7e6c60f 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -3,33 +3,16 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Threading; -using Hangfire.Logging; -using Hangfire.Storage.SQLite.Entities; namespace Hangfire.Storage.SQLite { - public class SQLiteDbConnectionFactory - { - private readonly Func _getConnection; - - public SQLiteDbConnectionFactory(Func getConnection) - { - _getConnection = getConnection; - } - - public SQLiteConnection Create() - { - return _getConnection(); - } - } - public class SQLiteStorage : JobStorage { private readonly string _connectionString; private readonly SQLiteDbConnectionFactory _dbConnectionFactory; private readonly SQLiteStorageOptions _storageOptions; + private ConcurrentQueue _dbContextPool = new ConcurrentQueue(); /// /// Queue providers collection @@ -91,8 +74,6 @@ public override IMonitoringApi GetMonitoringApi() return new SQLiteMonitoringApi(dbContext, QueueProviders); } - private readonly ConcurrentQueue _dbContextPool = new ConcurrentQueue(); - /// /// Opens connection to database /// @@ -104,11 +85,23 @@ public HangfireDbContext CreateAndOpenConnection() return dbContext; } - dbContext = new PooledHangfireDbContext(_dbConnectionFactory.Create(), ctx => _dbContextPool.Enqueue(ctx), _storageOptions.Prefix); + dbContext = new PooledHangfireDbContext(_dbConnectionFactory.Create(), ctx => EnqueueOrPhaseOut(ctx), _storageOptions.Prefix); dbContext.Init(_storageOptions); return dbContext; } + private void EnqueueOrPhaseOut(PooledHangfireDbContext dbContext) + { + if (_dbContextPool.Count < 10) + { + _dbContextPool.Enqueue(dbContext); + } + else + { + dbContext.PhaseOut = true; + } + } + /// /// Returns text representation of the object /// From ae2058f93d7d42148846e1c7b5d32e5b0d269b99 Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 20:23:21 +0200 Subject: [PATCH 10/21] remove SuppressFinalize.this --- src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs index cd36bf6..f37ef4e 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs @@ -144,7 +144,6 @@ protected virtual void Dispose(bool disposing) Database?.Dispose(); Database = null; } - GC.SuppressFinalize(this); } public void Dispose() From 7c9edf7529423ae102df4dc6c8d702589af5582f Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 20:25:39 +0200 Subject: [PATCH 11/21] clarify reason for URI-mode in ConnectionUtls --- .../Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs b/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs index 6afd0fb..28ba152 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/Utils/ConnectionUtils.cs @@ -15,11 +15,13 @@ public static SQLiteStorage CreateStorage() public static SQLiteStorage CreateStorage(SQLiteStorageOptions storageOptions) { // See SQLite Docs: https://www.sqlite.org/c3ref/c_open_autoproxy.html - const SQLiteOpenFlags SQLITE_OPEN_MEMORY = (SQLiteOpenFlags) 0x00000080; + // const SQLiteOpenFlags SQLITE_OPEN_MEMORY = (SQLiteOpenFlags) 0x00000080; const SQLiteOpenFlags SQLITE_OPEN_URI = (SQLiteOpenFlags) 0x00000040; const SQLiteOpenFlags flags = // open the database in memory // SQLITE_OPEN_MEMORY | - // for whatever reason, if we don't use URI-mode, shared in-memory databases dont work. + // SQLiteOpenFlags.SharedCache | + // for whatever reason, if we don't use URI-mode, + // shared in-memory databases dont work properly. SQLITE_OPEN_URI | // open the database in read/write mode SQLiteOpenFlags.ReadWrite | From 8ca3a48c3a40dd60041f704645a156153f6f814c Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 20:40:39 +0200 Subject: [PATCH 12/21] make PoolSize configurable, show _databasePath in ToString --- .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 26 ++++++++++++++++--- .../SQLiteStorageOptions.cs | 16 ++++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index 7e6c60f..d540655 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -6,9 +6,9 @@ namespace Hangfire.Storage.SQLite { - public class SQLiteStorage : JobStorage + public class SQLiteStorage : JobStorage, IDisposable { - private readonly string _connectionString; + private readonly string _databasePath; private readonly SQLiteDbConnectionFactory _dbConnectionFactory; private readonly SQLiteStorageOptions _storageOptions; @@ -57,6 +57,7 @@ public SQLiteStorage(SQLiteDbConnectionFactory dbConnectionFactory, SQLiteStorag using (var dbContext = CreateAndOpenConnection()) { + _databasePath = dbContext.Database.DatabasePath; // Use this to initialize the database as soon as possible // in case of error, the user will immediately get an exception at startup } @@ -92,7 +93,7 @@ public HangfireDbContext CreateAndOpenConnection() private void EnqueueOrPhaseOut(PooledHangfireDbContext dbContext) { - if (_dbContextPool.Count < 10) + if (_dbContextPool.Count < _storageOptions.PoolSize) { _dbContextPool.Enqueue(dbContext); } @@ -107,7 +108,24 @@ private void EnqueueOrPhaseOut(PooledHangfireDbContext dbContext) /// public override string ToString() { - return $"Connection string: {_connectionString}, prefix: {_storageOptions.Prefix}"; + return $"Database path: {_databasePath}, prefix: {_storageOptions.Prefix}"; + } + + private bool _disposed; + public void Dispose() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(SQLiteStorage)); + } + foreach (var dbContext in _dbContextPool) + { + dbContext.PhaseOut = true; + dbContext.Dispose(); + } + + _dbContextPool = null; + _disposed = true; } /// diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs index 378b14f..8d3b5f2 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs @@ -42,6 +42,7 @@ public TimeSpan QueuePollInterval { throw new ArgumentException(message, nameof(value)); } + if (value != value.Duration()) { throw new ArgumentException(message, nameof(value)); @@ -65,6 +66,7 @@ public TimeSpan DistributedLockLifetime { throw new ArgumentException(message, nameof(value)); } + if (value != value.Duration()) { throw new ArgumentException(message, nameof(value)); @@ -95,7 +97,8 @@ public TimeSpan DistributedLockLifetime public TimeSpan CountersAggregateInterval { get; set; } /// - /// Set AutoVacuum Mode In SQLite + /// Set AutoVacuum Mode In SQLite. + /// Defaults to . /// public AutoVacuum AutoVacuumSelected { get; set; } = AutoVacuum.NONE; @@ -106,6 +109,10 @@ public enum AutoVacuum INCREMENTAL = 2 } + /// + /// Set journal_mode in SQLite. + /// Defaults to . + /// public JournalModes JournalMode { get; set; } = JournalModes.DELETE; public enum JournalModes @@ -117,5 +124,10 @@ public enum JournalModes WAL, OFF } + + /// + /// Limits the amount of pooled SQLiteConnections. + /// + public int PoolSize { get; set; } = 10; } -} +} \ No newline at end of file From fa216458fd114faa40b91a43775ccd51a32b2405 Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 22:46:52 +0200 Subject: [PATCH 13/21] leak fix: SQLiteMonitoringApi now uses per-request SQLiteConnection --- .../SQLiteMonitoringApi.cs | 16 +++++++++------- .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 6 ++++-- .../SQLiteStorageOptions.cs | 6 +++--- src/samples/WebSample/Program.cs | 7 ++++++- .../SQLiteMonitoringApiFacts.cs | 7 ++++--- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs b/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs index e2aa0be..3a96f86 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using Hangfire.Common; using Hangfire.States; using Hangfire.Storage.Monitoring; @@ -12,25 +11,28 @@ namespace Hangfire.Storage.SQLite { public class SQLiteMonitoringApi : IMonitoringApi { - private readonly HangfireDbContext _dbContext; + private readonly SQLiteStorage _storage; private readonly PersistentJobQueueProviderCollection _queueProviders; /// /// /// - /// + /// /// - public SQLiteMonitoringApi(HangfireDbContext database, PersistentJobQueueProviderCollection queueProviders) + public SQLiteMonitoringApi(SQLiteStorage storage, PersistentJobQueueProviderCollection queueProviders) { - _dbContext = database; + _storage = storage; _queueProviders = queueProviders; } private T UseConnection(Func action) { - var result = action(_dbContext); - return result; + using (var dbContext = _storage.CreateAndOpenConnection()) + { + var result = action(dbContext); + return result; + } } private JobList GetJobs(HangfireDbContext connection, int from, int count, string stateName, Func, TDto> selector) diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index d540655..4ef9dd1 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -11,6 +11,7 @@ public class SQLiteStorage : JobStorage, IDisposable private readonly string _databasePath; private readonly SQLiteDbConnectionFactory _dbConnectionFactory; + private readonly SQLiteStorageOptions _storageOptions; private ConcurrentQueue _dbContextPool = new ConcurrentQueue(); @@ -71,8 +72,7 @@ public override IStorageConnection GetConnection() public override IMonitoringApi GetMonitoringApi() { - var dbContext = CreateAndOpenConnection(); - return new SQLiteMonitoringApi(dbContext, QueueProviders); + return new SQLiteMonitoringApi(this, QueueProviders); } /// @@ -112,12 +112,14 @@ public override string ToString() } private bool _disposed; + public void Dispose() { if (_disposed) { throw new ObjectDisposedException(nameof(SQLiteStorage)); } + foreach (var dbContext in _dbContextPool) { dbContext.PhaseOut = true; diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs index 8d3b5f2..98923cf 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorageOptions.cs @@ -111,9 +111,9 @@ public enum AutoVacuum /// /// Set journal_mode in SQLite. - /// Defaults to . + /// Defaults to . /// - public JournalModes JournalMode { get; set; } = JournalModes.DELETE; + public JournalModes JournalMode { get; set; } = JournalModes.WAL; public enum JournalModes { @@ -128,6 +128,6 @@ public enum JournalModes /// /// Limits the amount of pooled SQLiteConnections. /// - public int PoolSize { get; set; } = 10; + public int PoolSize { get; set; } = 20; } } \ No newline at end of file diff --git a/src/samples/WebSample/Program.cs b/src/samples/WebSample/Program.cs index 66f7eb1..ecf53ec 100644 --- a/src/samples/WebSample/Program.cs +++ b/src/samples/WebSample/Program.cs @@ -16,7 +16,12 @@ .UseSimpleAssemblyNameTypeSerializer() .UseRecommendedSerializerSettings() .UseSQLiteStorage("Hangfire.db", - new SQLiteStorageOptions() { AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL, JobExpirationCheckInterval = TimeSpan.FromSeconds(30) }) + new SQLiteStorageOptions() + { + AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL, + JournalMode = SQLiteStorageOptions.JournalModes.WAL, + JobExpirationCheckInterval = TimeSpan.FromSeconds(30) + }) .UseHeartbeatPage(checkInterval: TimeSpan.FromSeconds(10)) .UseJobsLogger()); services.AddHangfireServer(); diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs index acf551f..4bc9e9e 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs @@ -282,9 +282,10 @@ public void ProcessingJobs_ReturnsProcessingJobsOnly_WhenMultipleJobsExistsInPro private void UseMonitoringApi(Action action) { - using var database = ConnectionUtils.CreateConnection(); - var connection = new SQLiteMonitoringApi(database, _providers); - action(database, connection); + using var storage = ConnectionUtils.CreateStorage(); + var connection = new SQLiteMonitoringApi(storage, _providers); + using var dbContext = storage.CreateAndOpenConnection(); + action(dbContext, connection); } private HangfireJob CreateJobInState(HangfireDbContext database, string stateName, Func visitor = null) From 301c6c7b8bb0b4fa8b134694d43838aeb2542191 Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Fri, 30 Jun 2023 22:50:52 +0200 Subject: [PATCH 14/21] revert changes in WebSample, as WAL is now default --- src/samples/WebSample/Program.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/samples/WebSample/Program.cs b/src/samples/WebSample/Program.cs index ecf53ec..66f7eb1 100644 --- a/src/samples/WebSample/Program.cs +++ b/src/samples/WebSample/Program.cs @@ -16,12 +16,7 @@ .UseSimpleAssemblyNameTypeSerializer() .UseRecommendedSerializerSettings() .UseSQLiteStorage("Hangfire.db", - new SQLiteStorageOptions() - { - AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL, - JournalMode = SQLiteStorageOptions.JournalModes.WAL, - JobExpirationCheckInterval = TimeSpan.FromSeconds(30) - }) + new SQLiteStorageOptions() { AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL, JobExpirationCheckInterval = TimeSpan.FromSeconds(30) }) .UseHeartbeatPage(checkInterval: TimeSpan.FromSeconds(10)) .UseJobsLogger()); services.AddHangfireServer(); From 8fa1226b5ed1a0c7fceeb39a48179f443ba7752e Mon Sep 17 00:00:00 2001 From: John heckendorf Date: Sat, 1 Jul 2023 12:05:18 +0200 Subject: [PATCH 15/21] Add CheckDisposed to ensure Storage is not used anymore after being Disposed. --- .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index 4ef9dd1..7fe98ed 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace Hangfire.Storage.SQLite { @@ -66,12 +67,14 @@ public SQLiteStorage(SQLiteDbConnectionFactory dbConnectionFactory, SQLiteStorag public override IStorageConnection GetConnection() { + CheckDisposed(); var dbContext = CreateAndOpenConnection(); return new HangfireSQLiteConnection(dbContext, _storageOptions, QueueProviders); } public override IMonitoringApi GetMonitoringApi() { + CheckDisposed(); return new SQLiteMonitoringApi(this, QueueProviders); } @@ -81,6 +84,7 @@ public override IMonitoringApi GetMonitoringApi() /// Database context public HangfireDbContext CreateAndOpenConnection() { + CheckDisposed(); if (_dbContextPool.TryDequeue(out var dbContext)) { return dbContext; @@ -93,6 +97,12 @@ public HangfireDbContext CreateAndOpenConnection() private void EnqueueOrPhaseOut(PooledHangfireDbContext dbContext) { + if (_disposed) + { + dbContext.PhaseOut = true; + return; + } + if (_dbContextPool.Count < _storageOptions.PoolSize) { _dbContextPool.Enqueue(dbContext); @@ -117,7 +127,7 @@ public void Dispose() { if (_disposed) { - throw new ObjectDisposedException(nameof(SQLiteStorage)); + return; } foreach (var dbContext in _dbContextPool) @@ -130,6 +140,22 @@ public void Dispose() _disposed = true; } + private void CheckDisposed() + { + if (_disposed) + { + // This pattern is used to keep the IL-small and the cost for CheckDisposed low + // by not having a throw new... statement inside the check function itself + ThrowObjectDisposedException(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowObjectDisposedException() + { + throw new ObjectDisposedException(nameof(SQLiteStorage)); + } + /// /// /// From dfce307c06b31f77f312c2f7599298814652db54 Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 30 Dec 2023 20:03:33 -0400 Subject: [PATCH 16/21] 0.4.0 --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 32eaa60..464066b 100644 --- a/.gitignore +++ b/.gitignore @@ -329,3 +329,5 @@ ASALocalRun/ # MFractors (Xamarin productivity tool) working folder .mfractor/ *.db +/src/samples/WebSample/Hangfire.db-shm +/src/samples/WebSample/Hangfire.db-wal From 8ca24366a00f96f75fd7028d08008752ea7ad65e Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 30 Dec 2023 20:03:36 -0400 Subject: [PATCH 17/21] 0.4.0 --- src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs | 9 --------- .../Hangfire.Storage.SQLite/SQLiteDistributedLock.cs | 6 +++--- .../SQLiteWriteOnlyTransaction.cs | 2 +- src/samples/WebSample/WebSample.csproj | 7 ++++--- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs index f37ef4e..539b011 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireDbContext.cs @@ -38,15 +38,6 @@ public class HangfireDbContext : IDisposable /// Table prefix internal HangfireDbContext(SQLiteConnection connection, string prefix = "hangfire") { - //UTC - Internal JSON - GlobalConfiguration.Configuration - .UseSerializerSettings(new JsonSerializerSettings() - { - DateTimeZoneHandling = DateTimeZoneHandling.Utc, - DateFormatHandling = DateFormatHandling.IsoDateFormat, - DateFormatString = "yyyy-MM-dd HH:mm:ss.fff" - }); - Database = connection; ConnectionId = Guid.NewGuid().ToString(); diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs index 11cbd3b..71d3043 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs @@ -160,11 +160,11 @@ private void Acquire(TimeSpan timeout) throw new DistributedLockTimeoutException(_resource); } } - catch (DistributedLockTimeoutException ex) + catch (DistributedLockTimeoutException) { throw; } - catch (Exception ex) + catch (Exception) { throw; } @@ -183,7 +183,7 @@ private void Release() lock (EventWaitHandleName) Monitor.Pulse(EventWaitHandleName); } - catch (Exception ex) + catch (Exception) { throw; } diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs index 322da3d..b5a74fc 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs @@ -326,7 +326,7 @@ public override void SetJobState(string jobId, IState state) _.Database.Commit(); } - catch (Exception ex) + catch (Exception) { _.Database.Rollback(); diff --git a/src/samples/WebSample/WebSample.csproj b/src/samples/WebSample/WebSample.csproj index f3fd659..d3dab83 100644 --- a/src/samples/WebSample/WebSample.csproj +++ b/src/samples/WebSample/WebSample.csproj @@ -1,15 +1,16 @@ - net6.0 + net8.0 enable enable - - + + + From aaad37ff0418d6ef339b52b2900d77172e3c8dc5 Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 30 Dec 2023 20:37:05 -0400 Subject: [PATCH 18/21] ... --- .../Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj | 4 ++-- src/samples/WebSample/Program.cs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj index 7096a08..976ff54 100644 --- a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj +++ b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj @@ -2,6 +2,7 @@ netstandard2.0 + true netstandard2.0;net48 @@ -32,8 +33,7 @@ - - + \ No newline at end of file diff --git a/src/samples/WebSample/Program.cs b/src/samples/WebSample/Program.cs index 66f7eb1..cfc5754 100644 --- a/src/samples/WebSample/Program.cs +++ b/src/samples/WebSample/Program.cs @@ -15,8 +15,7 @@ .SetDataCompatibilityLevel(CompatibilityLevel.Version_180) .UseSimpleAssemblyNameTypeSerializer() .UseRecommendedSerializerSettings() - .UseSQLiteStorage("Hangfire.db", - new SQLiteStorageOptions() { AutoVacuumSelected = SQLiteStorageOptions.AutoVacuum.FULL, JobExpirationCheckInterval = TimeSpan.FromSeconds(30) }) + .UseSQLiteStorage("Hangfire.db") .UseHeartbeatPage(checkInterval: TimeSpan.FromSeconds(10)) .UseJobsLogger()); services.AddHangfireServer(); From 6eeeb0a87750780ddeea946c89ae58df813ee972 Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 30 Dec 2023 20:46:58 -0400 Subject: [PATCH 19/21] .... --- src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj index 976ff54..1d40db5 100644 --- a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj +++ b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj @@ -2,7 +2,6 @@ netstandard2.0 - true netstandard2.0;net48 @@ -22,7 +21,6 @@ An Alternative SQLite Storage for Hangfire 0.4.0 - - From 6503d063d86de55402ff55b9ba9adc4e4ed37e6c Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Sat, 27 Jan 2024 09:15:07 -0400 Subject: [PATCH 20/21] V0.4.1 --- .../CountersAggregator.cs | 18 ++- .../Hangfire.Storage.SQLite.csproj | 5 +- .../HangfireSQLiteConnection.cs | 38 +++-- src/main/Hangfire.Storage.SQLite/Retry.cs | 131 ++++++++++++++++++ .../SQLiteDistributedLock.cs | 31 +++-- .../SQLiteMonitoringApi.cs | 81 +++++------ .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 2 +- .../SQLiteWriteOnlyTransaction.cs | 108 ++++++++------- .../SQLiteMonitoringApiFacts.cs | 2 +- 9 files changed, 289 insertions(+), 127 deletions(-) create mode 100644 src/main/Hangfire.Storage.SQLite/Retry.cs diff --git a/src/main/Hangfire.Storage.SQLite/CountersAggregator.cs b/src/main/Hangfire.Storage.SQLite/CountersAggregator.cs index 432d012..66f682c 100644 --- a/src/main/Hangfire.Storage.SQLite/CountersAggregator.cs +++ b/src/main/Hangfire.Storage.SQLite/CountersAggregator.cs @@ -2,9 +2,7 @@ using Hangfire.Logging; using Hangfire.Server; using System; -using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using Hangfire.Storage.SQLite.Entities; @@ -13,9 +11,9 @@ namespace Hangfire.Storage.SQLite /// /// Represents Counter collection aggregator for SQLite database /// - #pragma warning disable CS0618 +#pragma warning disable CS0618 public class CountersAggregator : IBackgroundProcess, IServerComponent - #pragma warning restore CS0618 +#pragma warning restore CS0618 { private static readonly ILog Logger = LogProvider.For(); private const int NumberOfRecordsInSinglePass = 1000; @@ -50,11 +48,17 @@ public void Execute([NotNull] BackgroundProcessContext context) /// /// Cancellation token public void Execute(CancellationToken cancellationToken) + { + // DANIEL WAS HERE + Retry.Twice((_) => _Execute(cancellationToken)); + } + + private void _Execute(CancellationToken cancellationToken) { Logger.DebugFormat("Aggregating records in 'Counter' table..."); long removedCount = 0; - + do { using (var storageConnection = (HangfireSQLiteConnection)_storage.GetConnection()) @@ -97,7 +101,7 @@ public void Execute(CancellationToken cancellationToken) counter.Value += item.Value; counter.ExpireAt = item.ExpireAt > aggregatedItem.ExpireAt ? (item.ExpireAt > DateTime.MinValue ? item.ExpireAt : DateTime.MinValue) - : (aggregatedItem.ExpireAt > DateTime.MinValue ? + : (aggregatedItem.ExpireAt > DateTime.MinValue ? aggregatedItem.ExpireAt : DateTime.MinValue); storageDb.Database.Update(counter); } @@ -134,4 +138,4 @@ public override string ToString() return "SQLite Counter Collection Aggregator"; } } -} +} \ No newline at end of file diff --git a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj index 1d40db5..7c4da02 100644 --- a/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj +++ b/src/main/Hangfire.Storage.SQLite/Hangfire.Storage.SQLite.csproj @@ -7,7 +7,7 @@ netstandard2.0;net48 - 0.4.0 + 0.4.1 RaisedApp RaisedApp Copyright © 2019 - Present @@ -20,7 +20,8 @@ Hangfire Storage SQLite An Alternative SQLite Storage for Hangfire - 0.4.0 + 0.4.1 + - Stability and retry enhancements introduced by: Daniel Lindblom diff --git a/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs b/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs index ca546aa..a5a0034 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs @@ -38,15 +38,11 @@ public HangfireSQLiteConnection( _queueProviders = queueProviders ?? throw new ArgumentNullException(nameof(queueProviders)); } - public override void Dispose() - { - DbContext.Dispose(); - base.Dispose(); - } - public override IDisposable AcquireDistributedLock(string resource, TimeSpan timeout) { - return new SQLiteDistributedLock($"HangFire:{resource}", timeout, DbContext, _storageOptions); + return Retry.Twice((_) => + new SQLiteDistributedLock($"HangFire:{resource}", timeout, DbContext, _storageOptions) + ); } public override void AnnounceServer(string serverId, ServerContext context) @@ -93,6 +89,14 @@ public override string CreateExpiredJob(Job job, IDictionary par if (parameters == null) throw new ArgumentNullException(nameof(parameters)); + // DANIEL WAS HERE + return Retry.Twice( + (attempt) => _CreateExpiredJob(job, parameters, createdAt, expireIn) + ); + } + + private string _CreateExpiredJob(Job job, IDictionary parameters, DateTime createdAt, TimeSpan expireIn) + { lock (_lock) { var invocationData = InvocationData.SerializeJob(job); @@ -343,17 +347,29 @@ public override void Heartbeat(string serverId) throw new ArgumentNullException(nameof(serverId)); } - var server = DbContext.HangfireServerRepository.FirstOrDefault(_ => _.Id == serverId); + // DANIEL WAS HERE: + // Something fishy is going on here, a BackgroundServerGoneException is unexpectedly thrown + // https://github.com/HangfireIO/Hangfire/blob/master/src/Hangfire.Core/Server/ServerHeartbeatProcess.cs + // Changing to + + // DANIEL WAS HERE: + // var server = DbContext.HangfireServerRepository.FirstOrDefault(_ => _.Id == serverId); + var server = Retry.Twice((attempts) => + // Forcing a query (read somewhere that sqlite-net handles FirstOrDefault differently ) + DbContext.HangfireServerRepository.Where(_ => _.Id == serverId) + .ToArray() + .FirstOrDefault() + ); if (server == null) throw new BackgroundServerGoneException(); server.LastHeartbeat = DateTime.UtcNow; - var affected = DbContext.Database.Update(server); + // DANIEL WAS HERE: + // var affected = DbContext.Database.Update(server); + var affected = Retry.Twice((_) => DbContext.Database.Update(server)); if (affected == 0) - { throw new BackgroundServerGoneException(); - } } public override void RemoveServer(string serverId) diff --git a/src/main/Hangfire.Storage.SQLite/Retry.cs b/src/main/Hangfire.Storage.SQLite/Retry.cs new file mode 100644 index 0000000..71ddfe7 --- /dev/null +++ b/src/main/Hangfire.Storage.SQLite/Retry.cs @@ -0,0 +1,131 @@ +using System; +using System.Linq; +using System.Threading; + +namespace Hangfire.Storage.SQLite +{ + + // Daniel Lindblom WAS HERE: + // Added this utility as a light alternative to use Polly since I assume you do not want to drag + // that dependency in to this library. + + public static class Retrying + { + public const int Once = 1; + public const int Twice = 2; + } + + public static class Retry + { + public static TimeSpan DefaultDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public static void Execute(Action action, CancellationToken token, params TimeSpan[] delays) + { + if (delays is null || delays.Length == 0) + delays = new TimeSpan[] { DefaultDelay }; + + var retries = delays.Length; + var tries = 0; + while (!token.IsCancellationRequested) + { + try + { + action(tries + 1); + return; + } + catch + { + var delay = delays[tries % delays.Length]; + if (++tries > retries) throw; + token.WaitHandle.WaitOne(delay); + } + } + throw new OperationCanceledException(); + } + public static void Execute(Action action, params TimeSpan[] delays) + { + Execute(action, CancellationToken.None, delays); + } + + public static void Execute(Action action, int retries = Retrying.Once, TimeSpan? delay = null) + { + delay = delay ?? DefaultDelay; + var delays = Enumerable.Range(1, retries).Select(_ => delay.Value).ToArray(); + Execute(action, delays); + } + + public static void Execute(Action action, CancellationToken token, int retries = Retrying.Once, TimeSpan? delay = null) + { + delay = delay ?? DefaultDelay; + var delays = Enumerable.Range(1, retries).Select(_ => delay.Value).ToArray(); + Execute(action, token, delays); + } + + public static TResult Execute(Func func, CancellationToken token, params TimeSpan[] delays) + { + if (delays is null || delays.Length == 0) + delays = new TimeSpan[] { DefaultDelay }; + + var retries = delays.Length; + var tries = 0; + while (!token.IsCancellationRequested) + { + try + { + return func(tries + 1); + } + catch + { + var delay = delays[tries % delays.Length]; + if (++tries > retries) throw; + token.WaitHandle.WaitOne(delay); + } + } + throw new OperationCanceledException(); + } + + public static TResult Execute(Func func, params TimeSpan[] delays) + { + return Execute(func, CancellationToken.None, delays); + } + + public static TResult Execute(Func func, int retries = Retrying.Once, TimeSpan? delay = null) + { + delay = delay ?? DefaultDelay; + var delays = Enumerable.Range(1, retries).Select(_ => delay.Value).ToArray(); + return Execute(func, delays); + } + + public static TResult Execute(Func func, CancellationToken token, int retries = Retrying.Once, TimeSpan? delay = null) + { + delay = delay ?? DefaultDelay; + var delays = Enumerable.Range(1, retries).Select(_ => delay.Value).ToArray(); + return Execute(func, token, delays); + } + + public static void Once(Action action, CancellationToken? token = null, TimeSpan? delay = null) + { + token = token ?? CancellationToken.None; + delay = delay ?? DefaultDelay; + Execute(action, token.Value, Retrying.Once, delay.Value); + } + public static void Twice(Action action, CancellationToken? token = null, TimeSpan? delay = null) + { + token = token ?? CancellationToken.None; + delay = delay ?? DefaultDelay; + Execute(action, token.Value, Retrying.Twice, delay.Value); + } + public static TResult Once(Func func, CancellationToken? token = null, TimeSpan? delay = null) + { + token = token ?? CancellationToken.None; + delay = delay ?? DefaultDelay; + return Execute(func, token.Value, Retrying.Once, delay.Value); + } + public static TResult Twice(Func func, CancellationToken? token = null, TimeSpan? delay = null) + { + token = token ?? CancellationToken.None; + delay = delay ?? DefaultDelay; + return Execute(func, token.Value, Retrying.Twice, delay.Value); + } + } +} \ No newline at end of file diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs index 71d3043..2d1693b 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs @@ -160,13 +160,13 @@ private void Acquire(TimeSpan timeout) throw new DistributedLockTimeoutException(_resource); } } - catch (DistributedLockTimeoutException) + catch (DistributedLockTimeoutException ex) { - throw; + throw ex; } - catch (Exception) + catch (Exception ex) { - throw; + throw ex; } } @@ -176,26 +176,31 @@ private void Acquire(TimeSpan timeout) /// private void Release() { - try - { + // DANIEL WAS HERE: + Retry.Twice((retry) => { + // Remove resource lock (if it's still ours) _dbContext.DistributedLockRepository.Delete(_ => _.Resource == _resource && _.ResourceKey == _resourceKey); lock (EventWaitHandleName) Monitor.Pulse(EventWaitHandleName); } - catch (Exception) - { - throw; - } + + ); } + private void Cleanup() { try { - // Delete expired locks (of any owner) - _dbContext.DistributedLockRepository. - Delete(x => x.Resource == _resource && x.ExpireAt < DateTime.UtcNow); + // DANIEL WAS HERE: + Retry.Twice((_) => { + + // Delete expired locks (of any owner) + _dbContext.DistributedLockRepository. + Delete(x => x.Resource == _resource && x.ExpireAt < DateTime.UtcNow); + } + ); } catch (Exception ex) { diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs b/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs index 3a96f86..5128117 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using Hangfire.Common; using Hangfire.States; using Hangfire.Storage.Monitoring; @@ -11,28 +12,25 @@ namespace Hangfire.Storage.SQLite { public class SQLiteMonitoringApi : IMonitoringApi { - private readonly SQLiteStorage _storage; + private readonly HangfireDbContext _dbContext; private readonly PersistentJobQueueProviderCollection _queueProviders; /// /// /// - /// + /// /// - public SQLiteMonitoringApi(SQLiteStorage storage, PersistentJobQueueProviderCollection queueProviders) + public SQLiteMonitoringApi(HangfireDbContext database, PersistentJobQueueProviderCollection queueProviders) { - _storage = storage; + _dbContext = database; _queueProviders = queueProviders; } private T UseConnection(Func action) { - using (var dbContext = _storage.CreateAndOpenConnection()) - { - var result = action(dbContext); - return result; - } + var result = action(_dbContext); + return result; } private JobList GetJobs(HangfireDbContext connection, int from, int count, string stateName, Func, TDto> selector) @@ -194,7 +192,7 @@ private Dictionary GetTimelineStats(HangfireDbContext connection var result = new Dictionary(); for (var i = 0; i < stringDates.Count; i++) { - var value = valuesAggregatorMap[$"stats:{type}:{stringDates[i]}"]; + var value = valuesAggregatorMap[valuesAggregatorMap.Keys.ElementAt(i)]; result.Add(dates[i], value.ToInt64()); } @@ -227,7 +225,7 @@ private Dictionary GetHourlyTimelineStats(HangfireDbContext conn var result = new Dictionary(); for (var i = 0; i < dates.Count; i++) { - var value = valuesAggregatorMap[$"stats:{type}:{dates[i]:yyyy-MM-dd-HH}"]; + var value = valuesAggregatorMap[valuesAggregatorMap.Keys.ElementAt(i)]; result.Add(dates[i], value.ToInt64()); } @@ -416,26 +414,29 @@ public JobList FetchedJobs(string queue, int from, int perPage) /// public StatisticsDto GetStatistics() { - return UseConnection(ctx => - { - var stats = new StatisticsDto(); - - int GetCountIfExists(string name) => ctx.HangfireJobRepository.Count(_ => _.StateName == name); - - stats.Enqueued = GetCountIfExists(EnqueuedState.StateName); - stats.Failed = GetCountIfExists(FailedState.StateName); - stats.Processing = GetCountIfExists(ProcessingState.StateName); - stats.Scheduled = GetCountIfExists(ScheduledState.StateName); - stats.Servers = ctx.HangfireServerRepository.Count(); - stats.Succeeded = GetCountIfExists(SucceededState.StateName); - stats.Deleted = GetCountIfExists(DeletedState.StateName); - stats.Recurring = ctx.SetRepository.Count(_ => _.Key == "recurring-jobs"); - stats.Queues = _queueProviders - .SelectMany(x => x.GetJobQueueMonitoringApi(ctx).GetQueues()) - .Count(); - - return stats; - }); + // DANIEL WAS HERE: + return Retry.Twice((attempts) => + + UseConnection(ctx => + { + var stats = new StatisticsDto(); + + int GetCountIfExists(string name) => ctx.HangfireJobRepository.Count(_ => _.StateName == name); + + stats.Enqueued = GetCountIfExists(EnqueuedState.StateName); + stats.Failed = GetCountIfExists(FailedState.StateName); + stats.Processing = GetCountIfExists(ProcessingState.StateName); + stats.Scheduled = GetCountIfExists(ScheduledState.StateName); + stats.Servers = ctx.HangfireServerRepository.Count(); + stats.Succeeded = GetCountIfExists(SucceededState.StateName); + stats.Deleted = GetCountIfExists(DeletedState.StateName); + stats.Recurring = ctx.SetRepository.Count(_ => _.Key == "recurring-jobs"); + stats.Queues = _queueProviders + .SelectMany(x => x.GetJobQueueMonitoringApi(ctx).GetQueues()) + .Count(); + + return stats; + })); } /// @@ -470,19 +471,19 @@ public JobDetailsDto JobDetails(string jobId) var job = _.HangfireJobRepository.FirstOrDefault(x => x.Id == iJobId); var jobHistory = _.StateRepository.Where(x => x.JobId == iJobId).ToList(); var jobParameters = - _.JobParameterRepository - .Where(x => x.JobId == iJobId) - .GroupBy(x => x.Name, x => new { x.Value, x.Id }, (name, val) => new - { - Name = name, - Value = val + _.JobParameterRepository + .Where(x => x.JobId == iJobId) + .GroupBy(x => x.Name, x => new { x.Value, x.Id }, (name, val) => new + { + Name = name, + Value = val .OrderByDescending(x => x.Id) .FirstOrDefault() .Value - }) - .ToList() - .ToDictionary(x => x.Name, x => x.Value); + }) + .ToList() + .ToDictionary(x => x.Name, x => x.Value); if (job == null) return null; diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index 7fe98ed..bde08c6 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -75,7 +75,7 @@ public override IStorageConnection GetConnection() public override IMonitoringApi GetMonitoringApi() { CheckDisposed(); - return new SQLiteMonitoringApi(this, QueueProviders); + return new SQLiteMonitoringApi(CreateAndOpenConnection(), QueueProviders); } /// diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs index b5a74fc..0e796c7 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs @@ -30,12 +30,12 @@ public SQLiteWriteOnlyTransaction(HangfireDbContext connection, _dbContext = connection ?? throw new ArgumentNullException(nameof(connection)); _queueProviders = queueProviders ?? throw new ArgumentNullException(nameof(queueProviders)); } - + private void QueueCommand(Action action) { _commandQueue.Enqueue(action); } - + public override void AddJobState(string jobId, IState state) { QueueCommand(_ => @@ -51,7 +51,7 @@ public override void AddJobState(string jobId, IState state) }; _.Database.Insert(jobState); - }); + }); } /// @@ -67,12 +67,12 @@ public override void AddToQueue(string queue, string jobId) QueueCommand(_ => { persistentQueue.Enqueue(queue, jobId); - }); + }); } public override void AddToSet(string key, string value) { - AddToSet(key, value, 0.0); + AddToSet(key, value, 0.0); } /// @@ -113,13 +113,17 @@ public override void AddToSet(string key, string value, double score) public override void Commit() { - lock (_lockObject) - { - _commandQueue.ToList().ForEach(_ => + // DANIEL WAS HERE + Retry.Twice((attempts) => { + + lock (_lockObject) { - _.Invoke(_dbContext); - }); - } + _commandQueue.ToList().ForEach(_ => + { + _.Invoke(_dbContext); + }); + } + }); } /// @@ -129,7 +133,7 @@ public override void Commit() /// public override void DecrementCounter(string key) { - QueueCommand(_ => + QueueCommand(_ => { _.Database.Insert(new Counter { @@ -140,7 +144,7 @@ public override void DecrementCounter(string key) }); } - + /// /// /// @@ -167,12 +171,12 @@ public override void DecrementCounter(string key, TimeSpan expireIn) /// public override void ExpireJob(string jobId, TimeSpan expireIn) { - QueueCommand(_ => + QueueCommand(_ => { var iJobId = int.Parse(jobId); var job = _.HangfireJobRepository.FirstOrDefault(x => x.Id == iJobId); - if (job != null) + if (job != null) { var expireAt = DateTime.UtcNow.Add(expireIn); job.ExpireAt = expireAt; @@ -181,16 +185,16 @@ public override void ExpireJob(string jobId, TimeSpan expireIn) _.Database.Execute($"UPDATE [{DefaultValues.StateTblName}] SET ExpireAt = {expireAt.Ticks} WHERE JobId = {jobId}"); _.Database.Execute($"UPDATE [{DefaultValues.JobParameterTblName}] SET ExpireAt = {expireAt.Ticks} WHERE JobId = {jobId}"); } - }); + }); } - + /// /// /// /// public override void IncrementCounter(string key) { - QueueCommand(_ => + QueueCommand(_ => { _.Database.Insert(new Counter { @@ -214,7 +218,7 @@ public override void IncrementCounter(string key, TimeSpan expireIn) }); }); } - + /// /// /// @@ -222,7 +226,7 @@ public override void IncrementCounter(string key, TimeSpan expireIn) /// public override void InsertToList(string key, string value) { - QueueCommand(_ => + QueueCommand(_ => { _.Database.Insert(new HangfireList { @@ -232,26 +236,26 @@ public override void InsertToList(string key, string value) }); }); } - + /// /// /// /// public override void PersistJob(string jobId) { - QueueCommand(_ => + QueueCommand(_ => { var iJobId = int.Parse(jobId); var job = _.HangfireJobRepository.FirstOrDefault(x => x.Id == iJobId); - if (job != null) + if (job != null) { job.ExpireAt = DateTime.MinValue; _.Database.Update(job); } }); } - + /// /// /// @@ -259,7 +263,7 @@ public override void PersistJob(string jobId) /// public override void RemoveFromList(string key, string value) { - QueueCommand(_ => + QueueCommand(_ => { _.HangfireListRepository.Delete(x => x.Key == key && x.Value == value); }); @@ -272,7 +276,7 @@ public override void RemoveFromList(string key, string value) /// public override void RemoveFromSet(string key, string value) { - QueueCommand(_ => + QueueCommand(_ => { _.SetRepository.Delete(x => x.Key == key && x.Value == value); }); @@ -288,7 +292,7 @@ public override void RemoveHash(string key) if (key == null) throw new ArgumentNullException(nameof(key)); - QueueCommand(_ => + QueueCommand(_ => { _.HashRepository.Delete(x => x.Key == key); }); @@ -301,7 +305,7 @@ public override void RemoveHash(string key) /// public override void SetJobState(string jobId, IState state) { - QueueCommand(_ => + QueueCommand(_ => { var iJobId = int.Parse(jobId); var job = _.HangfireJobRepository.FirstOrDefault(x => x.Id == iJobId); @@ -326,16 +330,16 @@ public override void SetJobState(string jobId, IState state) _.Database.Commit(); } - catch (Exception) + catch (Exception ex) { _.Database.Rollback(); - - throw; + + throw ex; } } - }); + }); } - + /// /// /// @@ -349,12 +353,12 @@ public override void SetRangeInHash(string key, IEnumerable { var hash = new Hash @@ -376,7 +380,7 @@ public override void SetRangeInHash(string key, IEnumerable @@ -422,7 +426,7 @@ public override void ExpireHash(string key, TimeSpan expireIn) } }); } - + /// /// /// @@ -432,18 +436,18 @@ public override void ExpireHash(string key, TimeSpan expireIn) public override void ExpireSet(string key, TimeSpan expireIn) { if (key == null) throw new ArgumentNullException(nameof(key)); - + QueueCommand(x => { var states = x.SetRepository.Where(_ => _.Key == key).ToList(); - foreach(var state in states) + foreach (var state in states) { state.ExpireAt = DateTime.UtcNow.Add(expireIn); x.Database.Update(state); } }); - } - + } + /// /// /// @@ -455,12 +459,12 @@ public override void PersistSet(string key) QueueCommand(x => { var states = x.SetRepository.Where(_ => _.Key == key).ToList(); - foreach(var state in states) + foreach (var state in states) { state.ExpireAt = DateTime.MinValue; x.Database.Update(state); } - + }); } @@ -475,11 +479,11 @@ public override void PersistList(string key) QueueCommand(x => { var states = x.HangfireListRepository.Where(_ => _.Key == key).ToList(); - foreach(var state in states) + foreach (var state in states) { state.ExpireAt = DateTime.MinValue; x.Database.Update(state); - } + } }); } @@ -494,11 +498,11 @@ public override void PersistHash(string key) QueueCommand(x => { var states = x.HashRepository.Where(_ => _.Key == key).ToList(); - foreach(var state in states) + foreach (var state in states) { state.ExpireAt = DateTime.MinValue; x.Database.Update(state); - } + } }); } @@ -512,7 +516,7 @@ public override void AddRangeToSet(string key, IList items) { if (key == null) throw new ArgumentNullException(nameof(key)); if (items == null) throw new ArgumentNullException(nameof(items)); - + foreach (var item in items) { QueueCommand(x => @@ -549,7 +553,7 @@ public override void RemoveSet(string key) { if (key == null) throw new ArgumentNullException(nameof(key)); QueueCommand(x => x.SetRepository.Delete(_ => _.Key == key)); - } + } /// /// @@ -563,16 +567,16 @@ public override void TrimList(string key, int keepStartingFrom, int keepEndingAt { var start = keepStartingFrom + 1; var end = keepEndingAt + 1; - + var items = _.HangfireListRepository .Where(x => x.Key == key) .Reverse() - .Select((data, i) => new {Index = i + 1, Data = data.Id}) + .Select((data, i) => new { Index = i + 1, Data = data.Id }) .Where(x => !((x.Index >= start) && (x.Index <= end))) .Select(x => x.Data) .ToList(); - foreach(var id in items) + foreach (var id in items) { _.HangfireListRepository.Delete(x => x.Id == id); } diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs index 4bc9e9e..8e75a77 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs @@ -283,7 +283,7 @@ public void ProcessingJobs_ReturnsProcessingJobsOnly_WhenMultipleJobsExistsInPro private void UseMonitoringApi(Action action) { using var storage = ConnectionUtils.CreateStorage(); - var connection = new SQLiteMonitoringApi(storage, _providers); + var connection = new SQLiteMonitoringApi(storage.CreateAndOpenConnection(), _providers); using var dbContext = storage.CreateAndOpenConnection(); action(dbContext, connection); } From 1ee6a3f42f88c8fe08d7d9aa9b83adfa44b77421 Mon Sep 17 00:00:00 2001 From: Felix Clase Date: Mon, 5 Feb 2024 07:18:39 -0400 Subject: [PATCH 21/21] Memory leak fix.. --- .../HangfireSQLiteConnection.cs | 18 ++++++++---------- .../SQLiteDistributedLock.cs | 11 ++--------- .../SQLiteMonitoringApi.cs | 16 +++++++++------- .../Hangfire.Storage.SQLite/SQLiteStorage.cs | 2 +- .../SQLiteWriteOnlyTransaction.cs | 1 - .../SQLiteMonitoringApiFacts.cs | 2 +- 6 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs b/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs index a5a0034..0c23d29 100644 --- a/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs +++ b/src/main/Hangfire.Storage.SQLite/HangfireSQLiteConnection.cs @@ -38,6 +38,12 @@ public HangfireSQLiteConnection( _queueProviders = queueProviders ?? throw new ArgumentNullException(nameof(queueProviders)); } + public override void Dispose() + { + DbContext.Dispose(); + base.Dispose(); + } + public override IDisposable AcquireDistributedLock(string resource, TimeSpan timeout) { return Retry.Twice((_) => @@ -347,27 +353,19 @@ public override void Heartbeat(string serverId) throw new ArgumentNullException(nameof(serverId)); } - // DANIEL WAS HERE: - // Something fishy is going on here, a BackgroundServerGoneException is unexpectedly thrown - // https://github.com/HangfireIO/Hangfire/blob/master/src/Hangfire.Core/Server/ServerHeartbeatProcess.cs - // Changing to - - // DANIEL WAS HERE: - // var server = DbContext.HangfireServerRepository.FirstOrDefault(_ => _.Id == serverId); var server = Retry.Twice((attempts) => - // Forcing a query (read somewhere that sqlite-net handles FirstOrDefault differently ) DbContext.HangfireServerRepository.Where(_ => _.Id == serverId) .ToArray() .FirstOrDefault() ); + if (server == null) throw new BackgroundServerGoneException(); server.LastHeartbeat = DateTime.UtcNow; - // DANIEL WAS HERE: - // var affected = DbContext.Database.Update(server); var affected = Retry.Twice((_) => DbContext.Database.Update(server)); + if (affected == 0) throw new BackgroundServerGoneException(); } diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs index 2d1693b..ef4b534 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteDistributedLock.cs @@ -176,31 +176,24 @@ private void Acquire(TimeSpan timeout) /// private void Release() { - // DANIEL WAS HERE: Retry.Twice((retry) => { // Remove resource lock (if it's still ours) _dbContext.DistributedLockRepository.Delete(_ => _.Resource == _resource && _.ResourceKey == _resourceKey); lock (EventWaitHandleName) Monitor.Pulse(EventWaitHandleName); - } - - ); + }); } - private void Cleanup() { try { - // DANIEL WAS HERE: Retry.Twice((_) => { - // Delete expired locks (of any owner) _dbContext.DistributedLockRepository. Delete(x => x.Resource == _resource && x.ExpireAt < DateTime.UtcNow); - } - ); + }); } catch (Exception ex) { diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs b/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs index 5128117..72adbce 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteMonitoringApi.cs @@ -12,25 +12,28 @@ namespace Hangfire.Storage.SQLite { public class SQLiteMonitoringApi : IMonitoringApi { - private readonly HangfireDbContext _dbContext; + private readonly SQLiteStorage _storage; private readonly PersistentJobQueueProviderCollection _queueProviders; /// /// /// - /// + /// /// - public SQLiteMonitoringApi(HangfireDbContext database, PersistentJobQueueProviderCollection queueProviders) + public SQLiteMonitoringApi(SQLiteStorage storage, PersistentJobQueueProviderCollection queueProviders) { - _dbContext = database; + _storage = storage; _queueProviders = queueProviders; } private T UseConnection(Func action) { - var result = action(_dbContext); - return result; + using (var dbContext = _storage.CreateAndOpenConnection()) + { + var result = action(dbContext); + return result; + } } private JobList GetJobs(HangfireDbContext connection, int from, int count, string stateName, Func, TDto> selector) @@ -414,7 +417,6 @@ public JobList FetchedJobs(string queue, int from, int perPage) /// public StatisticsDto GetStatistics() { - // DANIEL WAS HERE: return Retry.Twice((attempts) => UseConnection(ctx => diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs index bde08c6..7fe98ed 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteStorage.cs @@ -75,7 +75,7 @@ public override IStorageConnection GetConnection() public override IMonitoringApi GetMonitoringApi() { CheckDisposed(); - return new SQLiteMonitoringApi(CreateAndOpenConnection(), QueueProviders); + return new SQLiteMonitoringApi(this, QueueProviders); } /// diff --git a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs index 0e796c7..65d3e19 100644 --- a/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs +++ b/src/main/Hangfire.Storage.SQLite/SQLiteWriteOnlyTransaction.cs @@ -113,7 +113,6 @@ public override void AddToSet(string key, string value, double score) public override void Commit() { - // DANIEL WAS HERE Retry.Twice((attempts) => { lock (_lockObject) diff --git a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs index 8e75a77..4bc9e9e 100644 --- a/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs +++ b/src/test/Hangfire.Storage.SQLite.Test/SQLiteMonitoringApiFacts.cs @@ -283,7 +283,7 @@ public void ProcessingJobs_ReturnsProcessingJobsOnly_WhenMultipleJobsExistsInPro private void UseMonitoringApi(Action action) { using var storage = ConnectionUtils.CreateStorage(); - var connection = new SQLiteMonitoringApi(storage.CreateAndOpenConnection(), _providers); + var connection = new SQLiteMonitoringApi(storage, _providers); using var dbContext = storage.CreateAndOpenConnection(); action(dbContext, connection); }