diff --git a/.ci/appsettings.override.postgres.docker.json b/.ci/appsettings.override.postgres.docker.json index 749cedff01..babee712d6 100644 --- a/.ci/appsettings.override.postgres.docker.json +++ b/.ci/appsettings.override.postgres.docker.json @@ -68,8 +68,11 @@ "ConnectionString": "User ID=files;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;" }, "BlobStorage": { - "CloudProvider": "Azure", - "ConnectionInfo": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" + "ProductName": "AzureStorageAccount", + "AzureStorageAccount": { + "ContainerName": "files", + "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" + } } } }, diff --git a/.ci/appsettings.override.postgres.local.json b/.ci/appsettings.override.postgres.local.json index 7e4cb7fc92..4ec30a71cf 100644 --- a/.ci/appsettings.override.postgres.local.json +++ b/.ci/appsettings.override.postgres.local.json @@ -68,8 +68,11 @@ "ConnectionString": "User ID=files;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" }, "BlobStorage": { - "CloudProvider": "Azure", - "ConnectionInfo": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + "ProductName": "AzureStorageAccount", + "AzureStorageAccount": { + "ContainerName": "files", + "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + } } } }, diff --git a/.ci/appsettings.override.sqlserver.docker.json b/.ci/appsettings.override.sqlserver.docker.json index 050034be42..3d5bb6976f 100644 --- a/.ci/appsettings.override.sqlserver.docker.json +++ b/.ci/appsettings.override.sqlserver.docker.json @@ -68,8 +68,11 @@ "ConnectionString": "Server=sqlserver;Database=enmeshed;User Id=files;Password=Passw0rd;TrustServerCertificate=True" }, "BlobStorage": { - "CloudProvider": "Azure", - "ConnectionInfo": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" + "ProductName": "AzureStorageAccount", + "AzureStorageAccount": { + "ContainerName": "files", + "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" + } } } }, diff --git a/.ci/appsettings.override.sqlserver.local.json b/.ci/appsettings.override.sqlserver.local.json index eb601335ed..8a3cb3c8ca 100644 --- a/.ci/appsettings.override.sqlserver.local.json +++ b/.ci/appsettings.override.sqlserver.local.json @@ -68,8 +68,11 @@ "ConnectionString": "Server=localhost;Database=enmeshed;User Id=files;Password=Passw0rd;TrustServerCertificate=True" }, "BlobStorage": { - "CloudProvider": "Azure", - "ConnectionInfo": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + "ProductName": "AzureStorageAccount", + "AzureStorageAccount": { + "ContainerName": "files", + "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + } } } }, diff --git a/Applications/AdminApi/src/AdminApi/AdminApi.csproj b/Applications/AdminApi/src/AdminApi/AdminApi.csproj index d96ef9d937..bc060ae589 100644 --- a/Applications/AdminApi/src/AdminApi/AdminApi.csproj +++ b/Applications/AdminApi/src/AdminApi/AdminApi.csproj @@ -7,7 +7,7 @@ - + diff --git a/Applications/AdminApi/src/AdminApi/Dockerfile b/Applications/AdminApi/src/AdminApi/Dockerfile index 69f03916f4..2bd8acf899 100644 --- a/Applications/AdminApi/src/AdminApi/Dockerfile +++ b/Applications/AdminApi/src/AdminApi/Dockerfile @@ -98,7 +98,7 @@ RUN dotnet publish /p:ContinuousIntegrationBuild=true /p:UseAppHost=false --no-r RUN dotnet publish /p:ContinuousIntegrationBuild=true --configuration Release --output /app/publish/health "/src/Applications/HealthCheck/src/HealthCheck.csproj" #### Build Flutter Admin UI #### -FROM ghcr.io/cirruslabs/flutter:3.27.0 AS flutter-build-env +FROM ghcr.io/cirruslabs/flutter:3.27.1 AS flutter-build-env COPY Applications/AdminUi /src WORKDIR /src diff --git a/Applications/ConsumerApi/test/ConsumerApi.Tests.Integration/api.appsettings.local.override.json b/Applications/ConsumerApi/test/ConsumerApi.Tests.Integration/api.appsettings.local.override.json index e0b310764d..4cac42fafb 100644 --- a/Applications/ConsumerApi/test/ConsumerApi.Tests.Integration/api.appsettings.local.override.json +++ b/Applications/ConsumerApi/test/ConsumerApi.Tests.Integration/api.appsettings.local.override.json @@ -40,8 +40,11 @@ "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" }, "BlobStorage": { - "Provider": "Azure", - "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + "ProductName": "AzureStorageAccount", + "AzureStorageAccount": { + "ContainerName": "files", + "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + } } } }, diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index 37d1d0d7fa..b05b141823 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -1,7 +1,8 @@  - + + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs index da3acfb0d9..d8c8cab34b 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs @@ -1,23 +1,20 @@ +using System.ComponentModel.DataAnnotations; using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; public static class AzureStorageAccountServiceCollectionExtensions { - public static void AddAzureStorageAccount(this IServiceCollection services, - Action setupOptions) - { - var options = new AzureStorageAccountOptions(); - setupOptions.Invoke(options); - - services.AddAzureStorageAccount(options); - } - - public static void AddAzureStorageAccount(this IServiceCollection services, AzureStorageAccountOptions options) { - services.Configure(opt => opt.ConnectionString = options.ConnectionString); + services.AddSingleton>(new OptionsWrapper(options)); + services.Configure(opt => + { + opt.ConnectionString = options.ConnectionString; + opt.ContainerName = options.ContainerName; + }); services.AddSingleton(); services.AddScoped(); } @@ -25,5 +22,9 @@ public static void AddAzureStorageAccount(this IServiceCollection services, Azur public class AzureStorageAccountOptions { - public string ConnectionString { get; set; } = string.Empty; + public required string ConnectionString { get; set; } + + [Required] + [MinLength(2)] + public required string ContainerName { get; set; } } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs index b6cf815560..df347f7d2b 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs @@ -14,12 +14,12 @@ public class BlobStorageHealthCheck : IHealthCheck private static int _numberOfTries; private readonly IBlobStorage _storage; - private readonly string _bucketName; + private readonly string _rootFolderName; - public BlobStorageHealthCheck(IBlobStorage storage, string bucketName) + public BlobStorageHealthCheck(IBlobStorage storage, string rootFolderName) { _storage = storage; - _bucketName = bucketName; + _rootFolderName = rootFolderName; } public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) @@ -42,7 +42,7 @@ private async Task IsUploadPossible(string filename) { try { - _storage.Add(_bucketName, filename, FILE_TEXT.GetBytes()); + _storage.Add(_rootFolderName, filename, FILE_TEXT.GetBytes()); await _storage.SaveAsync(); return true; } @@ -56,7 +56,7 @@ private async Task IsDownloadPossible(string filename) { try { - var downloadBytes = await _storage.FindAsync(_bucketName, filename); + var downloadBytes = await _storage.FindAsync(_rootFolderName, filename); var downloadedString = Encoding.UTF8.GetString(downloadBytes); return downloadedString == FILE_TEXT; } @@ -70,7 +70,7 @@ private async Task IsDeletionPossible(string filename) { try { - _storage.Remove(_bucketName, filename); + _storage.Remove(_rootFolderName, filename); await _storage.SaveAsync(); return true; } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index f6c9c85826..2da410b940 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -1,6 +1,8 @@ +using System.ComponentModel.DataAnnotations; using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; +using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; using Backbone.Tooling.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -9,9 +11,6 @@ namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage; public static class BlobStorageServiceCollectionExtensions { - public const string AZURE_CLOUD_PROVIDER = "Azure"; - public const string GOOGLE_CLOUD_PROVIDER = "GoogleCloud"; - public static void AddBlobStorage(this IServiceCollection services, Action setupOptions) { var options = new BlobStorageOptions(); @@ -22,43 +21,67 @@ public static void AddBlobStorage(this IServiceCollection services, Action { azureStorageAccountOptions.ConnectionString = options.ConnectionInfo!; }); + case BlobStorageOptions.AZURE_STORAGE_ACCOUNT: + services.AddAzureStorageAccount(options.AzureStorageAccount!); + break; + case BlobStorageOptions.GOOGLE_CLOUD_STORAGE: + services.AddGoogleCloudStorage(options.GoogleCloudStorage!); break; - case GOOGLE_CLOUD_PROVIDER: - services.AddGoogleCloudStorage(googleCloudStorageOptions => - { - googleCloudStorageOptions.GcpAuthJson = options.ConnectionInfo; - googleCloudStorageOptions.BucketName = options.Container; - }); + case BlobStorageOptions.S3_BUCKET: + services.AddS3(options.S3Bucket!); break; + default: { - if (options.CloudProvider.IsNullOrEmpty()) + if (options.ProductName.IsNullOrEmpty()) throw new NotSupportedException("No cloud provider was specified."); throw new NotSupportedException( - $"{options.CloudProvider} is not a currently supported cloud provider."); + $"{options.ProductName} is not a currently supported cloud provider."); } } services.AddHealthChecks().Add( new HealthCheckRegistration( "blob_storage", - sp => new BlobStorageHealthCheck(sp.GetRequiredService(), options.Container), + sp => new BlobStorageHealthCheck(sp.GetRequiredService(), options.RootFolder), HealthStatus.Unhealthy, null ) ); } } -public class BlobStorageOptions +public class BlobStorageOptions : IValidatableObject { - public string CloudProvider { get; set; } = null!; + public const string AZURE_STORAGE_ACCOUNT = "AzureStorageAccount"; + public const string GOOGLE_CLOUD_STORAGE = "GoogleCloudStorage"; + public const string S3_BUCKET = "S3Bucket"; + + [RegularExpression($"{AZURE_STORAGE_ACCOUNT}|{GOOGLE_CLOUD_STORAGE}|{S3_BUCKET}")] + public string ProductName { get; set; } = null!; + + public AzureStorageAccountOptions? AzureStorageAccount { get; set; } + + public GoogleCloudStorageOptions? GoogleCloudStorage { get; set; } - public string Container { get; set; } = null!; + public S3BucketOptions? S3Bucket { get; set; } - public string? ConnectionInfo { get; set; } + public string RootFolder => ProductName switch + { + AZURE_STORAGE_ACCOUNT => AzureStorageAccount!.ContainerName, + GOOGLE_CLOUD_STORAGE => GoogleCloudStorage!.BucketName, + S3_BUCKET => S3Bucket!.BucketName, + _ => throw new Exception("Unsupported ProductName") + }; + + public IEnumerable Validate(ValidationContext validationContext) + { + if (ProductName == AZURE_STORAGE_ACCOUNT && AzureStorageAccount == null) + yield return new ValidationResult($"The property '{nameof(AzureStorageAccount)}' must be set when the {nameof(ProductName)} is '{AZURE_STORAGE_ACCOUNT}'.", [nameof(AzureStorageAccount)]); + + if (ProductName == GOOGLE_CLOUD_STORAGE && GoogleCloudStorage == null) + yield return new ValidationResult($"The property '{nameof(GoogleCloudStorage)}' must be set when the {nameof(ProductName)} is '{GOOGLE_CLOUD_STORAGE}'.", [nameof(GoogleCloudStorage)]); + } } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs index cb4f12835b..b6509051f9 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs @@ -1,57 +1,33 @@ +using System.ComponentModel.DataAnnotations; using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.Tooling.Extensions; using Google.Apis.Auth.OAuth2; using Google.Cloud.Storage.V1; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; public static class GoogleCloudStorageServiceCollectionExtensions { - public static void AddGoogleCloudStorage(this IServiceCollection services, - Action setupOptions) - { - services.Configure(setupOptions); - - var options = new GoogleCloudStorageOptions(); - setupOptions.Invoke(options); - - services.AddGoogleCloudStorage(options); - } - public static void AddGoogleCloudStorage(this IServiceCollection services, GoogleCloudStorageOptions options) { services.AddSingleton(_ => { - var storageClient = options.GcpAuthJson.IsNullOrEmpty() + var storageClient = options.ServiceAccountJson.IsNullOrEmpty() ? StorageClient.Create() - : StorageClient.Create(GoogleCredential.FromJson(options.GcpAuthJson)); + : StorageClient.Create(GoogleCredential.FromJson(options.ServiceAccountJson)); return storageClient; }); - services.AddScoped(sp => - { - var storageClient = sp.GetService(); - var logger = sp.GetService>(); - - if (storageClient == null) - { - throw new Exception("A StorageClient was not registered in the dependency container."); - } - - if (logger == null) - { - throw new Exception("A Logger was not registered in the dependency container."); - } - - return new GoogleCloudStorage(storageClient, logger); - }); + services.AddScoped(); } } public class GoogleCloudStorageOptions { - public string? GcpAuthJson { get; set; } - public string BucketName { get; set; } = string.Empty; + public required string? ServiceAccountJson { get; set; } + + [Required] + [MinLength(2)] + public required string BucketName { get; set; } } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs new file mode 100644 index 0000000000..b4e4a74281 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs @@ -0,0 +1,217 @@ +using System.Net; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Backbone.BuildingBlocks.Application.Abstractions.Exceptions; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public class S3BlobStorage : IBlobStorage, IDisposable +{ + private readonly AmazonS3Client _s3Client; + private readonly List _changedBlobs; + private readonly IList _removedBlobs; + private readonly string _bucketName; + private readonly ILogger _logger; + + public S3BlobStorage(IOptions config, ILogger logger) + { + var s3Config = new AmazonS3Config + { + ServiceURL = config.Value.ServiceUrl, + ForcePathStyle = true + }; + + _s3Client = new AmazonS3Client(config.Value.AccessKeyId, config.Value.SecretAccessKey, s3Config); + _changedBlobs = []; + _removedBlobs = []; + _bucketName = config.Value.BucketName; + _logger = logger; + } + + public void Add(string folder, string id, byte[] content) + { + _changedBlobs.Add(new ChangedBlob(folder, id, content)); + } + + public void Remove(string folder, string id) + { + _removedBlobs.Add(new RemovedBlob(folder, id)); + } + + public void Dispose() + { + _changedBlobs.Clear(); + _removedBlobs.Clear(); + } + + public async Task FindAsync(string folder, string id) + { + _logger.LogTrace("Reading blob with key '{blobId}'...", id); + + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{id}" + }; + + using var response = await _s3Client.GetObjectAsync(request); + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + + _logger.LogTrace("Found blob with key '{blobId}'.", id); + return memoryStream.ToArray(); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", id); + throw new NotFoundException("Blob", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error downloading blob with key '{blobId}'.", id); + throw; + } + } + + public Task> FindAllAsync(string folder, string? prefix = null) + { + return Task.FromResult(FindAllBlobsAsync(folder, prefix)); + } + + private async IAsyncEnumerable FindAllBlobsAsync(string folder, string? prefix) + { + _logger.LogTrace("Listing all blobs..."); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = prefix != null ? $"{folder}/{prefix}" : folder + }; + + ListObjectsV2Response response; + do + { + response = await _s3Client.ListObjectsV2Async(request); + + foreach (var obj in response.S3Objects) + { + yield return obj.Key; + } + + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + + _logger.LogTrace("Found all blobs."); + } + + public async Task SaveAsync() + { + await UploadChangedBlobs(); + await DeleteRemovedBlobs(); + } + + private async Task UploadChangedBlobs() + { + _logger.LogTrace("Uploading '{changedBlobsCount}' changed blobs...", _changedBlobs.Count); + + var changedBlobs = new List(_changedBlobs); + + foreach (var blob in changedBlobs) + { + await EnsureKeyDoesNotExist(blob.Folder, blob.Name); + + using var memoryStream = new MemoryStream(blob.Content); + + try + { + _logger.LogTrace("Uploading blob with key '{blobName}'...", blob.Name); + + var request = new TransferUtilityUploadRequest + { + InputStream = memoryStream, + Key = $"{blob.Folder}/{blob.Name}", + BucketName = _bucketName + }; + + var transferUtility = new TransferUtility(_s3Client); + await transferUtility.UploadAsync(request); + + _logger.LogTrace("Upload of blob with key '{blobName}' was successful.", blob.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error uploading blob with key '{blobName}'.", blob.Name); + throw; + } + finally + { + _changedBlobs.Remove(blob); + } + } + } + + private async Task EnsureKeyDoesNotExist(string folder, string key) + { + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{key}" + }; + + await _s3Client.GetObjectAsync(request); + + _logger.LogError("A blob with key '{blobName}' already exists.", key); + throw new BlobAlreadyExistsException(key); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + } + } + + private async Task DeleteRemovedBlobs() + { + _logger.LogTrace("Deleting '{removedBlobsCount}' blobs...", _removedBlobs.Count); + + var blobsToDelete = new List(_removedBlobs); + + foreach (var blob in blobsToDelete) + { + try + { + var request = new DeleteObjectRequest + { + BucketName = _bucketName, + Key = $"{blob.Folder}/{blob.Name}" + }; + + await _s3Client.DeleteObjectAsync(request); + + _removedBlobs.Remove(blob); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", blob.Name); + throw new NotFoundException($"Blob with key '{blob.Name}' was not found.", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting blob with key '{blobName}'.", blob.Name); + throw; + } + } + + _logger.LogTrace("Deletion successful."); + } + + private record ChangedBlob(string Folder, string Name, byte[] Content); + + private record RemovedBlob(string Folder, string Name); +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs new file mode 100644 index 0000000000..ea537db405 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs @@ -0,0 +1,45 @@ +using System.ComponentModel.DataAnnotations; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.DependencyInjection; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public static class S3ServiceCollectionExtensions +{ + public static void AddS3(this IServiceCollection services, + Action setupOptions) + { + var options = new S3BucketOptions(); + setupOptions.Invoke(options); + + services.AddS3(options); + } + + public static void AddS3(this IServiceCollection services, S3BucketOptions options) + { + services.Configure(s3Options => + { + s3Options.BucketName = options.BucketName; + s3Options.AccessKeyId = options.AccessKeyId; + s3Options.SecretAccessKey = options.SecretAccessKey; + s3Options.ServiceUrl = options.ServiceUrl; + }); + + services.AddScoped(); + } +} + +public class S3BucketOptions +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKeyId { get; set; } = string.Empty; + + [Required] + public string SecretAccessKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; +} diff --git a/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/BuildingBlocks.Infrastructure.Tests.csproj b/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/BuildingBlocks.Infrastructure.Tests.csproj index 298c6b40ce..259d9af36d 100644 --- a/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/BuildingBlocks.Infrastructure.Tests.csproj +++ b/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/BuildingBlocks.Infrastructure.Tests.csproj @@ -5,7 +5,7 @@ - + diff --git a/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/Tests/AzureStorageAccountTests.cs b/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/Tests/AzureStorageAccountTests.cs index 65e082659c..9584e02407 100644 --- a/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/Tests/AzureStorageAccountTests.cs +++ b/BuildingBlocks/test/BuildingBlocks.Infrastructure.Tests/Tests/AzureStorageAccountTests.cs @@ -63,7 +63,11 @@ private static IBlobStorage ProvisionAzureStorageTests() var services = new ServiceCollection() .AddLogging(); - services.AddAzureStorageAccount(x => { x.ConnectionString = "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://127.0.0.1;"; }); + services.AddAzureStorageAccount(new AzureStorageAccountOptions + { + ConnectionString = "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://127.0.0.1;", + ContainerName = "test" + }); var serviceProvider = services.BuildServiceProvider(); return serviceProvider.GetRequiredService(); diff --git a/Infrastructure/Infrastructure.csproj b/Infrastructure/Infrastructure.csproj index 25f6676291..aca44cc403 100644 --- a/Infrastructure/Infrastructure.csproj +++ b/Infrastructure/Infrastructure.csproj @@ -1,7 +1,7 @@ - + diff --git a/Modules/Devices/src/Devices.Application/Devices.Application.csproj b/Modules/Devices/src/Devices.Application/Devices.Application.csproj index 73c9988014..b181bfc046 100644 --- a/Modules/Devices/src/Devices.Application/Devices.Application.csproj +++ b/Modules/Devices/src/Devices.Application/Devices.Application.csproj @@ -3,8 +3,8 @@ - - + + diff --git a/Modules/Devices/src/Devices.Infrastructure/Devices.Infrastructure.csproj b/Modules/Devices/src/Devices.Infrastructure/Devices.Infrastructure.csproj index c861b3a5fc..c6e929c4aa 100644 --- a/Modules/Devices/src/Devices.Infrastructure/Devices.Infrastructure.csproj +++ b/Modules/Devices/src/Devices.Infrastructure/Devices.Infrastructure.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index 5440e727b5..2888dc11de 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -1,4 +1,5 @@ using System.ComponentModel.DataAnnotations; +using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage; using Backbone.Modules.Files.Application; namespace Backbone.Modules.Files.ConsumerApi; @@ -17,19 +18,7 @@ public class InfrastructureConfiguration public SqlDatabaseConfiguration SqlDatabase { get; set; } = new(); [Required] - public BlobStorageConfiguration BlobStorage { get; set; } = new(); - - public class BlobStorageConfiguration - { - [Required] - [MinLength(1)] - [RegularExpression("Azure|GoogleCloud")] - public string CloudProvider { get; set; } = string.Empty; - - public string ConnectionInfo { get; set; } = string.Empty; - - public string ContainerName { get; set; } = string.Empty; - } + public BlobStorageOptions BlobStorage { get; set; } = new(); public class SqlDatabaseConfiguration { diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index 87b4943bfa..9b3e0375d6 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -4,7 +4,6 @@ using Backbone.Modules.Files.Application; using Backbone.Modules.Files.Application.Extensions; using Backbone.Modules.Files.Infrastructure.Persistence; -using Backbone.Tooling.Extensions; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -29,12 +28,7 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati options.DbOptions.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; options.DbOptions.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; - options.BlobStorageOptions.ConnectionInfo = parsedConfiguration.Infrastructure.BlobStorage.ConnectionInfo; - options.BlobStorageOptions.CloudProvider = parsedConfiguration.Infrastructure.BlobStorage.CloudProvider; - options.BlobStorageOptions.Container = - parsedConfiguration.Infrastructure.BlobStorage.ContainerName.IsNullOrEmpty() - ? "files" - : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; + options.BlobStorageOptions = parsedConfiguration.Infrastructure.BlobStorage; }); if (parsedConfiguration.Infrastructure.SqlDatabase.EnableHealthCheck) diff --git a/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs b/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs index 00ca0777df..3fa3bfdea5 100644 --- a/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs +++ b/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection; namespace Backbone.Modules.Files.Infrastructure.Persistence; + public static class IServiceCollectionExtensions { public static void AddPersistence(this IServiceCollection services, Action setupOptions) @@ -19,10 +20,9 @@ public static void AddPersistence(this IServiceCollection services, Action(blobOptions => - blobOptions.RootFolder = options.BlobStorageOptions.Container); services.AddBlobStorage(options.BlobStorageOptions); + services.Configure(blobOptions => blobOptions.RootFolder = options.BlobStorageOptions.RootFolder); services.AddTransient(); } } diff --git a/appsettings.override.json b/appsettings.override.json index 7d623edf05..5460f32b20 100644 --- a/appsettings.override.json +++ b/appsettings.override.json @@ -88,9 +88,11 @@ // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver }, "BlobStorage": { - "CloudProvider": "Azure", - "ConnectionInfo": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;", - "ContainerName": "" + "ProductName": "AzureStorageAccount", + "AzureStorageAccount": { + "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;", + "ContainerName": "files" + } } } }, diff --git a/docker-compose/.env b/docker-compose/.env index b0892408a2..5d474a1777 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -10,7 +10,4 @@ ES_KIBANA_PORT=5601 # Port to expose Logstash HTTP API to the host ES_LOGSTASH_PORT=31311 -# Azurite configuration -ENMESHED_BLOB_STORAGE_CONNECTION_STRING = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" - BACKBONE_VERSION=5.9.1 diff --git a/docker-compose/appsettings.override.json b/docker-compose/appsettings.override.json index 2cc6146066..35ab93fc9f 100644 --- a/docker-compose/appsettings.override.json +++ b/docker-compose/appsettings.override.json @@ -73,9 +73,11 @@ // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=files;Password=Passw0rd;TrustServerCertificate=True" // sqlserver }, "BlobStorage": { - "CloudProvider": "Azure", - "ConnectionInfo": "", - "ContainerName": "" + "ProductName": "AzureStorageAccount", + "AzureStorageAccount": { + "ConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;", + "ContainerName": "files" + } } } }, diff --git a/docker-compose/docker-compose.services.yml b/docker-compose/docker-compose.services.yml index a7ece88ae9..cae610cd87 100644 --- a/docker-compose/docker-compose.services.yml +++ b/docker-compose/docker-compose.services.yml @@ -4,7 +4,7 @@ services: container_name: consumerApi hostname: consumerApi environment: - - Modules__Files__Infrastructure__BlobStorage__ConnectionInfo="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" + - Modules__Files__Infrastructure__BlobStorage__ConnectionString="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;" ports: - "8081:8080" diff --git a/helm/values.yaml b/helm/values.yaml index 01b06f93d7..38dbd96684 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -17,11 +17,11 @@ admincli: # resources - the resources for the Admin CLI container (see https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) resources: requests: - cpu: "200m" + cpu: "50m" memory: "128Mi" limits: - cpu: "400m" - memory: "1Gi" + cpu: "50m" + memory: "128Mi" # securityContext - securityContext for the Consumer API container (see https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-1) securityContext: {} @@ -666,12 +666,30 @@ global: # connectionString - should be set via environment variable connectionString: "" blobStorage: - # cloudProvider - possible values: "GoogleCloud"/"Azure" - # cloudProvider: "" - # connectionInfo - should be set via environment variable - # connectionInfo: "" - # containerName - the name of the Bucket if CloudProvider=GoogleCloud; the name of the Container if CloudProvider=Azure - # containerName: "" + # productName - possible values: "AzureStorageAccount"/"GoogleCloudStorage" + productName: "" + # azureStorageAccount - only applicable if ProductName is "AzureStorageAccount" + # azureStorageAccount: + # containerName - the name of the container that should be used to store the files + # containerName: "" + # connectionString - should be set via environment variable + # connectionString: "" + # googleCloudStorage - only applicable if ProductName is "GoogleCloudStorage" + # googleCloudStorage: + # serviceAccountJson - the content of the service account json file that should be used to authenticate + # serviceAccountJson: "" + # bucketName - the name of the bucket that should be used to store the files + # bucketName: "" + # s3Bucket - only applicable if ProductName is "S3Bucket" + # s3Bucket: + # accessKeyId - the access key id that should be used to authenticate + # accessKeyId: "" + # secretAccessKey - the secret access key that should be used to authenticate + # secretAccessKey: "" + # bucketName - the name of the bucket that should be used to store the files + # bucketName: "" + # serviceUrl - the url of the S3 service + # serviceUrl: "" messages: application: # didDomainName - the didDomainName that should be used when generating Identity Addresses