[YOMA-611] Opportunity Completions Import (#1178)

* Code completed; Pending testing and bug fixing

* Testing and bug fixing

src/api/Yoma.Core.sln

@@ -24,6 +24,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		src\scripts\index_fragmentation_postgre.sql = src\scripts\index_fragmentation_postgre.sql
 		src\scripts\index_rebuild_reorganize_ms.sql = src\scripts\index_rebuild_reorganize_ms.sql
 		src\scripts\index_rebuild_reorganize_postgre.sql = src\scripts\index_rebuild_reorganize_postgre.sql
+		src\other\MyOpportunityInfoCsvImport_Sample.csv = src\other\MyOpportunityInfoCsvImport_Sample.csv
 		src\other\OpportunityInfoCsvImport_Sample.csv = src\other\OpportunityInfoCsvImport_Sample.csv
 	EndProjectSection
 EndProject

src/api/src/application/Yoma.Core.Api/Controllers/MyOpportunityController.cs

@@ -126,6 +126,21 @@ public async Task<IActionResult> FinalizeVerificationManual([FromBody] MyOpportu
 
       return StatusCode((int)HttpStatusCode.OK);
     }
+
+    [SwaggerOperation(Summary = "Import completions for the specified organization from a CSV file (Admin or Organization Admin roles required)")]
+    [HttpPost("action/verify/csv")]
+    [ProducesResponseType((int)HttpStatusCode.OK)]
+    [Authorize(Roles = $"{Constants.Role_Admin}, {Constants.Role_OrganizationAdmin}")]
+    public async Task<IActionResult> PerformActionImportVerificationFromCSV([FromForm] MyOpportunityRequestVerifyImportCsv request)
+    {
+      _logger.LogInformation("Handling request {requestName}", nameof(PerformActionImportVerificationFromCSV));
+
+      await _myOpportunityService.PerformActionImportVerificationFromCSV(request, true);
+
+      _logger.LogInformation("Request {requestName} handled", nameof(PerformActionImportVerificationFromCSV));
+
+      return StatusCode((int)HttpStatusCode.OK);
+    }
     #endregion Administrative Actions
 
     #region Authenticated User Based Actions
@@ -309,6 +324,6 @@ public async Task<IActionResult> PerformActionSendForVerificationManualDelete([F
       return StatusCode((int)HttpStatusCode.OK);
     }
     #endregion Authenticated User Based Actions
-    #endregion
+    #endregion Public Members
   }
 }

src/api/src/domain/Yoma.Core.Domain/ActionLink/Services/LinkService.cs

@@ -24,7 +24,6 @@
 using Yoma.Core.Domain.Entity.Models;
 using Yoma.Core.Domain.IdentityProvider.Extensions;
 using Yoma.Core.Domain.IdentityProvider.Interfaces;
-using Yoma.Core.Domain.Opportunity;
 using Yoma.Core.Domain.Opportunity.Extensions;
 using Yoma.Core.Domain.Opportunity.Interfaces;
 using Yoma.Core.Domain.ShortLinkProvider.Interfaces;

src/api/src/domain/Yoma.Core.Domain/Entity/Services/UserService.cs

@@ -290,8 +290,6 @@ public async Task<User> Upsert(UserRequest request)
       // profile fields updatable via UserProfileService.Update; identity provider is source of truth
       if (isNew)
       {
-        var kcUser = await _identityProviderClient.GetUserByUsername(request.Username)
-            ?? throw new InvalidOperationException($"{nameof(User)} with username '{request.Username}' does not exist");
         result.FirstName = request.FirstName;
         result.Surname = request.Surname;
         result.DisplayName = request.DisplayName;

src/api/src/domain/Yoma.Core.Domain/MyOpportunity/Interfaces/IMyOpportunityService.cs

@@ -51,5 +51,7 @@ public interface IMyOpportunityService
     Dictionary<Guid, int>? ListAggregatedOpportunityByCompleted(bool includeExpired);
 
     Task PerformActionInstantVerification(Guid linkId);
+
+    Task PerformActionImportVerificationFromCSV(MyOpportunityRequestVerifyImportCsv request, bool ensureOrganizationAuthorization);
   }
 }

src/api/src/domain/Yoma.Core.Domain/MyOpportunity/Models/MyOpportunityInfoCsvImport.cs

@@ -0,0 +1,22 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace Yoma.Core.Domain.MyOpportunity.Models
+{
+  public class MyOpportunityInfoCsvImport
+  {
+    public string? Email { get; set; }
+
+    public string? PhoneNumber { get; set; }
+
+    public string? FirstName { get; set; }
+
+    public string? Surname { get; set; }
+
+    public string? Gender { get; set; }
+
+    public string? Country { get; set; }
+
+    [Required]
+    public string OpporunityExternalId { get; set; }
+  }
+}

src/api/src/domain/Yoma.Core.Domain/MyOpportunity/Models/MyOpportunityRequestVerify.cs

@@ -22,6 +22,6 @@ public class MyOpportunityRequestVerify
     internal bool OverridePending { get; set; }
 
     [JsonIgnore]
-    internal bool InstantVerification { get; set; }
+    internal bool InstantOrImportedVerification { get; set; }
   }
 }

src/api/src/domain/Yoma.Core.Domain/MyOpportunity/Models/MyOpportunityRequestVerifyImportCsv.cs

@@ -0,0 +1,13 @@
+using Microsoft.AspNetCore.Http;
+
+namespace Yoma.Core.Domain.MyOpportunity.Models
+{
+  public class MyOpportunityRequestVerifyImportCsv
+  {
+    public IFormFile File { get; set; }
+
+    public Guid OrganizationId { get; set; }
+
+    public string? Comment { get; set; }
+  }
+}

src/api/src/domain/Yoma.Core.Domain/MyOpportunity/Services/MyOpportunityService.cs

@@ -32,6 +32,10 @@
 using Yoma.Core.Domain.Opportunity.Interfaces.Lookups;
 using Yoma.Core.Domain.Reward.Interfaces;
 using Yoma.Core.Domain.SSI.Interfaces;
+using CsvHelper.Configuration.Attributes;
+using System.Globalization;
+using System.Reflection;
+using Yoma.Core.Domain.Lookups.Interfaces;
 
 namespace Yoma.Core.Domain.MyOpportunity.Services
 {
@@ -54,10 +58,13 @@ public class MyOpportunityService : IMyOpportunityService
     private readonly ILinkService _linkService;
     private readonly INotificationURLFactory _notificationURLFactory;
     private readonly INotificationDeliveryService _notificationDeliveryService;
+    private readonly ICountryService _countryService;
+    private readonly IGenderService _genderService;
     private readonly MyOpportunitySearchFilterValidator _myOpportunitySearchFilterValidator;
     private readonly MyOpportunityRequestValidatorVerify _myOpportunityRequestValidatorVerify;
     private readonly MyOpportunityRequestValidatorVerifyFinalize _myOpportunityRequestValidatorVerifyFinalize;
     private readonly MyOpportunityRequestValidatorVerifyFinalizeBatch _myOpportunityRequestValidatorVerifyFinalizeBatch;
+    private readonly MyOpportunityRequestValidatorVerifyImportCsv _myOpportunityRequestValidatorVerifyImportCsv;
     private readonly IRepositoryBatchedWithNavigation<Models.MyOpportunity> _myOpportunityRepository;
     private readonly IRepository<MyOpportunityVerification> _myOpportunityVerificationRepository;
     private readonly IExecutionStrategyService _executionStrategyService;
@@ -85,10 +92,13 @@ public MyOpportunityService(ILogger<MyOpportunityService> logger,
         ILinkService linkService,
         INotificationURLFactory notificationURLFactory,
         INotificationDeliveryService notificationDeliveryService,
+        ICountryService countryService,
+        IGenderService genderService,
         MyOpportunitySearchFilterValidator myOpportunitySearchFilterValidator,
         MyOpportunityRequestValidatorVerify myOpportunityRequestValidatorVerify,
         MyOpportunityRequestValidatorVerifyFinalize myOpportunityRequestValidatorVerifyFinalize,
         MyOpportunityRequestValidatorVerifyFinalizeBatch myOpportunityRequestValidatorVerifyFinalizeBatch,
+        MyOpportunityRequestValidatorVerifyImportCsv myOpportunityRequestValidatorVerifyImportCsv,
         IRepositoryBatchedWithNavigation<Models.MyOpportunity> myOpportunityRepository,
         IRepository<MyOpportunityVerification> myOpportunityVerificationRepository,
         IExecutionStrategyService executionStrategyService)
@@ -109,10 +119,13 @@ public MyOpportunityService(ILogger<MyOpportunityService> logger,
       _linkService = linkService;
       _notificationURLFactory = notificationURLFactory;
       _notificationDeliveryService = notificationDeliveryService;
+      _countryService = countryService;
+      _genderService = genderService;
       _myOpportunitySearchFilterValidator = myOpportunitySearchFilterValidator;
       _myOpportunityRequestValidatorVerify = myOpportunityRequestValidatorVerify;
       _myOpportunityRequestValidatorVerifyFinalize = myOpportunityRequestValidatorVerifyFinalize;
       _myOpportunityRequestValidatorVerifyFinalizeBatch = myOpportunityRequestValidatorVerifyFinalizeBatch;
+      _myOpportunityRequestValidatorVerifyImportCsv = myOpportunityRequestValidatorVerifyImportCsv;
       _myOpportunityRepository = myOpportunityRepository;
       _myOpportunityVerificationRepository = myOpportunityVerificationRepository;
       _executionStrategyService = executionStrategyService;
@@ -727,7 +740,7 @@ await _executionStrategyService.ExecuteInExecutionStrategyAsync(async () =>
 
         await _linkService.LogUsage(link.Id);
 
-        var request = new MyOpportunityRequestVerify { InstantVerification = true };
+        var request = new MyOpportunityRequestVerify { InstantOrImportedVerification = true };
         await PerformActionSendForVerification(user, link.EntityId, request, null); //any verification method
 
         await FinalizeVerification(user, opportunity, VerificationStatus.Completed, true, "Auto-verification");
@@ -932,9 +945,149 @@ public async Task FinalizeVerificationManual(MyOpportunityRequestVerifyFinalize
 
       return queryGrouped.ToDictionary(o => o.OpportunityId, o => o.Count);
     }
+
+    public async Task PerformActionImportVerificationFromCSV(MyOpportunityRequestVerifyImportCsv request, bool ensureOrganizationAuthorization)
+    {
+      ArgumentNullException.ThrowIfNull(request, nameof(request));
+
+      await _myOpportunityRequestValidatorVerifyImportCsv.ValidateAndThrowAsync(request);
+
+      var organization = _organizationService.GetById(request.OrganizationId, false, false, ensureOrganizationAuthorization);
+
+      request.Comment = request.Comment?.Trim();
+      if (string.IsNullOrEmpty(request.Comment)) request.Comment = "Auto-verification";
+
+      using var stream = request.File.OpenReadStream();
+      using var reader = new StreamReader(stream);
+
+      var config = new CsvConfiguration(CultureInfo.InvariantCulture)
+      {
+        Delimiter = ",",
+        HasHeaderRecord = true,
+        MissingFieldFound = args =>
+        {
+          if (args.Context?.Reader?.HeaderRecord == null)
+            throw new ValidationException("The file is missing a header row");
+
+          var fieldName = args.HeaderNames?[args.Index] ?? $"Field at index {args.Index}";
+
+          var modelType = typeof(MyOpportunityInfoCsvImport);
+
+          var property = modelType.GetProperties(BindingFlags.Public | BindingFlags.Instance)
+              .FirstOrDefault(p =>
+                  string.Equals(
+                      p.GetCustomAttributes(typeof(NameAttribute), true)
+                          .Cast<NameAttribute>()
+                          .FirstOrDefault()?.Names.FirstOrDefault() ?? p.Name,
+                      fieldName,
+                      StringComparison.OrdinalIgnoreCase));
+
+          if (property == null) return;
+
+          var isRequired = property.GetCustomAttributes(typeof(System.ComponentModel.DataAnnotations.RequiredAttribute), true).Length > 0;
+
+          if (isRequired)
+          {
+            var rowNumber = args.Context?.Parser?.Row.ToString() ?? "Unknown";
+            throw new ValidationException($"Missing required field '{fieldName}' in row '{rowNumber}'");
+          }
+        },
+        BadDataFound = args =>
+        {
+          var rowNumber = args.Context?.Parser?.Row.ToString() ?? "Unknown";
+          throw new ValidationException($"Bad data format in row '{rowNumber}': Raw field data: '{args.Field}'");
+        }
+      };
+
+      using var csv = new CsvHelper.CsvReader(reader, config);
+
+      if (!csv.Read() || csv.Context?.Reader?.HeaderRecord?.Length == 0)
+        throw new ValidationException("The file is missing a header row");
+
+      csv.ReadHeader();
+
+      await _executionStrategyService.ExecuteInExecutionStrategyAsync(async () =>
+      {
+        using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionScopeAsyncFlowOption.Enabled);
+
+        while (await csv.ReadAsync())
+        {
+          var rowNumber = csv.Context?.Parser?.Row ?? -1;
+          try
+          {
+            var record = csv.GetRecord<MyOpportunityInfoCsvImport>();
+
+            await ProcessImportVerification(request, record);
+          }
+          catch (Exception ex)
+          {
+            throw new ValidationException($"Error processing row '{(rowNumber == -1 ? "Unknown" : rowNumber)}': {ex.Message}");
+          }
+        }
+        scope.Complete();
+      });
+    }
     #endregion
 
     #region Private Members
+    private async Task ProcessImportVerification(MyOpportunityRequestVerifyImportCsv requestImport, MyOpportunityInfoCsvImport item)
+    {
+      item.Email = string.IsNullOrWhiteSpace(item.Email) ? null : item.Email.Trim();
+      item.PhoneNumber = string.IsNullOrWhiteSpace(item.PhoneNumber) ? null : item.PhoneNumber.Trim();
+      item.FirstName = string.IsNullOrWhiteSpace(item.FirstName) ? null : item.FirstName.Trim();
+      item.Surname = string.IsNullOrWhiteSpace(item.Surname) ? null : item.Surname.Trim();
+
+      Domain.Lookups.Models.Country? country = null;
+      if (!string.IsNullOrEmpty(item.Country)) country = _countryService.GetByCodeAplha2(item.Country);
+
+      Domain.Lookups.Models.Gender? gender = null;
+      if (!string.IsNullOrEmpty(item.Gender)) gender = _genderService.GetByName(item.Gender);
+
+      var username = item.Email ?? item.PhoneNumber;
+      if (string.IsNullOrEmpty(username))
+        throw new ValidationException("Email or phone number required");
+
+      if (string.IsNullOrWhiteSpace(item.OpporunityExternalId))
+        throw new ValidationException("Opportunity external id required");
+
+      var opportunity = _opportunityService.GetByExternalId(requestImport.OrganizationId, item.OpporunityExternalId, true, true);
+      if (opportunity.VerificationMethod != VerificationMethod.Automatic)
+        throw new ValidationException($"Verification import not supported for opporunity '{opportunity.Title}'. The verification method must be set to 'Automatic'");
+
+      var user = _userService.GetByUsernameOrNull(username, false, false);
+      //user is created if not existing, or updated if not linked to an identity provider
+      if (user == null || !user.ExternalId.HasValue)
+      {
+        var request = new UserRequest
+        {
+          Id = user?.Id,
+          Username = username,
+          Email = item.Email,
+          PhoneNumber = item.PhoneNumber,
+          FirstName = item.FirstName,
+          Surname = item.Surname,
+          EmailConfirmed = item.Email == null ? null : false,
+          PhoneNumberConfirmed = item.PhoneNumber == null ? null : false,
+          CountryId = country?.Id,
+          GenderId = gender?.Id
+        };
+
+        user = await _userService.Upsert(request);
+      }
+
+      await _executionStrategyService.ExecuteInExecutionStrategyAsync(async () =>
+      {
+        using var scope = new TransactionScope(TransactionScopeOption.Required, TransactionScopeAsyncFlowOption.Enabled);
+
+        var requestVerify = new MyOpportunityRequestVerify { InstantOrImportedVerification = true };
+        await PerformActionSendForVerification(user, opportunity.Id, requestVerify, null); //any verification method
+
+        await FinalizeVerification(user, opportunity, VerificationStatus.Completed, true, requestImport.Comment);
+
+        scope.Complete();
+      });
+    }
+
     private static List<(DateTime WeekEnding, int Count)> SummaryGroupByWeekItems(List<MyOpportunityInfo> items)
     {
       var results = items
@@ -1198,7 +1351,7 @@ private async Task PerformActionSendForVerification(User user, Guid opportunityI
       myOpportunity.ZltoReward = opportunity.ZltoReward;
       myOpportunity.YomaReward = opportunity.YomaReward;
 
-      if (request.InstantVerification || opportunity.VerificationMethod == VerificationMethod.Automatic) return; //with instant-verifications or automatic verification pending notifications are not sent
+      if (request.InstantOrImportedVerification) return; //with instant or imported verifications, pending notifications are not sent
 
       //sent to youth
       await SendNotification(myOpportunity, NotificationType.Opportunity_Verification_Pending);
@@ -1328,7 +1481,7 @@ private async Task PerformActionSendForVerificationProcessVerificationTypes(MyOp
       Models.MyOpportunity myOpportunity,
       List<BlobObject> itemsNewBlobs)
     {
-      if (request.InstantVerification) return; //with instant-verifications bypass any verification type requirements
+      if (request.InstantOrImportedVerification) return; //with instant or imported verifications bypass any verification type requirements
       if (opportunity.VerificationTypes == null) return;
 
       foreach (var verificationType in opportunity.VerificationTypes)

src/api/src/domain/Yoma.Core.Domain/MyOpportunity/Validators/MyOpportunityRequestValidatorVerify.cs

@@ -20,7 +20,7 @@ public MyOpportunityRequestValidatorVerify()
           .WithMessage("3 or more coordinate points expected per coordinate set i.e. Point: X-coordinate (longitude -180 to +180), Y-coordinate (latitude -90 to +90), Z-elevation.")
           .When(x => x.Geometry != null && x.Geometry.Type != Core.SpatialType.None);
       //with instant-verifications start or end date not captured
-      RuleFor(x => x.DateStart).NotEmpty().When(x => !x.InstantVerification).WithMessage("{PropertyName} is required.");
+      RuleFor(x => x.DateStart).NotEmpty().When(x => !x.InstantOrImportedVerification).WithMessage("{PropertyName} is required.");
       RuleFor(model => model.DateEnd)
           .GreaterThanOrEqualTo(model => model.DateStart)
           .When(model => model.DateEnd.HasValue)

src/api/src/domain/Yoma.Core.Domain/MyOpportunity/Validators/MyOpportunityRequestValidatorVerifyImportCsv.cs

@@ -0,0 +1,31 @@
+using FluentValidation;
+using Yoma.Core.Domain.Entity.Interfaces;
+using Yoma.Core.Domain.MyOpportunity.Models;
+
+namespace Yoma.Core.Domain.MyOpportunity.Validators
+{
+  public class MyOpportunityRequestValidatorVerifyImportCsv : AbstractValidator<MyOpportunityRequestVerifyImportCsv>
+  {
+    #region Class Variables
+    private readonly IOrganizationService _organizationService;
+    #endregion
+
+    #region Constructor
+    public MyOpportunityRequestValidatorVerifyImportCsv(IOrganizationService organizationService)
+    {
+      _organizationService = organizationService;
+
+      RuleFor(x => x.File).Must(file => file != null && file.Length > 0).WithMessage("{PropertyName} is required.");
+      RuleFor(x => x.OrganizationId).NotEmpty().Must(OrganizationExists).WithMessage($"Specified organization does not exist.");
+    }
+    #endregion
+
+    #region Private Members
+    private bool OrganizationExists(Guid id)
+    {
+      if (id == Guid.Empty) return false;
+      return _organizationService.GetByIdOrNull(id, false, false, false) != null;
+    }
+    #endregion
+  }
+}

src/api/src/domain/Yoma.Core.Domain/Opportunity/Interfaces/IOpportunityService.cs

@@ -12,6 +12,8 @@ public interface IOpportunityService
 
     Models.Opportunity? GetByTitleOrNull(string title, bool includeChildItems, bool includeComputed);
 
+    Models.Opportunity GetByExternalId(Guid organizationId, string externalId, bool includeChildItems, bool includeComputed);
+
     Models.Opportunity? GetByExternalIdOrNull(Guid organizationId, string externalId, bool includeChildItems, bool includeComputed);
 
     List<Models.Opportunity> Contains(string value, bool includeChildItems, bool includeComputed);