Adding showgrounds fields to Sites, consuming showgrounds and adding …

src/KeeperData.Api/appsettings.json

@@ -67,7 +67,8 @@
     "SamHoldersEnabled": true,
     "SamHerdsEnabled": true,
     "SamPartiesEnabled": true,
-    "SamCommonLandsEnabled": true
+    "SamCommonLandsEnabled": true,
+    "SamShowgroundsEnabled":  true
   },
   "DataBridgeScanConfiguration": {
     "QueryPageSize": 100,

src/KeeperData.Application/Orchestration/ChangeScanning/Sam/Bulk/SamBulkScanContext.cs

@@ -10,4 +10,5 @@ public class SamBulkScanContext : ScanContext, IBulkScanContext
     public int PageSize { get; init; } = 100;
     public EntityScanContext Holders { get; init; } = new();
     public EntityScanContext Holdings { get; init; } = new();
+    public EntityScanContext Showgrounds { get; init; } = new();
 }

src/KeeperData.Application/Orchestration/ChangeScanning/Sam/Bulk/Steps/SamShowgroundBulkScanStep.cs

@@ -0,0 +1,64 @@
+using KeeperData.Application.Orchestration.ChangeScanning.BaseClasses;
+using KeeperData.Core.ApiClients.DataBridgeApi;
+using KeeperData.Core.ApiClients.DataBridgeApi.Configuration;
+using KeeperData.Core.ApiClients.DataBridgeApi.Contracts;
+using KeeperData.Core.Attributes;
+using KeeperData.Core.Messaging.Contracts.V1.Sam;
+using KeeperData.Core.Messaging.MessagePublishers;
+using KeeperData.Core.Messaging.MessagePublishers.Clients;
+using KeeperData.Core.Providers;
+using Microsoft.Extensions.Logging;
+
+namespace KeeperData.Application.Orchestration.ChangeScanning.Sam.Bulk.Steps;
+
+[StepOrder(2)]
+public class SamShowgroundBulkScanStep(
+    IDataBridgeClient dataBridgeClient,
+    IMessagePublisher<IntakeEventsQueueClient> intakeMessagePublisher,
+    DataBridgeScanConfiguration dataBridgeScanConfiguration,
+    IDelayProvider delayProvider,
+    ILogger<SamShowgroundBulkScanStep> logger)
+    : BulkScanStepBase<SamBulkScanContext, SamScanShowgroundIdentifier, SamImportHoldingMessage>(
+        dataBridgeClient,
+        intakeMessagePublisher,
+        dataBridgeScanConfiguration,
+        delayProvider,
+        logger)
+{
+    protected override string SelectFields => "CPH";
+    protected override string OrderBy => "CPH asc";
+
+    protected override EntityScanContext GetScanContext(SamBulkScanContext context) => context.Showgrounds;
+    protected override async Task<DataBridgeResponse<SamScanShowgroundIdentifier>> GetHoldingsAsync(
+        int top,
+        int skip,
+        string selectFields,
+        DateTime? updatedSince,
+        string orderBy,
+        CancellationToken cancellationToken)
+    {
+        var result = await DataBridgeClient.GetSamShowgroundsAsync<SamScanShowgroundIdentifier>(
+            top,
+            skip,
+            selectFields,
+            updatedSince,
+            orderBy,
+            cancellationToken);
+
+        return result ?? new DataBridgeResponse<SamScanShowgroundIdentifier> { CollectionName = "SamShowgrounds" };
+    }
+
+    protected override string ExtractIdentifier(SamScanShowgroundIdentifier holdingIdentifier)
+    {
+        return holdingIdentifier.CPH;
+    }
+
+    protected override SamImportHoldingMessage CreateImportMessage(string identifier)
+    {
+        return new SamImportHoldingMessage
+        {
+            Id = Guid.NewGuid(),
+            Identifier = identifier
+        };
+    }
+}

src/KeeperData.Application/Orchestration/ChangeScanning/Sam/Daily/SamDailyScanContext.cs

@@ -11,4 +11,5 @@ public class SamDailyScanContext : ScanContext
     public EntityScanContext Parties { get; init; } = new();
     public EntityScanContext Ports { get; init; } = new();
     public EntityScanContext CommonLands { get; init; } = new();
+    public EntityScanContext Showgrounds { get; init; } = new();
 }

src/KeeperData.Application/Orchestration/ChangeScanning/Sam/Daily/Steps/SamShowgroundDailyScanStep.cs

@@ -0,0 +1,63 @@
+using KeeperData.Application.Orchestration.ChangeScanning.BaseClasses;
+using KeeperData.Core.ApiClients.DataBridgeApi;
+using KeeperData.Core.ApiClients.DataBridgeApi.Configuration;
+using KeeperData.Core.ApiClients.DataBridgeApi.Contracts;
+using KeeperData.Core.Attributes;
+using KeeperData.Core.Messaging.Contracts.V1.Sam;
+using KeeperData.Core.Messaging.MessagePublishers;
+using KeeperData.Core.Messaging.MessagePublishers.Clients;
+using KeeperData.Core.Providers;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace KeeperData.Application.Orchestration.ChangeScanning.Sam.Daily.Steps;
+
+[StepOrder(6)]
+public class SamShowgroundDailyScanStep(
+    IDataBridgeClient dataBridgeClient,
+    IMessagePublisher<IntakeEventsQueueClient> intakeMessagePublisher,
+    DataBridgeScanConfiguration dataBridgeScanConfiguration,
+    IDelayProvider delayProvider,
+    IConfiguration configuration,
+    ILogger<SamShowgroundDailyScanStep> logger)
+    : DailyScanStepBase<SamDailyScanContext, SamScanShowgroundIdentifier>(dataBridgeClient, intakeMessagePublisher, dataBridgeScanConfiguration,
+        delayProvider, configuration, logger)
+{
+    private const string SelectFields = "CPH";
+    private const string OrderBy = "CPH asc";
+
+    protected override bool IsEntityEnabled()
+        => Configuration.GetValue<bool>("DataBridgeCollectionFlags:SamShowgroundsEnabled");
+
+    protected override EntityScanContext GetScanContext(SamDailyScanContext context)
+        => context.Showgrounds;
+
+    protected override async Task<DataBridgeResponse<SamScanShowgroundIdentifier>?> QueryDataAsync(
+        SamDailyScanContext context,
+        CancellationToken cancellationToken)
+        => await DataBridgeClient.GetSamShowgroundsAsync<SamScanShowgroundIdentifier>(
+            context.Showgrounds.CurrentTop,
+            context.Showgrounds.CurrentSkip,
+            SelectFields,
+            context.UpdatedSinceDateTime,
+            OrderBy,
+            cancellationToken);
+
+    protected override async Task PublishMessagesAsync(
+        DataBridgeResponse<SamScanShowgroundIdentifier> queryResponse,
+        CancellationToken cancellationToken)
+    {
+        var identifiers = queryResponse.Data
+            .Select(x => x.CPH)
+            .Where(x => !string.IsNullOrWhiteSpace(x))
+            .Distinct()
+            .ToList();
+
+        foreach (var id in identifiers)
+        {
+            var message = new SamUpdateHoldingMessage { Id = Guid.NewGuid(), Identifier = id };
+
+            await IntakeMessagePublisher.PublishAsync(message, cancellationToken);
+        }
+    }
+}

src/KeeperData.Application/Orchestration/Imports/Sam/Holdings/SamHoldingImportContext.cs

@@ -34,4 +34,5 @@ public class SamHoldingImportContext
 
     public List<SitePartyRoleRelationship> PartiesWithNoRelationshipToSiteToClean { get; set; } = [];
     public List<SamPort> RawPorts { get; set; } = [];
+    public List<SamShowground> RawShowgrounds { get; set; } = [];
 }

src/KeeperData.Application/Orchestration/Imports/Sam/Holdings/Steps/SamHoldingImportAggregationStep.cs

@@ -20,6 +20,7 @@ protected override async Task ExecuteCoreAsync(SamHoldingImportContext context,
         var getHerdsTask = _dataBridgeClient.GetSamHerdsAsync(context.Cph, cancellationToken);
         var getPortsTask = _dataBridgeClient.GetSamPortsAsync(context.Cph, cancellationToken);
         var getCommonLandsByCommonCphTask = _dataBridgeClient.GetSamCommonLandsByCommonCphAsync(context.Cph, cancellationToken);
+        var getShowgroundsTask = _dataBridgeClient.GetSamShowgroundsByCphAsync(context.Cph, cancellationToken);
 
         await Task.WhenAll(
             getHoldingsTask,
@@ -40,6 +41,7 @@ await Task.WhenAll(
 
         var parties = await GetSamPartiesAsync(context, cancellationToken);
         context.RawParties = SamPartyMapper.AggregatePartyAndHolder(parties, context.RawHolders);
+        context.RawShowgrounds = getShowgroundsTask.Result;
     }
 
     private async Task<List<SamParty>> GetSamPartiesAsync(SamHoldingImportContext context, CancellationToken cancellationToken)

src/KeeperData.Application/Orchestration/Imports/Sam/Holdings/Steps/SamHoldingImportGoldMappingStep.cs

@@ -60,6 +60,7 @@ protected override async Task ExecuteCoreAsync(SamHoldingImportContext context,
                 context.SilverHoldings,
                 context.GoldSiteGroupMarks,
                 context.GoldParties,
+                context.RawShowgrounds,
                 countryIdentifierLookupService.GetByIdAsync,
                 siteTypeLookupService.GetByCodeAsync,
                 siteIdentifierTypeLookupService.GetByCodeAsync,

src/KeeperData.Application/Orchestration/Imports/Sam/Mappings/SamHoldingMapper.cs

@@ -56,7 +56,6 @@ public static async Task<SamHoldingDocument> ToSilver(
         var result = new SamHoldingDocument
         {
             // Id - Leave to support upsert assigning Id
-
             LastUpdatedBatchId = h.BATCH_ID,
             CreatedDate = h.CreatedAtUtc ?? DateTime.UtcNow,
             LastUpdatedDate = h.UpdatedAtUtc ?? DateTime.UtcNow,
@@ -109,10 +108,8 @@ public static async Task<SamHoldingDocument> ToSilver(
                     AddressTown = h.TOWN,
                     AddressPostCode = h.POSTCODE,
                     CountrySubDivision = h.UK_INTERNAL_CODE,
-
                     CountryIdentifier = countryId,
                     CountryCode = countryCode,
-
                     UniquePropertyReferenceNumber = h.UDPRN
                 }
             },
@@ -184,6 +181,7 @@ public static SamHoldingDocument SelectAddressSource(List<SamHoldingDocument> si
         List<SamHoldingDocument> silverHoldings,
         List<SiteGroupMarkRelationshipDocument> goldSiteGroupMarks,
         List<PartyDocument> goldParties,
+        List<SamShowground> rawShowgrounds,
         Func<string?, CancellationToken, Task<CountryDocument?>> getCountryById,
         Func<string?, CancellationToken, Task<SiteTypeDocument?>> getSiteTypeByCode,
         Func<string?, CancellationToken, Task<SiteIdentifierTypeDocument?>> getSiteIdentifierTypeByCode,
@@ -233,6 +231,22 @@ public static SamHoldingDocument SelectAddressSource(List<SamHoldingDocument> si
             cphnSiteIdentifierTypeDocument.Name,
             cphnSiteIdentifierTypeDocument.LastModifiedDate);
 
+        var showground = rawShowgrounds?.FirstOrDefault();
+        DateTime? effectiveFromDate = null;
+        DateTime? effectiveToDate = null;
+        bool? approvalCurrentFlag = null;
+
+        if (showground != null)
+        {
+            effectiveFromDate = showground.START_DATE;
+            effectiveToDate = showground.END_DATE;
+            var now = DateTime.UtcNow;
+
+            approvalCurrentFlag =
+                (effectiveFromDate == null || now >= effectiveFromDate.Value)
+                && (effectiveToDate == null || now <= effectiveToDate.Value);
+        }
+
         var site = existingSite is not null
             ? await UpdateSiteAsync(
                 representative,
@@ -245,6 +259,9 @@ public static SamHoldingDocument SelectAddressSource(List<SamHoldingDocument> si
                 allDerivedActivities,
                 derivedSiteType,
                 cphnSiteIdentifierType,
+                effectiveFromDate,
+                effectiveToDate,
+                approvalCurrentFlag,
                 cancellationToken)
             : await CreateSiteAsync(
                 goldSiteId,
@@ -257,6 +274,9 @@ public static SamHoldingDocument SelectAddressSource(List<SamHoldingDocument> si
                 allDerivedActivities,
                 derivedSiteType,
                 cphnSiteIdentifierType,
+                effectiveFromDate,
+                effectiveToDate,
+                approvalCurrentFlag,
                 cancellationToken);
 
         return SiteDocument.FromDomain(site);
@@ -367,6 +387,9 @@ private static async Task<Site> CreateSiteAsync(
         List<SiteActivity> activities,
         SiteType? siteType,
         SiteIdentifierType? siteIdentifierType,
+        DateTime? effectiveFromDate,
+        DateTime? effectiveToDate,
+        bool? approvalCurrentFlag,
         CancellationToken cancellationToken)
     {
         var (address, communication) = await ResolveLocationPartsAsync(addressSource, getCountryById, cancellationToken);
@@ -394,7 +417,10 @@ private static async Task<Site> CreateSiteAsync(
             representative.CphTypeIdentifier,
             siteType,
             location,
-            isPermanentLandHolding ? representative.SecondaryCph : null);
+            representative.CphRelationshipType.IsPermanentLandHolding() ? representative.SecondaryCph : null,
+            effectiveFromDate,
+            effectiveToDate,
+            approvalCurrentFlag);
 
         ApplySiteData(site, goldSiteId, representative, goldSiteGroupMarks, goldParties, species, activities, siteIdentifierType);
 
@@ -412,6 +438,9 @@ private static async Task<Site> UpdateSiteAsync(
         List<SiteActivity> activities,
         SiteType? siteType,
         SiteIdentifierType? siteIdentifierType,
+        DateTime? effectiveFromDate,
+        DateTime? effectiveToDate,
+        bool? approvalCurrentFlag,
         CancellationToken cancellationToken)
     {
         var isPermanentLandHolding = representative.CphRelationshipType.IsPermanentLandHolding();
@@ -428,7 +457,10 @@ private static async Task<Site> UpdateSiteAsync(
             representative.Deleted,
             isPermanentLandHolding ? null : representative.SecondaryCph,
             representative.CphTypeIdentifier,
-            isPermanentLandHolding ? representative.SecondaryCph : null);
+            representative.CphRelationshipType.IsPermanentLandHolding() ? representative.SecondaryCph : null,
+            effectiveFromDate,
+            effectiveToDate,
+            approvalCurrentFlag);
 
         var (updatedAddress, updatedCommunication) = await ResolveLocationPartsAsync(addressSource, getCountryById, cancellationToken);
 

src/KeeperData.Core/ApiClients/DataBridgeApi/Contracts/SamShowground.cs

@@ -0,0 +1,24 @@
+using KeeperData.Core.ApiClients.DataBridgeApi.Converters;
+using System.Text.Json.Serialization;
+
+namespace KeeperData.Core.ApiClients.DataBridgeApi.Contracts;
+
+public class SamShowground : BronzeBase
+{
+    [JsonPropertyName("CPH")]
+    public string CPH { get; set; } = string.Empty;
+
+    [JsonPropertyName("START_DATE")]
+    [JsonConverter(typeof(SafeNullableDateTimeConverter))]
+    public DateTime? START_DATE { get; set; }
+
+    [JsonPropertyName("END_DATE")]
+    [JsonConverter(typeof(SafeNullableDateTimeConverter))]
+    public DateTime? END_DATE { get; set; }
+}
+
+public class SamScanShowgroundIdentifier
+{
+    [JsonPropertyName("CPH")]
+    public string CPH { get; set; } = string.Empty;
+}

src/KeeperData.Core/ApiClients/DataBridgeApi/DataBridgeApiRoutes.cs

@@ -12,4 +12,5 @@ public static class DataBridgeApiRoutes
     public const string GetSamHerds = "api/query/sam_herd";
     public const string GetSamPorts = "api/query/amls2_port";
     public const string GetSamCommonLands = "api/query/amls2_common_land";
+    public const string GetSamShowgrounds = "api/query/sam_showground";
 }

src/KeeperData.Core/ApiClients/DataBridgeApi/DataBridgeQueries.cs

@@ -161,4 +161,11 @@ public static Dictionary<string, string> SamPortsByCph(string id)
             [FilterKey] = $"CPH eq '{id}'"
         };
     }
+    public static Dictionary<string, string> SamShowgroundsByCph(string id)
+    {
+        return new Dictionary<string, string>
+        {
+            [FilterKey] = $"CPH eq '{id}'"
+        };
+    }
 }

src/KeeperData.Core/ApiClients/DataBridgeApi/IDataBridgeClient.cs

@@ -93,4 +93,13 @@ public interface IDataBridgeClient
         string? orderBy = null,
         CancellationToken cancellationToken = default);
     Task<List<SamCommonLand>> GetSamCommonLandsByCommonCphAsync(string cph, CancellationToken cancellationToken);
+
+    Task<DataBridgeResponse<T>?> GetSamShowgroundsAsync<T>(
+        int top,
+        int skip,
+        string? selectFields = null,
+        DateTime? updatedSinceDateTime = null,
+        string? orderBy = null,
+        CancellationToken cancellationToken = default);
+    Task<List<SamShowground>> GetSamShowgroundsByCphAsync(string cph, CancellationToken cancellationToken);
 }

src/KeeperData.Core/DTOs/SiteDto.cs

@@ -115,4 +115,13 @@ public class SiteDto
     public List<AssociatedHoldingDto> AssociatedCommonLands { get; set; } = [];
     [JsonPropertyName("permanentLandHoldingIdentifier")]
     public string? PermanentLandHoldingIdentifier { get; set; }
+
+    [JsonPropertyName("effectiveFromDate")]
+    public DateTime? EffectiveFromDate { get; set; }
+
+    [JsonPropertyName("effectiveToDate")]
+    public DateTime? EffectiveToDate { get; set; }
+
+    [JsonPropertyName("approvalCurrentFlag")]
+    public bool? ApprovalCurrentFlag { get; set; }
 }