adding cache for spills

Overflow.Web/Components/Pages/Spills.razor

@@ -21,15 +21,7 @@
 <SpillsTable Job=@job />
 
 @code {
-    private SouthernWaterApiJob? job;
-    [Inject] private IMongoDatabase database { get; set; }
+    private SouthernWaterApiJob? job => cache.CurrentSouthernWaterApiJob;
+    [Inject] private SpillCache cache { get; set; }
     // private bool showIds;
-
-    protected override async Task OnInitializedAsync()
-    {
-        job = database.GetCollection<SouthernWaterApiJob>(Static.CollectionName)
-            .AsQueryable()
-            .OrderByDescending(j => j.EndTime)
-            .FirstOrDefault();
-    }
 }

Overflow.Web/Components/Pages/SpillsCalendarPage.razor

@@ -25,16 +25,8 @@
 <SpillsCalendar Job="@job" GenuineOnly="@genuineOnly" />
 
 @code {
-    private SouthernWaterApiJob? job;
-    [Inject] private IMongoDatabase database { get; set; }
+    private SouthernWaterApiJob? job => cache.CurrentSouthernWaterApiJob;
+    [Inject] private SpillCache cache { get; set; }
     // private bool showIds;
     private bool genuineOnly = true;
-
-    protected override async Task OnInitializedAsync()
-    {
-        job = database.GetCollection<SouthernWaterApiJob>(Static.CollectionName)
-            .AsQueryable()
-            .OrderByDescending(j => j.EndTime)
-            .FirstOrDefault();
-    }
 }

Overflow.Web/Program.cs

@@ -60,6 +60,22 @@ builder.Services.AddQuartz(q =>
         .WithCronSchedule(builder.Configuration.GetSection("SouthernWater").GetValue<string>("Cron") ?? "0 0 8 * * ?")
         .WithDescription("Periodic trigger for Southern Water API pulling")
     );
+
+    var cacheKey = new JobKey("cache-refresh", "cache");
+
+    q.AddJob<CacheReloadJob>(j => j
+        .WithDescription("Refresh caches")
+        .WithIdentity(cacheKey)
+        .UsingJobData("IsFull", false)
+    );
+
+    q.AddTrigger(t => t
+        .WithIdentity("cache-refresh-trigger")
+        .ForJob(cacheKey)
+        .StartNow()
+        .WithCronSchedule(builder.Configuration.GetSection("Cache").GetValue<string>("Cron") ?? "0 0 8 * * ?")
+        .WithDescription("Periodic trigger for cache refreshing")
+    );
 });
 
 // ASP.NET Core hosting
@@ -74,6 +90,10 @@ builder.Services.AddSingleton<SouthernWaterApi>();
 builder.Services.AddScoped<SouthernWaterApiJobRunner, SouthernWaterApiJobRunnerPersisting>();
 builder.Services.AddTransient<SouthernWaterJob>();
 
+builder.Services.AddSingleton<SpillCache>();
+builder.Services.AddSingleton<SouthernWaterSpillCache>();
+builder.Services.AddHostedService<LoadCacheOnStart>();
+
 builder.Services.AddRadzenComponents();
 
 var app = builder.Build();

Overflow.Web/appsettings.json

@@ -11,5 +11,8 @@
   "AllowedHosts": "*",
   "SouthernWater": {
     "Cron": "0 26 20 * * ?"
+  },
+  "Cache": {
+    "Cron": "0 */3 * * * ?"
   }
 }

Overflow/CacheReloadJob.cs

@@ -0,0 +1,15 @@
+using Microsoft.Extensions.Logging;
+using Quartz;
+
+namespace Overflow;
+
+[DisallowConcurrentExecution]
+public class CacheReloadJob(SpillCache cache, ILogger<CacheReloadJob> logger): IJob
+{
+    public Task Execute(IJobExecutionContext context)
+    {
+        logger.LogDebug("Refreshing caches");
+        cache.Refresh();
+        return Task.CompletedTask;
+    }
+}

Overflow/Overflow.csproj

@@ -10,6 +10,8 @@
       <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+      <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
+      <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
       <PackageReference Include="MongoDB.Driver" Version="2.25.0" />
       <PackageReference Include="Quartz" Version="3.9.0" />
     </ItemGroup>

Overflow/SouthernWater/SouthernWaterApi.cs

@@ -141,7 +141,7 @@ public partial class SouthernWaterApi
             }
             catch (TaskCanceledException e)
             {
-                _logger.LogError(e, "HTTP Timeout Exception while loading page [{}], waiting {} before retrying", page, Static.Interval);
+                _logger.LogError(e, "HTTP Timeout Exception while loading page [{}], waiting {} before retrying", page, 1.5 * Static.Interval);
                 await Task.Delay(1.5 * Static.Interval);
             }
         }

Overflow/SouthernWater/SouthernWaterApiJob.cs

@@ -11,5 +11,6 @@ public class SouthernWaterApiJob
     public DateTime? EndTime { get; set; }
     public TimeSpan Interval { get; set; }
     public int TotalItems { get; set; }
+    [BsonIgnore]
     public List<Spill> Spills { get; set; }
 }

Overflow/SouthernWater/SouthernWaterApiJobRunner.cs

@@ -36,6 +36,8 @@ public class SouthernWaterApiJobRunner(SouthernWaterApi client, ILogger<Southern
 
             _logger.LogInformation("Processing page [{}/{}]", page.currentPage, page.totalPages);
 
+            page.items.ForEach(s => s.JobId = job._id);
+
             job.TotalItems = page.totalItems;
             job.Spills.AddRange(page.items);
             try
@@ -86,22 +88,17 @@ public class SouthernWaterApiJobRunnerPersisting(
     IMongoDatabase mongo)
     : SouthernWaterApiJobRunner(client, logger)
 {
-    private readonly IMongoCollection<SouthernWaterApiJob> _collection = mongo.GetCollection<SouthernWaterApiJob>(Static.CollectionName);
+    private readonly IMongoCollection<SouthernWaterApiJob> _jobCollection = mongo.GetCollection<SouthernWaterApiJob>(Static.JobCollectionName);
+    private readonly IMongoCollection<Spill> _spillCollection = mongo.GetCollection<Spill>(Static.SpillCollectionName);
 
     protected override async Task JobCreated(SouthernWaterApiJob job)
     {
-        await _collection.InsertOneAsync(job);
+        await _jobCollection.InsertOneAsync(job);
     }
 
     protected override async Task PageLoaded(SouthernWaterApiJob job, PagedItems<Spill> page)
     {
-        var finder = Builders<SouthernWaterApiJob>.Filter
-            .Eq(j => j._id, job._id);
-
-        var update = Builders<SouthernWaterApiJob>.Update
-            .PushEach(j => j.Spills, page.items);
-
-        await _collection.UpdateOneAsync(finder, update);
+        await _spillCollection.InsertManyAsync(page.items);
     }
 
     protected override async Task JobFinished(SouthernWaterApiJob job)
@@ -113,6 +110,6 @@ public class SouthernWaterApiJobRunnerPersisting(
             .Set(j => j.EndTime, job.EndTime)
             .Set(j => j.TotalItems, job.TotalItems);
 
-        await _collection.UpdateOneAsync(finder, update);
+        await _jobCollection.UpdateOneAsync(finder, update);
     }
 }

Overflow/SouthernWater/SouthernWaterSpillCache.cs

@@ -0,0 +1,44 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+
+namespace Overflow.SouthernWater;
+
+public class SouthernWaterSpillCache(IServiceProvider serviceProvider, ILogger<SouthernWaterSpillCache> logger)
+{
+    private readonly ILogger _logger = logger;
+
+    private SouthernWaterApiJob _currentJob;
+
+    public SouthernWaterApiJob CurrentJob => _currentJob;
+
+    public void ReloadJob()
+    {
+        _logger.LogInformation("Refreshing Southern Water Spills");
+
+        using var scope = serviceProvider.CreateScope();
+
+        var database = scope.ServiceProvider.GetRequiredService<IMongoDatabase>();
+
+        var job = database.GetCollection<SouthernWaterApiJob>(Static.JobCollectionName)
+            .AsQueryable()
+            .OrderByDescending(j => j.EndTime)
+            .FirstOrDefault();
+
+        if (job is not null)
+        {
+            job.Spills = database.GetCollection<Spill>(Static.SpillCollectionName)
+                .AsQueryable()
+                .Where(s => s.JobId == job._id)
+                .ToList();
+
+            _currentJob = job;
+
+            _logger.LogInformation("Southern Water Spills cache refreshed");
+        }
+        else
+        {
+            _logger.LogWarning("No Southern Water Spills returned");
+        }
+    }
+}

Overflow/SouthernWater/Spill.cs

@@ -6,6 +6,11 @@ namespace Overflow.SouthernWater;
 
 public class Spill
 {
+    [JsonIgnore]
+    public ObjectId _id { get; set; }
+    [JsonIgnore]
+    public ObjectId JobId { get; set; }
+
     [JsonPropertyName("id")]
     public int sw_id { get; set; }
     public int eventId { get; set; }

Overflow/SpillCache.cs

@@ -0,0 +1,35 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Overflow.SouthernWater;
+using Quartz;
+
+namespace Overflow;
+
+public class SpillCache(SouthernWaterSpillCache southernWaterSpillCache, ILogger<SpillCache> logger)
+{
+    public SouthernWaterApiJob CurrentSouthernWaterApiJob => southernWaterSpillCache.CurrentJob;
+
+    public void Refresh()
+    {
+        logger.LogDebug("Refreshing caches");
+
+        southernWaterSpillCache.ReloadJob();
+    }
+}
+
+public class LoadCacheOnStart(ISchedulerFactory scheduler, SpillCache cache, ILogger<LoadCacheOnStart> logger) : IHostedService
+{
+    public async Task StartAsync(CancellationToken cancellationToken)
+    {
+        logger.LogInformation("Loading caches for startup");
+
+        await (await scheduler.GetScheduler()).TriggerJob(new JobKey("cache-refresh", "cache"));
+        // cache.Refresh();
+
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+}

Overflow/Static.cs

@@ -3,7 +3,8 @@ namespace Overflow;
 public static class Static
 {
     public static readonly string DatabaseName = "overflow";
-    public static readonly string CollectionName = "southern_water_api_job";
+    public static readonly string JobCollectionName = "southern_water_api_job";
+    public static readonly string SpillCollectionName = "southern_water_spills";
 
     public static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
     public static readonly int IntervalWiggleSeconds = 10;