shifting spills to a separate collection

Overflow.Web/Components/Pages/Spills.razor

@@ -27,9 +27,17 @@
 
     protected override async Task OnInitializedAsync()
     {
-        job = database.GetCollection<SouthernWaterApiJob>(Static.CollectionName)
+        job = database.GetCollection<SouthernWaterApiJob>(Static.JobCollectionName)
             .AsQueryable()
             .OrderByDescending(j => j.EndTime)
             .FirstOrDefault();
+
+        if (job is not null)
+        {
+            job.Spills = database.GetCollection<Spill>(Static.SpillCollectionName)
+                .AsQueryable()
+                .Where(s => s.JobId == job._id)
+                .ToList();
+        }
     }
 }

Overflow.Web/Components/Pages/SpillsCalendarPage.razor

@@ -32,9 +32,17 @@
 
     protected override async Task OnInitializedAsync()
     {
-        job = database.GetCollection<SouthernWaterApiJob>(Static.CollectionName)
+        job = database.GetCollection<SouthernWaterApiJob>(Static.JobCollectionName)
             .AsQueryable()
             .OrderByDescending(j => j.EndTime)
             .FirstOrDefault();
+
+        if (job is not null)
+        {
+            job.Spills = database.GetCollection<Spill>(Static.SpillCollectionName)
+                .AsQueryable()
+                .Where(s => s.JobId == job._id)
+                .ToList();
+        }
     }
 }

Overflow/SouthernWater/SouthernWaterApi.cs

@@ -141,7 +141,7 @@ public partial class SouthernWaterApi
             }
             catch (TaskCanceledException e)
             {
-                _logger.LogError(e, "HTTP Timeout Exception while loading page [{}], waiting {} before retrying", page, Static.Interval);
+                _logger.LogError(e, "HTTP Timeout Exception while loading page [{}], waiting {} before retrying", page, 1.5 * Static.Interval);
                 await Task.Delay(1.5 * Static.Interval);
             }
         }

Overflow/SouthernWater/SouthernWaterApiJob.cs

@@ -11,5 +11,6 @@ public class SouthernWaterApiJob
     public DateTime? EndTime { get; set; }
     public TimeSpan Interval { get; set; }
     public int TotalItems { get; set; }
+    [BsonIgnore]
     public List<Spill> Spills { get; set; }
 }

Overflow/SouthernWater/SouthernWaterApiJobRunner.cs

@@ -36,6 +36,8 @@ public class SouthernWaterApiJobRunner(SouthernWaterApi client, ILogger<Southern
 
             _logger.LogInformation("Processing page [{}/{}]", page.currentPage, page.totalPages);
 
+            page.items.ForEach(s => s.JobId = job._id);
+
             job.TotalItems = page.totalItems;
             job.Spills.AddRange(page.items);
             try
@@ -86,22 +88,17 @@ public class SouthernWaterApiJobRunnerPersisting(
     IMongoDatabase mongo)
     : SouthernWaterApiJobRunner(client, logger)
 {
-    private readonly IMongoCollection<SouthernWaterApiJob> _collection = mongo.GetCollection<SouthernWaterApiJob>(Static.CollectionName);
+    private readonly IMongoCollection<SouthernWaterApiJob> _jobCollection = mongo.GetCollection<SouthernWaterApiJob>(Static.JobCollectionName);
+    private readonly IMongoCollection<Spill> _spillCollection = mongo.GetCollection<Spill>(Static.SpillCollectionName);
 
     protected override async Task JobCreated(SouthernWaterApiJob job)
     {
-        await _collection.InsertOneAsync(job);
+        await _jobCollection.InsertOneAsync(job);
     }
 
     protected override async Task PageLoaded(SouthernWaterApiJob job, PagedItems<Spill> page)
     {
-        var finder = Builders<SouthernWaterApiJob>.Filter
+        await _spillCollection.InsertManyAsync(page.items);
-            .Eq(j => j._id, job._id);
-
-        var update = Builders<SouthernWaterApiJob>.Update
-            .PushEach(j => j.Spills, page.items);
-
-        await _collection.UpdateOneAsync(finder, update);
     }
 
     protected override async Task JobFinished(SouthernWaterApiJob job)
@@ -113,6 +110,6 @@ public class SouthernWaterApiJobRunnerPersisting(
             .Set(j => j.EndTime, job.EndTime)
             .Set(j => j.TotalItems, job.TotalItems);
 
-        await _collection.UpdateOneAsync(finder, update);
+        await _jobCollection.UpdateOneAsync(finder, update);
     }
 }

Overflow/SouthernWater/Spill.cs

@@ -6,6 +6,11 @@ namespace Overflow.SouthernWater;
 
 public class Spill
 {
+    [JsonIgnore]
+    public ObjectId _id { get; set; }
+    [JsonIgnore]
+    public ObjectId JobId { get; set; }
+
     [JsonPropertyName("id")]
     public int sw_id { get; set; }
     public int eventId { get; set; }

Overflow/Static.cs

@@ -3,7 +3,8 @@ namespace Overflow;
 public static class Static
 {
     public static readonly string DatabaseName = "overflow";
-    public static readonly string CollectionName = "southern_water_api_job";
+    public static readonly string JobCollectionName = "southern_water_api_job";
+    public static readonly string SpillCollectionName = "southern_water_spills";
 
     public static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
     public static readonly int IntervalWiggleSeconds = 10;