adding quartz job, interval wiggle, fixed page parameters

Overflow.CLI/Program.cs

@@ -10,6 +10,6 @@ var driver = new MongoClient("mongodb://localhost");
 var api = new SouthernWaterApi(new HttpClient());
 await api.LoadApiUrl();
 
-var runner = new SouthernWaterApiJobRunnerPersisting(api, NullLogger<SouthernWaterApiJobRunner>.Instance, driver.GetDatabase("overflow"));
+var runner = new SouthernWaterApiJobRunnerPersisting(api, NullLogger<SouthernWaterApiJobRunner>.Instance, driver.GetDatabase(Static.DatabaseName));
 
 await runner.LoadSpills(5);

Overflow.Web/Overflow.Web.csproj

@@ -9,6 +9,9 @@
     <ItemGroup>
       <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="8.0.6" />
       <PackageReference Include="MongoDB.Driver" Version="2.25.0" />
+      <PackageReference Include="NLog" Version="5.3.2" />
+      <PackageReference Include="NLog.Extensions.Logging" Version="5.3.11" />
+      <PackageReference Include="NLog.Web.AspNetCore" Version="5.3.11" />
       <PackageReference Include="Quartz" Version="3.9.0" />
       <PackageReference Include="Quartz.AspNetCore" Version="3.9.0" />
       <PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.9.0" />
@@ -28,4 +31,10 @@
       <_ContentIncludedByDefault Remove="wwwroot\favicon.png" />
     </ItemGroup>
     
+    <ItemGroup>
+        <None Update="nlog.config">
+            <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        </None>
+    </ItemGroup>
+
 </Project>

Overflow.Web/Program.cs

@@ -3,6 +3,7 @@ using Overflow;
 
 
 using MongoDB.Driver;
+using NLog.Extensions.Logging;
 using Overflow.SouthernWater;
 using Quartz;
 using Quartz.AspNetCore;
@@ -14,9 +15,12 @@ builder.Services.AddRazorComponents()
     .AddInteractiveServerComponents()
     .AddInteractiveWebAssemblyComponents();
 
+builder.Logging.ClearProviders();
+builder.Logging.AddNLog(builder.Configuration);
+
 var driver = new MongoClient(builder.Configuration.GetConnectionString("Default"));
 builder.Services.AddSingleton(driver);
-builder.Services.AddScoped<IMongoDatabase>(s => s.GetRequiredService<MongoClient>().GetDatabase("overflow"));
+builder.Services.AddScoped<IMongoDatabase>(s => s.GetRequiredService<MongoClient>().GetDatabase(Static.DatabaseName));
 
 builder.Services.AddControllers();
 
@@ -33,6 +37,29 @@ builder.Services.Configure<QuartzOptions>(options =>
 builder.Services.AddQuartz(q =>
 {
     // base Quartz scheduler, job and trigger configuration
+
+    q.UseSimpleTypeLoader();
+    q.UseInMemoryStore();
+    q.UseDefaultThreadPool(tp =>
+    {
+        tp.MaxConcurrency = 5;
+    });
+
+    var swKey = new JobKey("southern-water-api", "southern-water");
+
+    q.AddJob<SouthernWaterJob>(j => j
+        .WithDescription("Pull spills data from Southern Water API")
+        .WithIdentity(swKey)
+        .UsingJobData("IsFull", false)
+    );
+
+    q.AddTrigger(t => t
+        .WithIdentity("southern-water-api-trigger")
+        .ForJob(swKey)
+        .StartNow()
+        .WithCronSchedule(builder.Configuration.GetSection("SouthernWater").GetValue<string>("Cron") ?? "0 0 8 * * ?")
+        .WithDescription("Periodic trigger for Southern Water API pulling")
+    );
 });
 
 // ASP.NET Core hosting
@@ -45,6 +72,7 @@ builder.Services.AddQuartzServer(options =>
 builder.Services.AddHttpClient();
 builder.Services.AddSingleton<SouthernWaterApi>();
 builder.Services.AddScoped<SouthernWaterApiJobRunner, SouthernWaterApiJobRunnerPersisting>();
+builder.Services.AddTransient<SouthernWaterJob>();
 
 var app = builder.Build();
 

Overflow.Web/appsettings.json

@@ -8,5 +8,8 @@
   "ConnectionStrings": {
     "Default": "mongodb://localhost"
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  "SouthernWater": {
+    "Cron": "0 26 20 * * ?"
+  }
 }

Overflow.Web/nlog.config

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- XSD manual extracted from package NLog.Schema: https://www.nuget.org/packages/NLog.Schema-->
+<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd" xsi:schemaLocation="NLog NLog.xsd"
+      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+      autoReload="true"
+      throwConfigExceptions="true"
+      internalLogFile="./log/overflow.nlog.log"
+      internalLogLevel="Info" >
+
+    <extensions>
+        <add assembly="NLog.Web.AspNetCore"/>
+    </extensions>
+
+    <variable name="scopeFormat"
+              value="${all-event-properties:format=[[key]\:[value]]:includeScopeProperties=true:separator= }"/>
+
+    <variable name="format"
+              value="${longdate}|${level:uppercase=true}|${callsite}:${callsite-linenumber}|${message}${onexception:inner=${newline}}${exception:format=tostring,data:exceptionDataSeparator=\r\n}${newline}    ${scopeFormat}"/>
+
+    <!-- the targets to write to -->
+    <targets>
+        <!-- write logs to file -->
+        <target xsi:type="File"
+                name="logfile"
+                fileName="./log/overflow-${shortdate}.log"
+                layout="${format}" />
+        <target xsi:type="File"
+                name="tracefile"
+                fileName="./log/overflow.trace-${shortdate}.log"
+                layout="${format}" />
+        <target xsi:type="ColoredConsole"
+                name="logconsole"
+                layout="${format}" />
+    </targets>
+
+    <!-- rules to map from logger name to target -->
+    <rules>
+        <!--<logger name="*" minlevel="Trace" writeTo="tracefile" />-->
+        <logger name="Overflow.*" minlevel="Debug" writeTo="logconsole" />
+
+        <!--Output hosting lifetime messages to console target for faster startup detection -->
+        <logger name="Microsoft.Hosting.Lifetime" minlevel="Info" writeTo="logconsole" final="true" />
+
+        <!--Skip non-critical Microsoft logs and so log only own logs (BlackHole) -->
+        <logger name="Microsoft.*" maxlevel="Info" final="true" />
+        <logger name="System.Net.Http.*" maxlevel="Info" final="true" />
+
+        <!--<logger name="*" minlevel="Debug" writeTo="logfile" />-->
+        <logger name="Overflow.*" minlevel="Info" writeTo="logfile" />
+        <logger name="Quartz.*" minlevel="Info" writeTo="logfile" />
+        <logger name="Microsoft.*" minlevel="Warning" writeTo="logfile" />
+    </rules>
+</nlog>

Overflow/Overflow.csproj

@@ -11,6 +11,7 @@
       <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
       <PackageReference Include="MongoDB.Driver" Version="2.25.0" />
+      <PackageReference Include="Quartz" Version="3.9.0" />
     </ItemGroup>
 
     <ItemGroup>

Overflow/SouthernWater/SouthernWaterApi.cs

@@ -1,6 +1,7 @@
 using System.Net.Http.Json;
 using System.Text.Json;
 using System.Text.RegularExpressions;
+using Quartz.Util;
 
 namespace Overflow.SouthernWater;
 
@@ -64,9 +65,11 @@ public partial class SouthernWaterApi
 
     public async Task<PagedItems<Spill>?> GetSpills(int page = 1, JsonSerialiser? jsonSerialiser = null)
     {
+        if (baseUrl.IsNullOrWhiteSpace()) await LoadApiUrl();
+
         var request = new HttpRequestMessage()
         {
-            RequestUri = new Uri(baseUrl + spillsEndpoint),
+            RequestUri = new Uri(baseUrl + spillsEndpoint + "?page=" + page),
             Method = HttpMethod.Get,
             Headers =
             {
@@ -84,8 +87,6 @@ public partial class SouthernWaterApi
             }
         };
 
-        request.Options.TryAdd("page", page);
-
         var content = await _client.SendAsync(request);
 
         content.EnsureSuccessStatusCode();
@@ -106,11 +107,13 @@ public partial class SouthernWaterApi
 
     public async IAsyncEnumerable<PagedItems<Spill>?> GetAllSpills(TimeSpan interval, int? pageLimit = null, JsonSerialiser? jsonSerialiser = null)
     {
+        Random rnd = new Random();
+
         var firstPage = await GetSpills(page: 1, jsonSerialiser);
 
         yield return firstPage;
 
-        await Task.Delay(interval);
+        await Task.Delay(interval + TimeSpan.FromSeconds(rnd.Next(-Static.IntervalWiggleSeconds, Static.IntervalWiggleSeconds)));
 
         var pageCount = Math.Min(pageLimit ?? int.MaxValue, firstPage?.totalPages ?? 1);
 

Overflow/SouthernWater/SouthernWaterApiJobRunner.cs

@@ -10,7 +10,7 @@ public class SouthernWaterApiJobRunner(SouthernWaterApi client, ILogger<Southern
 
     public async Task<SouthernWaterApiJob> LoadSpills(int? pageLimit = null)
     {
-        var interval = TimeSpan.FromSeconds(30);
+        var interval = Static.Interval;
         var job = new SouthernWaterApiJob
         {
             StartTime = DateTime.UtcNow,

Overflow/SouthernWater/SouthernWaterJob.cs

@@ -0,0 +1,11 @@
+using Quartz;
+
+namespace Overflow.SouthernWater;
+
+public class SouthernWaterJob(SouthernWaterApiJobRunner jobRunner) : IJob
+{
+    public async Task Execute(IJobExecutionContext context)
+    {
+        await jobRunner.LoadSpills();
+    }
+}

Overflow/Static.cs

@@ -4,4 +4,7 @@ public static class Static
 {
     public static readonly string DatabaseName = "overflow";
     public static readonly string CollectionName = "southern_water_api_job";
+
+    public static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
+    public static readonly int IntervalWiggleSeconds = 10;
 }