diff --git a/Overflow.CLI/Program.cs b/Overflow.CLI/Program.cs index f841cc4..19bbb70 100644 --- a/Overflow.CLI/Program.cs +++ b/Overflow.CLI/Program.cs @@ -10,6 +10,6 @@ var driver = new MongoClient("mongodb://localhost"); var api = new SouthernWaterApi(new HttpClient()); await api.LoadApiUrl(); -var runner = new SouthernWaterApiJobRunnerPersisting(api, NullLogger.Instance, driver.GetDatabase("overflow")); +var runner = new SouthernWaterApiJobRunnerPersisting(api, NullLogger.Instance, driver.GetDatabase(Static.DatabaseName)); await runner.LoadSpills(5); \ No newline at end of file diff --git a/Overflow.Web/Overflow.Web.csproj b/Overflow.Web/Overflow.Web.csproj index 0facca4..ed76358 100644 --- a/Overflow.Web/Overflow.Web.csproj +++ b/Overflow.Web/Overflow.Web.csproj @@ -9,6 +9,9 @@ + + + @@ -27,5 +30,11 @@ <_ContentIncludedByDefault Remove="wwwroot\bootstrap\bootstrap.min.css.map" /> <_ContentIncludedByDefault Remove="wwwroot\favicon.png" /> + + + + PreserveNewest + + diff --git a/Overflow.Web/Program.cs b/Overflow.Web/Program.cs index f189b9e..3264ffd 100644 --- a/Overflow.Web/Program.cs +++ b/Overflow.Web/Program.cs @@ -3,6 +3,7 @@ using Overflow; using MongoDB.Driver; +using NLog.Extensions.Logging; using Overflow.SouthernWater; using Quartz; using Quartz.AspNetCore; @@ -14,9 +15,12 @@ builder.Services.AddRazorComponents() .AddInteractiveServerComponents() .AddInteractiveWebAssemblyComponents(); +builder.Logging.ClearProviders(); +builder.Logging.AddNLog(builder.Configuration); + var driver = new MongoClient(builder.Configuration.GetConnectionString("Default")); builder.Services.AddSingleton(driver); -builder.Services.AddScoped(s => s.GetRequiredService().GetDatabase("overflow")); +builder.Services.AddScoped(s => s.GetRequiredService().GetDatabase(Static.DatabaseName)); builder.Services.AddControllers(); @@ -33,6 +37,29 @@ builder.Services.Configure(options => builder.Services.AddQuartz(q => { // base Quartz scheduler, job and trigger configuration + + q.UseSimpleTypeLoader(); + q.UseInMemoryStore(); + q.UseDefaultThreadPool(tp => + { + tp.MaxConcurrency = 5; + }); + + var swKey = new JobKey("southern-water-api", "southern-water"); + + q.AddJob(j => j + .WithDescription("Pull spills data from Southern Water API") + .WithIdentity(swKey) + .UsingJobData("IsFull", false) + ); + + q.AddTrigger(t => t + .WithIdentity("southern-water-api-trigger") + .ForJob(swKey) + .StartNow() + .WithCronSchedule(builder.Configuration.GetSection("SouthernWater").GetValue("Cron") ?? "0 0 8 * * ?") + .WithDescription("Periodic trigger for Southern Water API pulling") + ); }); // ASP.NET Core hosting @@ -45,6 +72,7 @@ builder.Services.AddQuartzServer(options => builder.Services.AddHttpClient(); builder.Services.AddSingleton(); builder.Services.AddScoped(); +builder.Services.AddTransient(); var app = builder.Build(); diff --git a/Overflow.Web/appsettings.json b/Overflow.Web/appsettings.json index aa4a1ae..612c7fa 100644 --- a/Overflow.Web/appsettings.json +++ b/Overflow.Web/appsettings.json @@ -8,5 +8,8 @@ "ConnectionStrings": { "Default": "mongodb://localhost" }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "SouthernWater": { + "Cron": "0 26 20 * * ?" + } } diff --git a/Overflow.Web/nlog.config b/Overflow.Web/nlog.config new file mode 100644 index 0000000..37a4f2d --- /dev/null +++ b/Overflow.Web/nlog.config @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Overflow/Overflow.csproj b/Overflow/Overflow.csproj index 7b5d24f..4aa6a63 100644 --- a/Overflow/Overflow.csproj +++ b/Overflow/Overflow.csproj @@ -11,6 +11,7 @@ + diff --git a/Overflow/SouthernWater/SouthernWaterApi.cs b/Overflow/SouthernWater/SouthernWaterApi.cs index e225b8c..bec200b 100644 --- a/Overflow/SouthernWater/SouthernWaterApi.cs +++ b/Overflow/SouthernWater/SouthernWaterApi.cs @@ -1,6 +1,7 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.RegularExpressions; +using Quartz.Util; namespace Overflow.SouthernWater; @@ -64,9 +65,11 @@ public partial class SouthernWaterApi public async Task?> GetSpills(int page = 1, JsonSerialiser? jsonSerialiser = null) { + if (baseUrl.IsNullOrWhiteSpace()) await LoadApiUrl(); + var request = new HttpRequestMessage() { - RequestUri = new Uri(baseUrl + spillsEndpoint), + RequestUri = new Uri(baseUrl + spillsEndpoint + "?page=" + page), Method = HttpMethod.Get, Headers = { @@ -84,8 +87,6 @@ public partial class SouthernWaterApi } }; - request.Options.TryAdd("page", page); - var content = await _client.SendAsync(request); content.EnsureSuccessStatusCode(); @@ -106,11 +107,13 @@ public partial class SouthernWaterApi public async IAsyncEnumerable?> GetAllSpills(TimeSpan interval, int? pageLimit = null, JsonSerialiser? jsonSerialiser = null) { + Random rnd = new Random(); + var firstPage = await GetSpills(page: 1, jsonSerialiser); yield return firstPage; - await Task.Delay(interval); + await Task.Delay(interval + TimeSpan.FromSeconds(rnd.Next(-Static.IntervalWiggleSeconds, Static.IntervalWiggleSeconds))); var pageCount = Math.Min(pageLimit ?? int.MaxValue, firstPage?.totalPages ?? 1); diff --git a/Overflow/SouthernWater/SouthernWaterApiJobRunner.cs b/Overflow/SouthernWater/SouthernWaterApiJobRunner.cs index 847558e..6ebc234 100644 --- a/Overflow/SouthernWater/SouthernWaterApiJobRunner.cs +++ b/Overflow/SouthernWater/SouthernWaterApiJobRunner.cs @@ -10,7 +10,7 @@ public class SouthernWaterApiJobRunner(SouthernWaterApi client, ILogger LoadSpills(int? pageLimit = null) { - var interval = TimeSpan.FromSeconds(30); + var interval = Static.Interval; var job = new SouthernWaterApiJob { StartTime = DateTime.UtcNow, diff --git a/Overflow/SouthernWater/SouthernWaterJob.cs b/Overflow/SouthernWater/SouthernWaterJob.cs new file mode 100644 index 0000000..fc55d3b --- /dev/null +++ b/Overflow/SouthernWater/SouthernWaterJob.cs @@ -0,0 +1,11 @@ +using Quartz; + +namespace Overflow.SouthernWater; + +public class SouthernWaterJob(SouthernWaterApiJobRunner jobRunner) : IJob +{ + public async Task Execute(IJobExecutionContext context) + { + await jobRunner.LoadSpills(); + } +} \ No newline at end of file diff --git a/Overflow/Static.cs b/Overflow/Static.cs index 4b96244..e39bac7 100644 --- a/Overflow/Static.cs +++ b/Overflow/Static.cs @@ -4,4 +4,7 @@ public static class Static { public static readonly string DatabaseName = "overflow"; public static readonly string CollectionName = "southern_water_api_job"; + + public static readonly TimeSpan Interval = TimeSpan.FromSeconds(30); + public static readonly int IntervalWiggleSeconds = 10; } \ No newline at end of file