adding quartz job, interval wiggle, fixed page parameters

This commit is contained in:
Andy Pack 2024-06-09 21:12:45 +01:00
parent fe75c94cd2
commit ea631960c5
Signed by: sarsoo
GPG Key ID: A55BA3536A5E0ED7
10 changed files with 119 additions and 8 deletions

View File

@ -10,6 +10,6 @@ var driver = new MongoClient("mongodb://localhost");
var api = new SouthernWaterApi(new HttpClient()); var api = new SouthernWaterApi(new HttpClient());
await api.LoadApiUrl(); await api.LoadApiUrl();
var runner = new SouthernWaterApiJobRunnerPersisting(api, NullLogger<SouthernWaterApiJobRunner>.Instance, driver.GetDatabase("overflow")); var runner = new SouthernWaterApiJobRunnerPersisting(api, NullLogger<SouthernWaterApiJobRunner>.Instance, driver.GetDatabase(Static.DatabaseName));
await runner.LoadSpills(5); await runner.LoadSpills(5);

View File

@ -9,6 +9,9 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="8.0.6" /> <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="8.0.6" />
<PackageReference Include="MongoDB.Driver" Version="2.25.0" /> <PackageReference Include="MongoDB.Driver" Version="2.25.0" />
<PackageReference Include="NLog" Version="5.3.2" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.3.11" />
<PackageReference Include="NLog.Web.AspNetCore" Version="5.3.11" />
<PackageReference Include="Quartz" Version="3.9.0" /> <PackageReference Include="Quartz" Version="3.9.0" />
<PackageReference Include="Quartz.AspNetCore" Version="3.9.0" /> <PackageReference Include="Quartz.AspNetCore" Version="3.9.0" />
<PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.9.0" /> <PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.9.0" />
@ -27,5 +30,11 @@
<_ContentIncludedByDefault Remove="wwwroot\bootstrap\bootstrap.min.css.map" /> <_ContentIncludedByDefault Remove="wwwroot\bootstrap\bootstrap.min.css.map" />
<_ContentIncludedByDefault Remove="wwwroot\favicon.png" /> <_ContentIncludedByDefault Remove="wwwroot\favicon.png" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<None Update="nlog.config">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project> </Project>

View File

@ -3,6 +3,7 @@ using Overflow;
using MongoDB.Driver; using MongoDB.Driver;
using NLog.Extensions.Logging;
using Overflow.SouthernWater; using Overflow.SouthernWater;
using Quartz; using Quartz;
using Quartz.AspNetCore; using Quartz.AspNetCore;
@ -14,9 +15,12 @@ builder.Services.AddRazorComponents()
.AddInteractiveServerComponents() .AddInteractiveServerComponents()
.AddInteractiveWebAssemblyComponents(); .AddInteractiveWebAssemblyComponents();
builder.Logging.ClearProviders();
builder.Logging.AddNLog(builder.Configuration);
var driver = new MongoClient(builder.Configuration.GetConnectionString("Default")); var driver = new MongoClient(builder.Configuration.GetConnectionString("Default"));
builder.Services.AddSingleton(driver); builder.Services.AddSingleton(driver);
builder.Services.AddScoped<IMongoDatabase>(s => s.GetRequiredService<MongoClient>().GetDatabase("overflow")); builder.Services.AddScoped<IMongoDatabase>(s => s.GetRequiredService<MongoClient>().GetDatabase(Static.DatabaseName));
builder.Services.AddControllers(); builder.Services.AddControllers();
@ -33,6 +37,29 @@ builder.Services.Configure<QuartzOptions>(options =>
builder.Services.AddQuartz(q => builder.Services.AddQuartz(q =>
{ {
// base Quartz scheduler, job and trigger configuration // base Quartz scheduler, job and trigger configuration
q.UseSimpleTypeLoader();
q.UseInMemoryStore();
q.UseDefaultThreadPool(tp =>
{
tp.MaxConcurrency = 5;
});
var swKey = new JobKey("southern-water-api", "southern-water");
q.AddJob<SouthernWaterJob>(j => j
.WithDescription("Pull spills data from Southern Water API")
.WithIdentity(swKey)
.UsingJobData("IsFull", false)
);
q.AddTrigger(t => t
.WithIdentity("southern-water-api-trigger")
.ForJob(swKey)
.StartNow()
.WithCronSchedule(builder.Configuration.GetSection("SouthernWater").GetValue<string>("Cron") ?? "0 0 8 * * ?")
.WithDescription("Periodic trigger for Southern Water API pulling")
);
}); });
// ASP.NET Core hosting // ASP.NET Core hosting
@ -45,6 +72,7 @@ builder.Services.AddQuartzServer(options =>
builder.Services.AddHttpClient(); builder.Services.AddHttpClient();
builder.Services.AddSingleton<SouthernWaterApi>(); builder.Services.AddSingleton<SouthernWaterApi>();
builder.Services.AddScoped<SouthernWaterApiJobRunner, SouthernWaterApiJobRunnerPersisting>(); builder.Services.AddScoped<SouthernWaterApiJobRunner, SouthernWaterApiJobRunnerPersisting>();
builder.Services.AddTransient<SouthernWaterJob>();
var app = builder.Build(); var app = builder.Build();

View File

@ -8,5 +8,8 @@
"ConnectionStrings": { "ConnectionStrings": {
"Default": "mongodb://localhost" "Default": "mongodb://localhost"
}, },
"AllowedHosts": "*" "AllowedHosts": "*",
"SouthernWater": {
"Cron": "0 26 20 * * ?"
}
} }

53
Overflow.Web/nlog.config Normal file
View File

@ -0,0 +1,53 @@
<?xml version="1.0" encoding="utf-8" ?>
<!-- XSD manual extracted from package NLog.Schema: https://www.nuget.org/packages/NLog.Schema-->
<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd" xsi:schemaLocation="NLog NLog.xsd"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
autoReload="true"
throwConfigExceptions="true"
internalLogFile="./log/overflow.nlog.log"
internalLogLevel="Info" >
<extensions>
<add assembly="NLog.Web.AspNetCore"/>
</extensions>
<variable name="scopeFormat"
value="${all-event-properties:format=[[key]\:[value]]:includeScopeProperties=true:separator= }"/>
<variable name="format"
value="${longdate}|${level:uppercase=true}|${callsite}:${callsite-linenumber}|${message}${onexception:inner=${newline}}${exception:format=tostring,data:exceptionDataSeparator=\r\n}${newline} ${scopeFormat}"/>
<!-- the targets to write to -->
<targets>
<!-- write logs to file -->
<target xsi:type="File"
name="logfile"
fileName="./log/overflow-${shortdate}.log"
layout="${format}" />
<target xsi:type="File"
name="tracefile"
fileName="./log/overflow.trace-${shortdate}.log"
layout="${format}" />
<target xsi:type="ColoredConsole"
name="logconsole"
layout="${format}" />
</targets>
<!-- rules to map from logger name to target -->
<rules>
<!--<logger name="*" minlevel="Trace" writeTo="tracefile" />-->
<logger name="Overflow.*" minlevel="Debug" writeTo="logconsole" />
<!--Output hosting lifetime messages to console target for faster startup detection -->
<logger name="Microsoft.Hosting.Lifetime" minlevel="Info" writeTo="logconsole" final="true" />
<!--Skip non-critical Microsoft logs and so log only own logs (BlackHole) -->
<logger name="Microsoft.*" maxlevel="Info" final="true" />
<logger name="System.Net.Http.*" maxlevel="Info" final="true" />
<!--<logger name="*" minlevel="Debug" writeTo="logfile" />-->
<logger name="Overflow.*" minlevel="Info" writeTo="logfile" />
<logger name="Quartz.*" minlevel="Info" writeTo="logfile" />
<logger name="Microsoft.*" minlevel="Warning" writeTo="logfile" />
</rules>
</nlog>

View File

@ -11,6 +11,7 @@
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="MongoDB.Driver" Version="2.25.0" /> <PackageReference Include="MongoDB.Driver" Version="2.25.0" />
<PackageReference Include="Quartz" Version="3.9.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

View File

@ -1,6 +1,7 @@
using System.Net.Http.Json; using System.Net.Http.Json;
using System.Text.Json; using System.Text.Json;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Quartz.Util;
namespace Overflow.SouthernWater; namespace Overflow.SouthernWater;
@ -64,9 +65,11 @@ public partial class SouthernWaterApi
public async Task<PagedItems<Spill>?> GetSpills(int page = 1, JsonSerialiser? jsonSerialiser = null) public async Task<PagedItems<Spill>?> GetSpills(int page = 1, JsonSerialiser? jsonSerialiser = null)
{ {
if (baseUrl.IsNullOrWhiteSpace()) await LoadApiUrl();
var request = new HttpRequestMessage() var request = new HttpRequestMessage()
{ {
RequestUri = new Uri(baseUrl + spillsEndpoint), RequestUri = new Uri(baseUrl + spillsEndpoint + "?page=" + page),
Method = HttpMethod.Get, Method = HttpMethod.Get,
Headers = Headers =
{ {
@ -84,8 +87,6 @@ public partial class SouthernWaterApi
} }
}; };
request.Options.TryAdd("page", page);
var content = await _client.SendAsync(request); var content = await _client.SendAsync(request);
content.EnsureSuccessStatusCode(); content.EnsureSuccessStatusCode();
@ -106,11 +107,13 @@ public partial class SouthernWaterApi
public async IAsyncEnumerable<PagedItems<Spill>?> GetAllSpills(TimeSpan interval, int? pageLimit = null, JsonSerialiser? jsonSerialiser = null) public async IAsyncEnumerable<PagedItems<Spill>?> GetAllSpills(TimeSpan interval, int? pageLimit = null, JsonSerialiser? jsonSerialiser = null)
{ {
Random rnd = new Random();
var firstPage = await GetSpills(page: 1, jsonSerialiser); var firstPage = await GetSpills(page: 1, jsonSerialiser);
yield return firstPage; yield return firstPage;
await Task.Delay(interval); await Task.Delay(interval + TimeSpan.FromSeconds(rnd.Next(-Static.IntervalWiggleSeconds, Static.IntervalWiggleSeconds)));
var pageCount = Math.Min(pageLimit ?? int.MaxValue, firstPage?.totalPages ?? 1); var pageCount = Math.Min(pageLimit ?? int.MaxValue, firstPage?.totalPages ?? 1);

View File

@ -10,7 +10,7 @@ public class SouthernWaterApiJobRunner(SouthernWaterApi client, ILogger<Southern
public async Task<SouthernWaterApiJob> LoadSpills(int? pageLimit = null) public async Task<SouthernWaterApiJob> LoadSpills(int? pageLimit = null)
{ {
var interval = TimeSpan.FromSeconds(30); var interval = Static.Interval;
var job = new SouthernWaterApiJob var job = new SouthernWaterApiJob
{ {
StartTime = DateTime.UtcNow, StartTime = DateTime.UtcNow,

View File

@ -0,0 +1,11 @@
using Quartz;
namespace Overflow.SouthernWater;
public class SouthernWaterJob(SouthernWaterApiJobRunner jobRunner) : IJob
{
public async Task Execute(IJobExecutionContext context)
{
await jobRunner.LoadSpills();
}
}

View File

@ -4,4 +4,7 @@ public static class Static
{ {
public static readonly string DatabaseName = "overflow"; public static readonly string DatabaseName = "overflow";
public static readonly string CollectionName = "southern_water_api_job"; public static readonly string CollectionName = "southern_water_api_job";
public static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
public static readonly int IntervalWiggleSeconds = 10;
} }