sarsoo/Overflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: sarsoo:6ce1935dea5d5e50f75d412c3d8ab6bff21b7e99
Branches Tags
sarsoo:master
..
compare: sarsoo:c337b5b6a139201afe653391f60149667b2adde3
Branches Tags
sarsoo:master

No commits in common. "6ce1935dea5d5e50f75d412c3d8ab6bff21b7e99" and "c337b5b6a139201afe653391f60149667b2adde3" have entirely different histories.
6ce1935dea ... c337b5b6a1

13 changed files with 32 additions and 139 deletions
Show Stats Expand all files Collapse all files

12
Overflow.Web/Components/Pages/Spills.razor
View File

@ -21,7 +21,15 @@
<SpillsTable Job=@job />
@code {
private SouthernWaterApiJob? job => cache.CurrentSouthernWaterApiJob;
[Inject] private SpillCache cache { get; set; }
private SouthernWaterApiJob? job;
[Inject] private IMongoDatabase database { get; set; }
// private bool showIds;
protected override async Task OnInitializedAsync()
{
job = database.GetCollection<SouthernWaterApiJob>(Static.CollectionName)
.AsQueryable()
.OrderByDescending(j => j.EndTime)
.FirstOrDefault();
}
}

12
Overflow.Web/Components/Pages/SpillsCalendarPage.razor
View File

@ -25,8 +25,16 @@
<SpillsCalendar Job="@job" GenuineOnly="@genuineOnly" />
@code {
private SouthernWaterApiJob? job => cache.CurrentSouthernWaterApiJob;
[Inject] private SpillCache cache { get; set; }
private SouthernWaterApiJob? job;
[Inject] private IMongoDatabase database { get; set; }
// private bool showIds;
private bool genuineOnly = true;
protected override async Task OnInitializedAsync()
{
job = database.GetCollection<SouthernWaterApiJob>(Static.CollectionName)
.AsQueryable()
.OrderByDescending(j => j.EndTime)
.FirstOrDefault();
}
}

20
Overflow.Web/Program.cs
View File

@ -60,22 +60,6 @@ builder.Services.AddQuartz(q =>
.WithCronSchedule(builder.Configuration.GetSection("SouthernWater").GetValue<string>("Cron") ?? "0 0 8 * * ?")
.WithDescription("Periodic trigger for Southern Water API pulling")
);
var cacheKey = new JobKey("cache-refresh", "cache");
q.AddJob<CacheReloadJob>(j => j
.WithDescription("Refresh caches")
.WithIdentity(cacheKey)
.UsingJobData("IsFull", false)
);
q.AddTrigger(t => t
.WithIdentity("cache-refresh-trigger")
.ForJob(cacheKey)
.StartNow()
.WithCronSchedule(builder.Configuration.GetSection("Cache").GetValue<string>("Cron") ?? "0 0 8 * * ?")
.WithDescription("Periodic trigger for cache refreshing")
);
});
// ASP.NET Core hosting
@ -90,10 +74,6 @@ builder.Services.AddSingleton<SouthernWaterApi>();
builder.Services.AddScoped<SouthernWaterApiJobRunner, SouthernWaterApiJobRunnerPersisting>();
builder.Services.AddTransient<SouthernWaterJob>();
builder.Services.AddSingleton<SpillCache>();
builder.Services.AddSingleton<SouthernWaterSpillCache>();
builder.Services.AddHostedService<LoadCacheOnStart>();
builder.Services.AddRadzenComponents();
var app = builder.Build();

3
Overflow.Web/appsettings.json
View File

@ -11,8 +11,5 @@
"AllowedHosts": "*",
"SouthernWater": {
"Cron": "0 26 20 * * ?"
},
"Cache": {
"Cron": "0 */3 * * * ?"
}
}

15
Overflow/CacheReloadJob.cs
View File

@ -1,15 +0,0 @@
using Microsoft.Extensions.Logging;
using Quartz;
namespace Overflow;
[DisallowConcurrentExecution]
public class CacheReloadJob(SpillCache cache, ILogger<CacheReloadJob> logger): IJob
{
public Task Execute(IJobExecutionContext context)
{
logger.LogDebug("Refreshing caches");
cache.Refresh();
return Task.CompletedTask;
}
}

2
Overflow/Overflow.csproj
View File

@ -10,8 +10,6 @@
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageReference Include="MongoDB.Driver" Version="2.25.0" />
<PackageReference Include="Quartz" Version="3.9.0" />
</ItemGroup>

2
Overflow/SouthernWater/SouthernWaterApi.cs
View File

@ -141,7 +141,7 @@ public partial class SouthernWaterApi
}
catch (TaskCanceledException e)
{
_logger.LogError(e, "HTTP Timeout Exception while loading page [{}], waiting {} before retrying", page, 1.5 * Static.Interval);
_logger.LogError(e, "HTTP Timeout Exception while loading page [{}], waiting {} before retrying", page, Static.Interval);
await Task.Delay(1.5 * Static.Interval);
}
}

1
Overflow/SouthernWater/SouthernWaterApiJob.cs
View File

@ -11,6 +11,5 @@ public class SouthernWaterApiJob
public DateTime? EndTime { get; set; }
public TimeSpan Interval { get; set; }
public int TotalItems { get; set; }
[BsonIgnore]
public List<Spill> Spills { get; set; }
}

17
Overflow/SouthernWater/SouthernWaterApiJobRunner.cs
View File

@ -36,8 +36,6 @@ public class SouthernWaterApiJobRunner(SouthernWaterApi client, ILogger<Southern
_logger.LogInformation("Processing page [{}/{}]", page.currentPage, page.totalPages);
page.items.ForEach(s => s.JobId = job._id);
job.TotalItems = page.totalItems;
job.Spills.AddRange(page.items);
try
@ -88,17 +86,22 @@ public class SouthernWaterApiJobRunnerPersisting(
IMongoDatabase mongo)
: SouthernWaterApiJobRunner(client, logger)
{
private readonly IMongoCollection<SouthernWaterApiJob> _jobCollection = mongo.GetCollection<SouthernWaterApiJob>(Static.JobCollectionName);
private readonly IMongoCollection<Spill> _spillCollection = mongo.GetCollection<Spill>(Static.SpillCollectionName);
private readonly IMongoCollection<SouthernWaterApiJob> _collection = mongo.GetCollection<SouthernWaterApiJob>(Static.CollectionName);
protected override async Task JobCreated(SouthernWaterApiJob job)
{
await _jobCollection.InsertOneAsync(job);
await _collection.InsertOneAsync(job);
}
protected override async Task PageLoaded(SouthernWaterApiJob job, PagedItems<Spill> page)
{
await _spillCollection.InsertManyAsync(page.items);
var finder = Builders<SouthernWaterApiJob>.Filter
.Eq(j => j._id, job._id);
var update = Builders<SouthernWaterApiJob>.Update
.PushEach(j => j.Spills, page.items);
await _collection.UpdateOneAsync(finder, update);
}
protected override async Task JobFinished(SouthernWaterApiJob job)
@ -110,6 +113,6 @@ public class SouthernWaterApiJobRunnerPersisting(
.Set(j => j.EndTime, job.EndTime)
.Set(j => j.TotalItems, job.TotalItems);
await _jobCollection.UpdateOneAsync(finder, update);
await _collection.UpdateOneAsync(finder, update);
}
}

44
Overflow/SouthernWater/SouthernWaterSpillCache.cs
View File

@ -1,44 +0,0 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
namespace Overflow.SouthernWater;
public class SouthernWaterSpillCache(IServiceProvider serviceProvider, ILogger<SouthernWaterSpillCache> logger)
{
private readonly ILogger _logger = logger;
private SouthernWaterApiJob _currentJob;
public SouthernWaterApiJob CurrentJob => _currentJob;
public void ReloadJob()
{
_logger.LogInformation("Refreshing Southern Water Spills");
using var scope = serviceProvider.CreateScope();
var database = scope.ServiceProvider.GetRequiredService<IMongoDatabase>();
var job = database.GetCollection<SouthernWaterApiJob>(Static.JobCollectionName)
.AsQueryable()
.OrderByDescending(j => j.EndTime)
.FirstOrDefault();
if (job is not null)
{
job.Spills = database.GetCollection<Spill>(Static.SpillCollectionName)
.AsQueryable()
.Where(s => s.JobId == job._id)
.ToList();
_currentJob = job;
_logger.LogInformation("Southern Water Spills cache refreshed");
}
else
{
_logger.LogWarning("No Southern Water Spills returned");
}
}
}

5
Overflow/SouthernWater/Spill.cs
View File

@ -6,11 +6,6 @@ namespace Overflow.SouthernWater;
public class Spill
{
[JsonIgnore]
public ObjectId _id { get; set; }
[JsonIgnore]
public ObjectId JobId { get; set; }
[JsonPropertyName("id")]
public int sw_id { get; set; }
public int eventId { get; set; }

35
Overflow/SpillCache.cs
View File

@ -1,35 +0,0 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Overflow.SouthernWater;
using Quartz;
namespace Overflow;
public class SpillCache(SouthernWaterSpillCache southernWaterSpillCache, ILogger<SpillCache> logger)
{
public SouthernWaterApiJob CurrentSouthernWaterApiJob => southernWaterSpillCache.CurrentJob;
public void Refresh()
{
logger.LogDebug("Refreshing caches");
southernWaterSpillCache.ReloadJob();
}
}
public class LoadCacheOnStart(ISchedulerFactory scheduler, SpillCache cache, ILogger<LoadCacheOnStart> logger) : IHostedService
{
public async Task StartAsync(CancellationToken cancellationToken)
{
logger.LogInformation("Loading caches for startup");
await (await scheduler.GetScheduler()).TriggerJob(new JobKey("cache-refresh", "cache"));
// cache.Refresh();
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}

3
Overflow/Static.cs
View File

@ -3,8 +3,7 @@ namespace Overflow;
public static class Static
{
public static readonly string DatabaseName = "overflow";
public static readonly string JobCollectionName = "southern_water_api_job";
public static readonly string SpillCollectionName = "southern_water_spills";
public static readonly string CollectionName = "southern_water_api_job";
public static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
public static readonly int IntervalWiggleSeconds = 10;