From bf55f7222e06e3001ef49b648e0a4383a014038e Mon Sep 17 00:00:00 2001 From: andy Date: Fri, 25 Feb 2022 21:07:22 +0000 Subject: [PATCH] adding scrobble watcher job fixes #32 --- Selector.CLI/Command/HostCommand.cs | 11 +-- Selector.CLI/Extensions/ServiceExtensions.cs | 47 +++++++++++ Selector.CLI/Jobs/ScrobbleWatcherJob.cs | 88 ++++++++++++++++++++ Selector.CLI/Options.cs | 23 +++-- Selector.CLI/Selector.CLI.csproj | 2 + Selector.CLI/Services/ScrobbleMonitor.cs | 63 -------------- 6 files changed, 157 insertions(+), 77 deletions(-) create mode 100644 Selector.CLI/Jobs/ScrobbleWatcherJob.cs delete mode 100644 Selector.CLI/Services/ScrobbleMonitor.cs diff --git a/Selector.CLI/Command/HostCommand.cs b/Selector.CLI/Command/HostCommand.cs index 58d086b..3e1ade3 100644 --- a/Selector.CLI/Command/HostCommand.cs +++ b/Selector.CLI/Command/HostCommand.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.Logging; using NLog.Extensions.Logging; using Selector.Cache.Extensions; using Selector.CLI.Extensions; -using Selector.CLI.Services; using Selector.Events; using Selector.Extensions; using System; @@ -89,7 +88,8 @@ namespace Selector.CLI .AddSpotify(); services.ConfigureLastFm(config) - .ConfigureEqual(config); + .ConfigureEqual(config) + .ConfigureJobs(config); if (config.RedisOptions.Enabled) { @@ -119,13 +119,6 @@ namespace Selector.CLI services.AddHostedService(); } } - - if (config.ScrobbleOptions.Enabled) - { - Console.WriteLine("> Adding Scrobble Monitor Service"); - - services.AddHostedService(); - } } public static void ConfigureDefaultNlog(HostBuilderContext context, ILoggingBuilder builder) diff --git a/Selector.CLI/Extensions/ServiceExtensions.cs b/Selector.CLI/Extensions/ServiceExtensions.cs index 3fbf4cf..1f8a508 100644 --- a/Selector.CLI/Extensions/ServiceExtensions.cs +++ b/Selector.CLI/Extensions/ServiceExtensions.cs @@ -1,6 +1,8 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Quartz; using Selector.Cache.Extensions; +using Selector.CLI.Jobs; using Selector.Extensions; using Selector.Model; using Selector.Model.Services; @@ -36,6 +38,51 @@ namespace Selector.CLI.Extensions return services; } + public static IServiceCollection ConfigureJobs(this IServiceCollection services, RootOptions config) + { + if (config.JobOptions.Enabled) + { + Console.WriteLine("> Adding Jobs..."); + + services.AddQuartz(options => { + + options.UseMicrosoftDependencyInjectionJobFactory(); + + options.UseSimpleTypeLoader(); + options.UseInMemoryStore(); + options.UseDefaultThreadPool(tp => + { + tp.MaxConcurrency = 5; + }); + + var scrobbleKey = new JobKey("scrobble-watcher", "scrobble"); + + options.AddJob(j => j + .WithDescription("Watch recent scrobbles and mirror to database") + .WithIdentity(scrobbleKey) + ); + + options.AddTrigger(t => t + .WithIdentity("scrobble-watcher-trigger") + .ForJob(scrobbleKey) + .StartNow() + .WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever()) + .WithDescription("Periodic trigger for scrobble watcher") + ); + }); + + services.AddQuartzHostedService(options =>{ + + options.WaitForJobsToComplete = true; + }); + + services.AddTransient(); + services.AddTransient(); + } + + return services; + } + public static IServiceCollection ConfigureDb(this IServiceCollection services, RootOptions config) { if (config.DatabaseOptions.Enabled) diff --git a/Selector.CLI/Jobs/ScrobbleWatcherJob.cs b/Selector.CLI/Jobs/ScrobbleWatcherJob.cs new file mode 100644 index 0000000..1feb531 --- /dev/null +++ b/Selector.CLI/Jobs/ScrobbleWatcherJob.cs @@ -0,0 +1,88 @@ +using IF.Lastfm.Core.Api; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Quartz; +using Selector.Model; +using Selector.Model.Extensions; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Selector.CLI.Jobs +{ + public class ScrobbleWatcherJob : IJob + { + private readonly ILogger logger; + private readonly ILoggerFactory loggerFactory; + + private readonly IUserApi userApi; + private readonly IScrobbleRepository scrobbleRepo; + private readonly ApplicationDbContext db; + private readonly ScrobbleWatcherJobOptions options; + + public ScrobbleWatcherJob( + IUserApi _userApi, + IScrobbleRepository _scrobbleRepo, + ApplicationDbContext _db, + IOptions _options, + ILogger _logger, + ILoggerFactory _loggerFactory) + { + logger = _logger; + loggerFactory = _loggerFactory; + + userApi = _userApi; + scrobbleRepo = _scrobbleRepo; + db = _db; + options = _options.Value; + } + + public async Task Execute(IJobExecutionContext context) + { + logger.LogInformation("Starting scrobble watching job"); + + var users = db.Users + .AsEnumerable() + .Where(u => u.ScrobbleSavingEnabled()) + .ToArray(); + + foreach (var user in users) + { + logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername); + + if (options.From is not null && options.From.Value.Kind != DateTimeKind.Utc) + { + options.From = options.From.Value.ToUniversalTime(); + } + + if (options.To is not null && options.To.Value.Kind != DateTimeKind.Utc) + { + options.To = options.To.Value.ToUniversalTime(); + } + + var saver = new ScrobbleSaver( + userApi, + new() + { + User = user, + InterRequestDelay = options.InterRequestDelay, + From = options.From, + To = options.To, + PageSize = options.PageSize, + DontAdd = false, + DontRemove = false, + SimultaneousConnections = options.Simultaneous + }, + scrobbleRepo, + loggerFactory.CreateLogger(), + loggerFactory); + + await saver.Execute(context.CancellationToken); + + logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername); + } + } + } +} diff --git a/Selector.CLI/Options.cs b/Selector.CLI/Options.cs index c73f596..f60d13f 100644 --- a/Selector.CLI/Options.cs +++ b/Selector.CLI/Options.cs @@ -11,7 +11,8 @@ namespace Selector.CLI config.GetSection(FormatKeys( new[] { RootOptions.Key, WatcherOptions.Key})).Bind(options.WatcherOptions); config.GetSection(FormatKeys( new[] { RootOptions.Key, DatabaseOptions.Key})).Bind(options.DatabaseOptions); config.GetSection(FormatKeys( new[] { RootOptions.Key, RedisOptions.Key})).Bind(options.RedisOptions); - config.GetSection(FormatKeys( new[] { RootOptions.Key, ScrobbleMonitorOptions.Key})).Bind(options.ScrobbleOptions); + config.GetSection(FormatKeys( new[] { RootOptions.Key, JobsOptions.Key})).Bind(options.JobOptions); + config.GetSection(FormatKeys( new[] { RootOptions.Key, JobsOptions.Key, ScrobbleWatcherJobOptions.Key})).Bind(options.JobOptions.Scrobble); } public static RootOptions ConfigureOptions(this IConfiguration config) @@ -43,7 +44,7 @@ namespace Selector.CLI public string LastfmClient { get; set; } public string LastfmSecret { get; set; } public WatcherOptions WatcherOptions { get; set; } = new(); - public ScrobbleMonitorOptions ScrobbleOptions { get; set; } = new(); + public JobsOptions JobOptions { get; set; } = new(); public DatabaseOptions DatabaseOptions { get; set; } = new(); public RedisOptions RedisOptions { get; set; } = new(); public EqualityChecker Equality { get; set; } = EqualityChecker.Uri; @@ -93,11 +94,23 @@ namespace Selector.CLI public string ConnectionString { get; set; } } - public class ScrobbleMonitorOptions + public class JobsOptions + { + public const string Key = "Job"; + + public bool Enabled { get; set; } = true; + public ScrobbleWatcherJobOptions Scrobble { get; set; } = new(); + } + + public class ScrobbleWatcherJobOptions { public const string Key = "Scrobble"; - public bool Enabled { get; set; } = true; - public TimeSpan InterRequestDelay { get; set; } = new(0, 0, 0, 1, 0); + public TimeSpan InterJobDelay { get; set; } = TimeSpan.FromMinutes(5); + public TimeSpan InterRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100); + public DateTime? From { get; set; } = DateTime.UtcNow.AddDays(-14); + public DateTime? To { get; set; } + public int PageSize { get; set; } = 200; + public int Simultaneous { get; set; } = 3; } } diff --git a/Selector.CLI/Selector.CLI.csproj b/Selector.CLI/Selector.CLI.csproj index 2fa9d22..9514021 100644 --- a/Selector.CLI/Selector.CLI.csproj +++ b/Selector.CLI/Selector.CLI.csproj @@ -20,6 +20,8 @@ + + diff --git a/Selector.CLI/Services/ScrobbleMonitor.cs b/Selector.CLI/Services/ScrobbleMonitor.cs deleted file mode 100644 index 4351909..0000000 --- a/Selector.CLI/Services/ScrobbleMonitor.cs +++ /dev/null @@ -1,63 +0,0 @@ -using IF.Lastfm.Core.Api; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Selector.Model; -using Selector.Model.Extensions; -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace Selector.CLI.Services -{ - public class ScrobbleMonitor : IHostedService - { - private readonly ILogger logger; - private readonly ILoggerFactory loggerFactory; - private readonly ScrobbleMonitorOptions config; - private readonly IUserApi userApi; - private readonly IServiceScopeFactory serviceScopeFactory; - - public ScrobbleMonitor(ILogger _logger, IOptions _options, IUserApi _userApi, IServiceScopeFactory _serviceScopeFactory, ILoggerFactory _loggerFactory) - { - logger = _logger; - userApi = _userApi; - config = _options.Value; - serviceScopeFactory = _serviceScopeFactory; - loggerFactory = _loggerFactory; - } - - public async Task StartAsync(CancellationToken cancellationToken) - { - logger.LogInformation("Starting scrobble monitor"); - - using var scope = serviceScopeFactory.CreateScope(); - using var db = scope.ServiceProvider.GetRequiredService(); - - await RunScrobbleSavers(db, cancellationToken); - } - - public Task RunScrobbleSavers(ApplicationDbContext db, CancellationToken token) - { - using var scope = serviceScopeFactory.CreateScope(); - - foreach (var user in db.Users - .AsNoTracking() - .AsEnumerable() - .Where(u => u.ScrobbleSavingEnabled())) - { - //TODO - } - - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - } -}