diff --git a/Selector.CLI/Extensions/ServiceExtensions.cs b/Selector.CLI/Extensions/ServiceExtensions.cs index 489d917..a3b029d 100644 --- a/Selector.CLI/Extensions/ServiceExtensions.cs +++ b/Selector.CLI/Extensions/ServiceExtensions.cs @@ -55,23 +55,48 @@ namespace Selector.CLI.Extensions tp.MaxConcurrency = 5; }); - var scrobbleKey = new JobKey("scrobble-watcher", "scrobble"); + if (config.JobOptions.Scrobble.Enabled) + { + Console.WriteLine("> Adding Scrobble Jobs..."); - options.AddJob(j => j - .WithDescription("Watch recent scrobbles and mirror to database") - .WithIdentity(scrobbleKey) - ); + var scrobbleKey = new JobKey("scrobble-watcher-agile", "scrobble"); - options.AddTrigger(t => t - .WithIdentity("scrobble-watcher-trigger") - .ForJob(scrobbleKey) - .StartNow() - .WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever()) - .WithDescription("Periodic trigger for scrobble watcher") - ); + options.AddJob(j => j + .WithDescription("Watch recent scrobbles and mirror to database") + .WithIdentity(scrobbleKey) + .UsingJobData("IsFull", false) + ); + + options.AddTrigger(t => t + .WithIdentity("scrobble-watcher-agile-trigger") + .ForJob(scrobbleKey) + .StartNow() + .WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever()) + .WithDescription("Periodic trigger for scrobble watcher") + ); + + var fullScrobbleKey = new JobKey("scrobble-watcher-full", "scrobble"); + + options.AddJob(j => j + .WithDescription("Check all scrobbles and mirror to database") + .WithIdentity(fullScrobbleKey) + .UsingJobData("IsFull", true) + ); + + options.AddTrigger(t => t + .WithIdentity("scrobble-watcher-full-trigger") + .ForJob(fullScrobbleKey) + .WithCronSchedule(config.JobOptions.Scrobble.FullScrobbleCron) + .WithDescription("Periodic trigger for scrobble watcher") + ); + } + else + { + Console.WriteLine("> Skipping Scrobble Jobs..."); + } }); - services.AddQuartzHostedService(options =>{ + services.AddQuartzHostedService(options => { options.WaitForJobsToComplete = true; }); diff --git a/Selector.CLI/Jobs/ScrobbleWatcherJob.cs b/Selector.CLI/Jobs/ScrobbleWatcherJob.cs index 6c58783..dbab2f3 100644 --- a/Selector.CLI/Jobs/ScrobbleWatcherJob.cs +++ b/Selector.CLI/Jobs/ScrobbleWatcherJob.cs @@ -23,6 +23,10 @@ namespace Selector.CLI.Jobs private readonly ApplicationDbContext db; private readonly ScrobbleWatcherJobOptions options; + private static object databaseLock = new(); + + public bool IsFull { get; set; } + public ScrobbleWatcherJob( IUserApi _userApi, IScrobbleRepository _scrobbleRepo, @@ -42,47 +46,50 @@ namespace Selector.CLI.Jobs public async Task Execute(IJobExecutionContext context) { - logger.LogInformation("Starting scrobble watching job"); - - var users = db.Users - .AsEnumerable() - .Where(u => u.ScrobbleSavingEnabled()) - .ToArray(); - - foreach (var user in users) + try { - logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername); + logger.LogInformation("Starting scrobble watching job"); - if (options.From is not null && options.From.Value.Kind != DateTimeKind.Utc) + var users = db.Users + .AsEnumerable() + .Where(u => u.ScrobbleSavingEnabled()) + .ToArray(); + + foreach (var user in users) { - options.From = options.From.Value.ToUniversalTime(); - } + logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername); - if (options.To is not null && options.To.Value.Kind != DateTimeKind.Utc) - { - options.To = options.To.Value.ToUniversalTime(); - } - - var saver = new ScrobbleSaver( - userApi, - new() + DateTime? from = null; + if (options.From is not null && !IsFull) { - User = user, - InterRequestDelay = options.InterRequestDelay, - From = options.From, - To = options.To, - PageSize = options.PageSize, - DontAdd = false, - DontRemove = false, - SimultaneousConnections = options.Simultaneous - }, - scrobbleRepo, - loggerFactory.CreateLogger(), - loggerFactory); + from = options.From.Value.ToUniversalTime(); + } - await saver.Execute(context.CancellationToken); + var saver = new ScrobbleSaver( + userApi, + new() + { + User = user, + InterRequestDelay = options.InterRequestDelay, + From = from, + PageSize = options.PageSize, + DontAdd = false, + DontRemove = false, + SimultaneousConnections = options.Simultaneous + }, + scrobbleRepo, + loggerFactory.CreateLogger(), + loggerFactory, + databaseLock); - logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername); + await saver.Execute(context.CancellationToken); + + logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Error occured while saving scrobbles"); } } } diff --git a/Selector.CLI/Options.cs b/Selector.CLI/Options.cs index 781ed96..9d2b969 100644 --- a/Selector.CLI/Options.cs +++ b/Selector.CLI/Options.cs @@ -124,10 +124,10 @@ namespace Selector.CLI public const string Key = "Scrobble"; public bool Enabled { get; set; } = true; + public string FullScrobbleCron { get; set; } = "0 2 * * * ?"; public TimeSpan InterJobDelay { get; set; } = TimeSpan.FromMinutes(5); public TimeSpan InterRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100); public DateTime? From { get; set; } = DateTime.UtcNow.AddDays(-14); - public DateTime? To { get; set; } public int PageSize { get; set; } = 200; public int Simultaneous { get; set; } = 3; } diff --git a/Selector.CLI/ScrobbleSaver.cs b/Selector.CLI/ScrobbleSaver.cs index 21cd453..2c0c652 100644 --- a/Selector.CLI/ScrobbleSaver.cs +++ b/Selector.CLI/ScrobbleSaver.cs @@ -36,18 +36,21 @@ namespace Selector private readonly IUserApi userClient; private readonly ScrobbleSaverConfig config; - private Task batchTask; private BatchingOperation batchOperation; private readonly IScrobbleRepository scrobbleRepo; - public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger _logger, ILoggerFactory _loggerFactory = null) + private readonly object dbLock; + + public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger _logger, ILoggerFactory _loggerFactory = null, object _dbLock = null) { userClient = _userClient; config = _config; scrobbleRepo = _scrobbleRepository; logger = _logger; loggerFactory = _loggerFactory; + + dbLock = _dbLock ?? new(); } public async Task Execute(CancellationToken token) @@ -74,17 +77,7 @@ namespace Selector GetRequestsFromPageNumbers(2, page1.TotalPages) ); - batchTask = batchOperation.TriggerRequests(token); - } - - - logger.LogDebug("Pulling currently stored scrobbles"); - - var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To); - - if(batchTask is not null) - { - await batchTask; + await batchOperation.TriggerRequests(token); } IEnumerable scrobbles; @@ -108,50 +101,57 @@ namespace Selector .Select(s => (UserScrobble) s) .ToArray(); - logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length); + logger.LogInformation("Completed network scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length); - logger.LogDebug("Identifying difference sets"); - var time = Stopwatch.StartNew(); + logger.LogDebug("Pulling currently stored scrobbles"); - (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles); - - time.Stop(); - logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds); - - var timeDbOps = Stopwatch.StartNew(); - - if(!config.DontAdd) + lock (dbLock) { - foreach(var add in toAdd) + var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To); + + logger.LogDebug("Identifying difference sets"); + var time = Stopwatch.StartNew(); + + (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles); + + time.Stop(); + logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds); + + var timeDbOps = Stopwatch.StartNew(); + + if (!config.DontAdd) { - var scrobble = (UserScrobble) add; - scrobble.UserId = config.User.Id; - scrobbleRepo.Add(scrobble); + foreach (var add in toAdd) + { + var scrobble = (UserScrobble)add; + scrobble.UserId = config.User.Id; + scrobbleRepo.Add(scrobble); + } } - } - else - { - logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count()); - } - if (!config.DontRemove) - { - foreach (var remove in toRemove) + else { - var scrobble = (UserScrobble) remove; - scrobble.UserId = config.User.Id; - scrobbleRepo.Remove(scrobble); + logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count()); } - } - else - { - logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count()); - } - await scrobbleRepo.Save(); + if (!config.DontRemove) + { + foreach (var remove in toRemove) + { + var scrobble = (UserScrobble)remove; + scrobble.UserId = config.User.Id; + scrobbleRepo.Remove(scrobble); + } + } + else + { + logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count()); + } + _ = scrobbleRepo.Save().Result; - timeDbOps.Stop(); - logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds); + timeDbOps.Stop(); + logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds); - logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count()); + logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count()); + } } else { diff --git a/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml b/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml index 97b373d..5723384 100644 --- a/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml +++ b/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml @@ -16,7 +16,12 @@ - +
+ + + +
+ diff --git a/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml.cs b/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml.cs index 64f3dd9..5c851f2 100644 --- a/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml.cs +++ b/Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml.cs @@ -13,6 +13,7 @@ using Microsoft.AspNetCore.WebUtilities; using Selector.Model; using Selector.Events; +using Microsoft.Extensions.Logging; namespace Selector.Web.Areas.Identity.Pages.Account.Manage { @@ -21,12 +22,17 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage private readonly UserManager _userManager; private readonly UserEventBus UserEvent; + private readonly ILogger logger; + public LastFmModel( UserManager userManager, - UserEventBus userEvent) + UserEventBus userEvent, + ILogger _logger) { _userManager = userManager; UserEvent = userEvent; + + logger = _logger; } [TempData] @@ -39,6 +45,9 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage { [Display(Name = "Username")] public string Username { get; set; } + + [Display(Name = "Scrobble Saving")] + public bool ScrobbleSaving { get; set; } } private Task LoadAsync(ApplicationUser user) @@ -46,6 +55,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage Input = new InputModel { Username = user.LastFmUsername, + ScrobbleSaving = user.SaveScrobbles }; return Task.CompletedTask; @@ -63,7 +73,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage return Page(); } - public async Task OnPostChangeUsernameAsync() + public async Task OnPostSaveAsync() { var user = await _userManager.GetUserAsync(User); if (user == null) @@ -77,23 +87,54 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage return Page(); } + var changed = false; + if (Input.Username != user.LastFmUsername) { var oldUsername = user.LastFmUsername; user.LastFmUsername = Input.Username?.Trim(); - - await _userManager.UpdateAsync(user); + + changed = true; + UserEvent.OnLastfmCredChange(this, new LastfmChange { UserId = user.Id, PreviousUsername = oldUsername, NewUsername = user.LastFmUsername }); + logger.LogInformation("Changing username from {} to {}", oldUsername, user.LastFmUsername); + StatusMessage = "Username changed"; - return RedirectToPage(); } - StatusMessage = "Username unchanged"; + if (Input.ScrobbleSaving != user.SaveScrobbles) + { + user.SaveScrobbles = Input.ScrobbleSaving; + + logger.LogInformation("Changing scrobble saving from {} to {}", !Input.ScrobbleSaving, Input.ScrobbleSaving); + + if (changed) + { + StatusMessage += ", scrobble saving updated"; + } + else + { + StatusMessage = "Scrobble saving updated"; + changed = true; + } + } + + if (changed) + { + logger.LogInformation("Saving Last.fm settings for {}", user.LastFmUsername); + + await _userManager.UpdateAsync(user); + } + else + { + StatusMessage = "Settings unchanged"; + } + return RedirectToPage(); } }