adding scrobble saving to ui, adding full scrobble save

Selector.CLI/Extensions/ServiceExtensions.cs

@@ -55,23 +55,48 @@ namespace Selector.CLI.Extensions
                         tp.MaxConcurrency = 5;
                     });
 
-                    var scrobbleKey = new JobKey("scrobble-watcher", "scrobble");
+                    if (config.JobOptions.Scrobble.Enabled)
+                    {
+                        Console.WriteLine("> Adding Scrobble Jobs...");
+
+                        var scrobbleKey = new JobKey("scrobble-watcher-agile", "scrobble");
 
                         options.AddJob<ScrobbleWatcherJob>(j => j
                             .WithDescription("Watch recent scrobbles and mirror to database")
                             .WithIdentity(scrobbleKey)
+                            .UsingJobData("IsFull", false)
                         );
 
                         options.AddTrigger(t => t
-                        .WithIdentity("scrobble-watcher-trigger")
+                            .WithIdentity("scrobble-watcher-agile-trigger")
                             .ForJob(scrobbleKey)
                             .StartNow()
                             .WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever())
                             .WithDescription("Periodic trigger for scrobble watcher")
                         );
+
+                        var fullScrobbleKey = new JobKey("scrobble-watcher-full", "scrobble");
+
+                        options.AddJob<ScrobbleWatcherJob>(j => j
+                            .WithDescription("Check all scrobbles and mirror to database")
+                            .WithIdentity(fullScrobbleKey)
+                            .UsingJobData("IsFull", true)
+                        );
+
+                        options.AddTrigger(t => t
+                            .WithIdentity("scrobble-watcher-full-trigger")
+                            .ForJob(fullScrobbleKey)
+                            .WithCronSchedule(config.JobOptions.Scrobble.FullScrobbleCron)
+                            .WithDescription("Periodic trigger for scrobble watcher")
+                        );
+                    }   
+                    else
+                    {
+                        Console.WriteLine("> Skipping Scrobble Jobs...");
+                    }
                 });
 
-                services.AddQuartzHostedService(options =>{
+                services.AddQuartzHostedService(options => {
 
                     options.WaitForJobsToComplete = true;
                 });

Selector.CLI/Jobs/ScrobbleWatcherJob.cs

@@ -23,6 +23,10 @@ namespace Selector.CLI.Jobs
         private readonly ApplicationDbContext db;
         private readonly ScrobbleWatcherJobOptions options;
 
+        private static object databaseLock = new();
+
+        public bool IsFull { get; set; }
+
         public ScrobbleWatcherJob(
             IUserApi _userApi,
             IScrobbleRepository _scrobbleRepo,
@@ -41,6 +45,8 @@ namespace Selector.CLI.Jobs
         }
 
         public async Task Execute(IJobExecutionContext context)
+        {
+            try
             {
                 logger.LogInformation("Starting scrobble watching job");
 
@@ -53,14 +59,10 @@ namespace Selector.CLI.Jobs
                 {
                     logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername);
 
-                if (options.From is not null && options.From.Value.Kind != DateTimeKind.Utc)
+                    DateTime? from = null;
+                    if (options.From is not null && !IsFull)
                     {
-                    options.From = options.From.Value.ToUniversalTime();
+                        from = options.From.Value.ToUniversalTime();
-                }
-
-                if (options.To is not null && options.To.Value.Kind != DateTimeKind.Utc)
-                {
-                    options.To = options.To.Value.ToUniversalTime();
                     }
 
                     var saver = new ScrobbleSaver(
@@ -69,8 +71,7 @@ namespace Selector.CLI.Jobs
                         {
                             User = user,
                             InterRequestDelay = options.InterRequestDelay,
-                        From = options.From,
+                            From = from,
-                        To = options.To,
                             PageSize = options.PageSize,
                             DontAdd = false,
                             DontRemove = false,
@@ -78,12 +79,18 @@ namespace Selector.CLI.Jobs
                         },
                         scrobbleRepo,
                         loggerFactory.CreateLogger<ScrobbleSaver>(),
-                    loggerFactory);
+                        loggerFactory,
+                        databaseLock);
 
                     await saver.Execute(context.CancellationToken);
 
                     logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername);
                 }
             }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error occured while saving scrobbles");
+            }
+        }
     }
 }

Selector.CLI/Options.cs

@@ -124,10 +124,10 @@ namespace Selector.CLI
         public const string Key = "Scrobble";
 
         public bool Enabled { get; set; } = true;
+        public string FullScrobbleCron { get; set; } = "0 2 * * * ?";
         public TimeSpan InterJobDelay { get; set; } = TimeSpan.FromMinutes(5);
         public TimeSpan InterRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
         public DateTime? From { get; set; } = DateTime.UtcNow.AddDays(-14);
-        public DateTime? To { get; set; }
         public int PageSize { get; set; } = 200;
         public int Simultaneous { get; set; } = 3;
     }

Selector.CLI/ScrobbleSaver.cs

@@ -36,18 +36,21 @@ namespace Selector
 
         private readonly IUserApi userClient;
         private readonly ScrobbleSaverConfig config;
-        private Task batchTask;
         private BatchingOperation<ScrobbleRequest> batchOperation;
 
         private readonly IScrobbleRepository scrobbleRepo;
 
-        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null)
+        private readonly object dbLock;
+
+        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null, object _dbLock = null)
         {
             userClient = _userClient;
             config = _config;
             scrobbleRepo = _scrobbleRepository;
             logger = _logger;
             loggerFactory = _loggerFactory;
+
+            dbLock = _dbLock ?? new();
         }
 
         public async Task Execute(CancellationToken token)
@@ -74,17 +77,7 @@ namespace Selector
                         GetRequestsFromPageNumbers(2, page1.TotalPages)
                     );
 
-                    batchTask = batchOperation.TriggerRequests(token);
+                    await batchOperation.TriggerRequests(token);
-                }
-
-
-                logger.LogDebug("Pulling currently stored scrobbles");
-
-                var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To);
-
-                if(batchTask is not null)
-                {
-                    await batchTask;
                 }
 
                 IEnumerable<LastTrack> scrobbles;
@@ -108,7 +101,13 @@ namespace Selector
                     .Select(s => (UserScrobble) s)
                     .ToArray();
 
-                logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
+                logger.LogInformation("Completed network scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
+
+                logger.LogDebug("Pulling currently stored scrobbles");
+
+                lock (dbLock)
+                {
+                    var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To);
 
                     logger.LogDebug("Identifying difference sets");
                     var time = Stopwatch.StartNew();
@@ -120,11 +119,11 @@ namespace Selector
 
                     var timeDbOps = Stopwatch.StartNew();
 
-                if(!config.DontAdd)
+                    if (!config.DontAdd)
                     {
-                    foreach(var add in toAdd)
+                        foreach (var add in toAdd)
                         {
-                        var scrobble = (UserScrobble) add;
+                            var scrobble = (UserScrobble)add;
                             scrobble.UserId = config.User.Id;
                             scrobbleRepo.Add(scrobble);
                         }
@@ -137,7 +136,7 @@ namespace Selector
                     {
                         foreach (var remove in toRemove)
                         {
-                        var scrobble = (UserScrobble) remove;
+                            var scrobble = (UserScrobble)remove;
                             scrobble.UserId = config.User.Id;
                             scrobbleRepo.Remove(scrobble);
                         }
@@ -146,13 +145,14 @@ namespace Selector
                     {
                         logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count());
                     }
-                await scrobbleRepo.Save();
+                    _ = scrobbleRepo.Save().Result;
 
                     timeDbOps.Stop();
                     logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds);
 
                     logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
                 }
+            }
             else
             {
                 logger.LogError("Failed to pull first scrobble page for {}/{}", config.User.UserName, config.User.LastFmUsername);

Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml

@@ -16,7 +16,12 @@
                 <input asp-for="Input.Username" class="form-control" />
                 <span asp-validation-for="Input.Username" class="text-danger"></span>
             </div>
-            <button id="change-username-button" type="submit" asp-page-handler="ChangeUsername" class="btn btn-primary form-element">Change Username</button>
+            <div class="form-group form-element">
+                <label asp-for="Input.ScrobbleSaving"></label>
+                <input asp-for="Input.ScrobbleSaving" />
+                <span asp-validation-for="Input.ScrobbleSaving" class="text-danger"></span>
+            </div>
+            <button id="save-button" type="submit" asp-page-handler="Save" class="btn btn-primary form-element">Save</button>
         </form>
     </div>
 </div>

Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml.cs

@@ -13,6 +13,7 @@ using Microsoft.AspNetCore.WebUtilities;
 
 using Selector.Model;
 using Selector.Events;
+using Microsoft.Extensions.Logging;
 
 namespace Selector.Web.Areas.Identity.Pages.Account.Manage
 {
@@ -21,12 +22,17 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
         private readonly UserManager<ApplicationUser> _userManager;
         private readonly UserEventBus UserEvent;
 
+        private readonly ILogger<LastFmModel> logger;
+
         public LastFmModel(
             UserManager<ApplicationUser> userManager,
-            UserEventBus userEvent)
+            UserEventBus userEvent,
+            ILogger<LastFmModel> _logger)
         {
             _userManager = userManager;
             UserEvent = userEvent;
+
+            logger = _logger;
         }
 
         [TempData]
@@ -39,6 +45,9 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
         {
             [Display(Name = "Username")]
             public string Username { get; set; }
+
+            [Display(Name = "Scrobble Saving")]
+            public bool ScrobbleSaving { get; set; }
         }
 
         private Task LoadAsync(ApplicationUser user)
@@ -46,6 +55,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
             Input = new InputModel
             {
                 Username = user.LastFmUsername,
+                ScrobbleSaving = user.SaveScrobbles
             };
 
             return Task.CompletedTask;
@@ -63,7 +73,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
             return Page();
         }
 
-        public async Task<IActionResult> OnPostChangeUsernameAsync()
+        public async Task<IActionResult> OnPostSaveAsync()
         {
             var user = await _userManager.GetUserAsync(User);
             if (user == null)
@@ -77,23 +87,54 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
                 return Page();
             }
 
+            var changed = false;
+
             if (Input.Username != user.LastFmUsername)
             {
                 var oldUsername = user.LastFmUsername;
                 user.LastFmUsername = Input.Username?.Trim();
 
-                await _userManager.UpdateAsync(user);
+                changed = true;
+
                 UserEvent.OnLastfmCredChange(this, new LastfmChange { 
                     UserId = user.Id, 
                     PreviousUsername = oldUsername, 
                     NewUsername = user.LastFmUsername
                 });
 
+                logger.LogInformation("Changing username from {} to {}", oldUsername, user.LastFmUsername);
+
                 StatusMessage = "Username changed";
-                return RedirectToPage();
             }
 
-            StatusMessage = "Username unchanged";
+            if (Input.ScrobbleSaving != user.SaveScrobbles)
+            {
+                user.SaveScrobbles = Input.ScrobbleSaving;
+
+                logger.LogInformation("Changing scrobble saving from {} to {}", !Input.ScrobbleSaving, Input.ScrobbleSaving);
+                
+                if (changed)
+                {
+                    StatusMessage += ", scrobble saving updated";
+                }
+                else
+                {
+                    StatusMessage = "Scrobble saving updated";
+                    changed = true;
+                }
+            }
+
+            if (changed)
+            {
+                logger.LogInformation("Saving Last.fm settings for {}", user.LastFmUsername);
+
+                await _userManager.UpdateAsync(user);
+            }
+            else
+            {
+                StatusMessage = "Settings unchanged";
+            }
+
             return RedirectToPage();
         }
     }