adding scrobble saving to ui, adding full scrobble save

Selector.CLI/Extensions/ServiceExtensions.cs

@@ -55,23 +55,48 @@ namespace Selector.CLI.Extensions
                         tp.MaxConcurrency = 5;
                     });
 
-                    var scrobbleKey = new JobKey("scrobble-watcher", "scrobble");
+                    if (config.JobOptions.Scrobble.Enabled)
+                    {
+                        Console.WriteLine("> Adding Scrobble Jobs...");
 
-                    options.AddJob<ScrobbleWatcherJob>(j => j
+                        var scrobbleKey = new JobKey("scrobble-watcher-agile", "scrobble");
-                        .WithDescription("Watch recent scrobbles and mirror to database")
-                        .WithIdentity(scrobbleKey)
-                    );
 
-                    options.AddTrigger(t => t
+                        options.AddJob<ScrobbleWatcherJob>(j => j
-                        .WithIdentity("scrobble-watcher-trigger")
+                            .WithDescription("Watch recent scrobbles and mirror to database")
-                        .ForJob(scrobbleKey)
+                            .WithIdentity(scrobbleKey)
-                        .StartNow()
+                            .UsingJobData("IsFull", false)
-                        .WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever())
+                        );
-                        .WithDescription("Periodic trigger for scrobble watcher")
+
-                    );
+                        options.AddTrigger(t => t
+                            .WithIdentity("scrobble-watcher-agile-trigger")
+                            .ForJob(scrobbleKey)
+                            .StartNow()
+                            .WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever())
+                            .WithDescription("Periodic trigger for scrobble watcher")
+                        );
+
+                        var fullScrobbleKey = new JobKey("scrobble-watcher-full", "scrobble");
+
+                        options.AddJob<ScrobbleWatcherJob>(j => j
+                            .WithDescription("Check all scrobbles and mirror to database")
+                            .WithIdentity(fullScrobbleKey)
+                            .UsingJobData("IsFull", true)
+                        );
+
+                        options.AddTrigger(t => t
+                            .WithIdentity("scrobble-watcher-full-trigger")
+                            .ForJob(fullScrobbleKey)
+                            .WithCronSchedule(config.JobOptions.Scrobble.FullScrobbleCron)
+                            .WithDescription("Periodic trigger for scrobble watcher")
+                        );
+                    }   
+                    else
+                    {
+                        Console.WriteLine("> Skipping Scrobble Jobs...");
+                    }
                 });
 
-                services.AddQuartzHostedService(options =>{
+                services.AddQuartzHostedService(options => {
 
                     options.WaitForJobsToComplete = true;
                 });

Selector.CLI/Jobs/ScrobbleWatcherJob.cs

@@ -23,6 +23,10 @@ namespace Selector.CLI.Jobs
         private readonly ApplicationDbContext db;
         private readonly ScrobbleWatcherJobOptions options;
 
+        private static object databaseLock = new();
+
+        public bool IsFull { get; set; }
+
         public ScrobbleWatcherJob(
             IUserApi _userApi,
             IScrobbleRepository _scrobbleRepo,
@@ -42,47 +46,50 @@ namespace Selector.CLI.Jobs
 
         public async Task Execute(IJobExecutionContext context)
         {
-            logger.LogInformation("Starting scrobble watching job");
+            try
-
-            var users = db.Users
-                .AsEnumerable()
-                .Where(u => u.ScrobbleSavingEnabled())
-                .ToArray();
-
-            foreach (var user in users)
             {
-                logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername);
+                logger.LogInformation("Starting scrobble watching job");
 
-                if (options.From is not null && options.From.Value.Kind != DateTimeKind.Utc)
+                var users = db.Users
+                    .AsEnumerable()
+                    .Where(u => u.ScrobbleSavingEnabled())
+                    .ToArray();
+
+                foreach (var user in users)
                 {
-                    options.From = options.From.Value.ToUniversalTime();
+                    logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername);
-                }
 
-                if (options.To is not null && options.To.Value.Kind != DateTimeKind.Utc)
+                    DateTime? from = null;
-                {
+                    if (options.From is not null && !IsFull)
-                    options.To = options.To.Value.ToUniversalTime();
-                }
-
-                var saver = new ScrobbleSaver(
-                    userApi,
-                    new()
                     {
-                        User = user,
+                        from = options.From.Value.ToUniversalTime();
-                        InterRequestDelay = options.InterRequestDelay,
+                    }
-                        From = options.From,
-                        To = options.To,
-                        PageSize = options.PageSize,
-                        DontAdd = false,
-                        DontRemove = false,
-                        SimultaneousConnections = options.Simultaneous
-                    },
-                    scrobbleRepo,
-                    loggerFactory.CreateLogger<ScrobbleSaver>(),
-                    loggerFactory);
 
-                await saver.Execute(context.CancellationToken);
+                    var saver = new ScrobbleSaver(
+                        userApi,
+                        new()
+                        {
+                            User = user,
+                            InterRequestDelay = options.InterRequestDelay,
+                            From = from,
+                            PageSize = options.PageSize,
+                            DontAdd = false,
+                            DontRemove = false,
+                            SimultaneousConnections = options.Simultaneous
+                        },
+                        scrobbleRepo,
+                        loggerFactory.CreateLogger<ScrobbleSaver>(),
+                        loggerFactory,
+                        databaseLock);
 
-                logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername);
+                    await saver.Execute(context.CancellationToken);
+
+                    logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername);
+                }
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error occured while saving scrobbles");
             }
         }
     }

Selector.CLI/Options.cs

@@ -124,10 +124,10 @@ namespace Selector.CLI
         public const string Key = "Scrobble";
 
         public bool Enabled { get; set; } = true;
+        public string FullScrobbleCron { get; set; } = "0 2 * * * ?";
         public TimeSpan InterJobDelay { get; set; } = TimeSpan.FromMinutes(5);
         public TimeSpan InterRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
         public DateTime? From { get; set; } = DateTime.UtcNow.AddDays(-14);
-        public DateTime? To { get; set; }
         public int PageSize { get; set; } = 200;
         public int Simultaneous { get; set; } = 3;
     }

Selector.CLI/ScrobbleSaver.cs

@@ -36,18 +36,21 @@ namespace Selector
 
         private readonly IUserApi userClient;
         private readonly ScrobbleSaverConfig config;
-        private Task batchTask;
         private BatchingOperation<ScrobbleRequest> batchOperation;
 
         private readonly IScrobbleRepository scrobbleRepo;
 
-        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null)
+        private readonly object dbLock;
+
+        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null, object _dbLock = null)
         {
             userClient = _userClient;
             config = _config;
             scrobbleRepo = _scrobbleRepository;
             logger = _logger;
             loggerFactory = _loggerFactory;
+
+            dbLock = _dbLock ?? new();
         }
 
         public async Task Execute(CancellationToken token)
@@ -74,17 +77,7 @@ namespace Selector
                         GetRequestsFromPageNumbers(2, page1.TotalPages)
                     );
 
-                    batchTask = batchOperation.TriggerRequests(token);
+                    await batchOperation.TriggerRequests(token);
-                }
-
-
-                logger.LogDebug("Pulling currently stored scrobbles");
-
-                var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To);
-
-                if(batchTask is not null)
-                {
-                    await batchTask;
                 }
 
                 IEnumerable<LastTrack> scrobbles;
@@ -108,50 +101,57 @@ namespace Selector
                     .Select(s => (UserScrobble) s)
                     .ToArray();
 
-                logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
+                logger.LogInformation("Completed network scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
 
-                logger.LogDebug("Identifying difference sets");
+                logger.LogDebug("Pulling currently stored scrobbles");
-                var time = Stopwatch.StartNew();
 
-                (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
+                lock (dbLock)
-
-                time.Stop();
-                logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds);
-
-                var timeDbOps = Stopwatch.StartNew();
-
-                if(!config.DontAdd)
                 {
-                    foreach(var add in toAdd)
+                    var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To);
+
+                    logger.LogDebug("Identifying difference sets");
+                    var time = Stopwatch.StartNew();
+
+                    (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
+
+                    time.Stop();
+                    logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds);
+
+                    var timeDbOps = Stopwatch.StartNew();
+
+                    if (!config.DontAdd)
                     {
-                        var scrobble = (UserScrobble) add;
+                        foreach (var add in toAdd)
-                        scrobble.UserId = config.User.Id;
+                        {
-                        scrobbleRepo.Add(scrobble);
+                            var scrobble = (UserScrobble)add;
+                            scrobble.UserId = config.User.Id;
+                            scrobbleRepo.Add(scrobble);
+                        }
                     }
-                }
+                    else
-                else
-                {
-                    logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count());
-                }
-                if (!config.DontRemove)
-                {
-                    foreach (var remove in toRemove)
                     {
-                        var scrobble = (UserScrobble) remove;
+                        logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count());
-                        scrobble.UserId = config.User.Id;
-                        scrobbleRepo.Remove(scrobble);
                     }
-                }
+                    if (!config.DontRemove)
-                else
+                    {
-                {
+                        foreach (var remove in toRemove)
-                    logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count());
+                        {
-                }
+                            var scrobble = (UserScrobble)remove;
-                await scrobbleRepo.Save();
+                            scrobble.UserId = config.User.Id;
+                            scrobbleRepo.Remove(scrobble);
+                        }
+                    }
+                    else
+                    {
+                        logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count());
+                    }
+                    _ = scrobbleRepo.Save().Result;
 
-                timeDbOps.Stop();
+                    timeDbOps.Stop();
-                logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds);
+                    logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds);
 
-                logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
+                    logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
+                }
             }
             else
             {

Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml

@@ -16,7 +16,12 @@
                 <input asp-for="Input.Username" class="form-control" />
                 <span asp-validation-for="Input.Username" class="text-danger"></span>
             </div>
-            <button id="change-username-button" type="submit" asp-page-handler="ChangeUsername" class="btn btn-primary form-element">Change Username</button>
+            <div class="form-group form-element">
+                <label asp-for="Input.ScrobbleSaving"></label>
+                <input asp-for="Input.ScrobbleSaving" />
+                <span asp-validation-for="Input.ScrobbleSaving" class="text-danger"></span>
+            </div>
+            <button id="save-button" type="submit" asp-page-handler="Save" class="btn btn-primary form-element">Save</button>
         </form>
     </div>
 </div>

Selector.Web/Areas/Identity/Pages/Account/Manage/Lastfm.cshtml.cs

@@ -13,6 +13,7 @@ using Microsoft.AspNetCore.WebUtilities;
 
 using Selector.Model;
 using Selector.Events;
+using Microsoft.Extensions.Logging;
 
 namespace Selector.Web.Areas.Identity.Pages.Account.Manage
 {
@@ -21,12 +22,17 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
         private readonly UserManager<ApplicationUser> _userManager;
         private readonly UserEventBus UserEvent;
 
+        private readonly ILogger<LastFmModel> logger;
+
         public LastFmModel(
             UserManager<ApplicationUser> userManager,
-            UserEventBus userEvent)
+            UserEventBus userEvent,
+            ILogger<LastFmModel> _logger)
         {
             _userManager = userManager;
             UserEvent = userEvent;
+
+            logger = _logger;
         }
 
         [TempData]
@@ -39,6 +45,9 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
         {
             [Display(Name = "Username")]
             public string Username { get; set; }
+
+            [Display(Name = "Scrobble Saving")]
+            public bool ScrobbleSaving { get; set; }
         }
 
         private Task LoadAsync(ApplicationUser user)
@@ -46,6 +55,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
             Input = new InputModel
             {
                 Username = user.LastFmUsername,
+                ScrobbleSaving = user.SaveScrobbles
             };
 
             return Task.CompletedTask;
@@ -63,7 +73,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
             return Page();
         }
 
-        public async Task<IActionResult> OnPostChangeUsernameAsync()
+        public async Task<IActionResult> OnPostSaveAsync()
         {
             var user = await _userManager.GetUserAsync(User);
             if (user == null)
@@ -77,23 +87,54 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
                 return Page();
             }
 
+            var changed = false;
+
             if (Input.Username != user.LastFmUsername)
             {
                 var oldUsername = user.LastFmUsername;
                 user.LastFmUsername = Input.Username?.Trim();
-                
+
-                await _userManager.UpdateAsync(user);
+                changed = true;
+
                 UserEvent.OnLastfmCredChange(this, new LastfmChange { 
                     UserId = user.Id, 
                     PreviousUsername = oldUsername, 
                     NewUsername = user.LastFmUsername
                 });
 
+                logger.LogInformation("Changing username from {} to {}", oldUsername, user.LastFmUsername);
+
                 StatusMessage = "Username changed";
-                return RedirectToPage();
             }
 
-            StatusMessage = "Username unchanged";
+            if (Input.ScrobbleSaving != user.SaveScrobbles)
+            {
+                user.SaveScrobbles = Input.ScrobbleSaving;
+
+                logger.LogInformation("Changing scrobble saving from {} to {}", !Input.ScrobbleSaving, Input.ScrobbleSaving);
+                
+                if (changed)
+                {
+                    StatusMessage += ", scrobble saving updated";
+                }
+                else
+                {
+                    StatusMessage = "Scrobble saving updated";
+                    changed = true;
+                }
+            }
+
+            if (changed)
+            {
+                logger.LogInformation("Saving Last.fm settings for {}", user.LastFmUsername);
+
+                await _userManager.UpdateAsync(user);
+            }
+            else
+            {
+                StatusMessage = "Settings unchanged";
+            }
+
             return RedirectToPage();
         }
     }