System.Commandline, scrobble saving command

Selector.CLI/Command/CommandContext.cs

@@ -0,0 +1,16 @@
+using IF.Lastfm.Core.Api;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Selector.Model;
+
+namespace Selector.CLI
+{
+    public class CommandContext
+    {
+        public RootOptions Config { get; set; }
+        public ILoggerFactory Logger { get; set; }
+
+        public DbContextOptionsBuilder<ApplicationDbContext> DatabaseConfig { get; set; }        
+        public LastfmClient LastFmClient { get; set; }
+    }
+}

Selector.CLI/Command/HostCommand.cs

@@ -0,0 +1,144 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using NLog.Extensions.Logging;
+using Selector.Cache.Extensions;
+using Selector.CLI.Extensions;
+using Selector.CLI.Services;
+using Selector.Events;
+using Selector.Extensions;
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+
+namespace Selector.CLI
+{
+    public class HostRootCommand: RootCommand
+    {
+        public HostRootCommand()
+        {
+            Handler = CommandHandler.Create(() => HostCommand.Execute());
+        }
+    }
+
+    public class HostCommand : Command
+    {
+        public HostCommand(string name, string description = null) : base(name, description)
+        {
+            Handler = CommandHandler.Create(() => Execute());
+        }
+
+        public static int Execute()
+        {
+            try
+            {
+                CreateHostBuilder(Environment.GetCommandLineArgs(), ConfigureDefault, ConfigureDefaultNlog)
+                    .Build()
+                    .Run();
+            }
+            catch(Exception ex)
+            {
+                Console.WriteLine(ex);
+                return 1;
+            }
+
+            return 0;
+        }
+
+        public static RootOptions ConfigureOptions(HostBuilderContext context, IServiceCollection services)
+        {
+            services.Configure<RootOptions>(options =>
+            {
+                OptionsHelper.ConfigureOptions(options, context.Configuration);
+            });
+
+            var config = OptionsHelper.ConfigureOptions(context.Configuration);
+
+            services.Configure<SpotifyAppCredentials>(options =>
+            {
+                options.ClientId = config.ClientId;
+                options.ClientSecret = config.ClientSecret;
+            });
+
+            return config;
+        }
+
+        public static void ConfigureDefault(HostBuilderContext context, IServiceCollection services)
+        {
+            Console.WriteLine("~~~ Selector CLI ~~~");
+            Console.WriteLine();
+
+            Console.WriteLine("> Configuring...");
+            // CONFIG
+            var config = ConfigureOptions(context, services);
+            services.AddHttpClient();
+
+            Console.WriteLine("> Adding Services...");
+            // SERVICES
+            services.AddConsumerFactories();
+            if (config.RedisOptions.Enabled)
+            {
+                Console.WriteLine("> Adding caching consumers...");
+                services.AddCachingConsumerFactories();
+            }
+
+            services.AddWatcher()
+                    .AddEvents()
+                    .AddSpotify();
+
+            services.ConfigureLastFm(config)
+                    .ConfigureDb(config)
+                    .ConfigureEqual(config);
+
+            if (config.RedisOptions.Enabled)
+            {
+                Console.WriteLine("> Adding Redis...");
+                services.AddRedisServices(config.RedisOptions.ConnectionString);
+
+                Console.WriteLine("> Adding cache event maps...");
+                services.AddTransient<IEventMapping, FromPubSub.SpotifyLink>()
+                        .AddTransient<IEventMapping, FromPubSub.Lastfm>();
+
+                Console.WriteLine("> Adding caching Spotify consumers...");
+                services.AddCachingSpotify();
+            }
+
+            // HOSTED SERVICES
+            if (config.WatcherOptions.Enabled)
+            {
+                if (config.WatcherOptions.LocalEnabled)
+                {
+                    Console.WriteLine("> Adding Local Watcher Service");
+                    services.AddHostedService<LocalWatcherService>();
+                }
+
+                if (config.DatabaseOptions.Enabled)
+                {
+                    Console.WriteLine("> Adding Db Watcher Service");
+                    services.AddHostedService<DbWatcherService>();
+                }
+            }
+
+            if (config.ScrobbleOptions.Enabled)
+            {
+                Console.WriteLine("> Adding Scrobble Monitor Service");
+
+                services.AddHostedService<ScrobbleMonitor>();
+            }
+        }
+
+        public static void ConfigureDefaultNlog(HostBuilderContext context, ILoggingBuilder builder)
+        {
+            builder.ClearProviders()
+                   .SetMinimumLevel(LogLevel.Trace)
+                   .AddNLog(context.Configuration);
+        }
+
+        static IHostBuilder CreateHostBuilder(string[] args, Action<HostBuilderContext, IServiceCollection> BuildServices, Action<HostBuilderContext, ILoggingBuilder> BuildLogs)
+            => Host.CreateDefaultBuilder(args)
+                .UseWindowsService()
+                .UseSystemd()
+                .ConfigureServices((context, services) => BuildServices(context, services))
+                .ConfigureLogging((context, builder) => BuildLogs(context, builder));
+    }
+}

Selector.CLI/Command/Scrobble.cs

@@ -0,0 +1,114 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Selector.CLI.Extensions;
+using Selector.Model;
+using Selector.Model.Extensions;
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Selector.CLI
+{
+    public class ScrobbleCommand : Command
+    {
+        public ScrobbleCommand(string name, string description = null) : base(name, description)
+        {
+
+            var saveCommand = new ScrobbleSaveCommand("save", "save scrobbles to");
+            AddCommand(saveCommand);
+        }
+    }
+
+    public class ScrobbleSaveCommand : Command
+    {
+        public ScrobbleSaveCommand(string name, string description = null) : base(name, description) 
+        {
+            var fromOption = new Option<DateTime>("--from", getDefaultValue: () => DateTime.UtcNow.AddDays(-7), "Date from which to pull scrobbles");
+            AddOption(fromOption);
+
+            var toOption = new Option<DateTime>( "--to", getDefaultValue: () => DateTime.UtcNow.AddHours(1), "Last date for which to pull scrobbles");
+            AddOption(toOption);
+
+            var pageOption = new Option<int>("--page", getDefaultValue: () => 100, "number of scrobbles per page");
+            pageOption.AddAlias("-p");
+            AddOption(pageOption);
+
+            var delayOption = new Option<int>("--delay", getDefaultValue: () => 200, "milliseconds to delay");
+            delayOption.AddAlias("-d");
+            AddOption(delayOption);
+
+            var username = new Option<string>("--username", "user to pulls scrobbles for");
+            username.AddAlias("-u");
+            AddOption(username);
+
+            var dontAdd = new Option("--no-add", "don't add any scrobbles to the database");
+            dontAdd.AddAlias("-na");
+            AddOption(dontAdd);
+
+            var dontRemove = new Option("--no-remove", "don't remove any scrobbles from the database");
+            dontRemove.AddAlias("-nr");
+            AddOption(dontRemove);
+
+            Handler = CommandHandler.Create(async (DateTime from, DateTime to, int page, int delay, string username, bool noAdd, bool noRemove, CancellationToken token) => await Execute(from, to, page, delay, username, noAdd, noRemove, token));
+        }
+
+        public static async Task<int> Execute(DateTime from, DateTime to, int page, int delay, string username, bool noAdd, bool noRemove, CancellationToken token)
+        {
+            try
+            {
+                var context = new CommandContext().WithLogger().WithDb().WithLastfmApi();
+                var logger = context.Logger.CreateLogger("Scrobble");
+
+                var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger<ApplicationDbContext>());
+
+                logger.LogInformation("Running from {0} to {1}", from, to);
+
+                logger.LogInformation("Searching for {0}", username);
+                var user = db.Users.AsNoTracking().FirstOrDefault(u => u.UserName == username);
+
+                if (user is not null)
+                {
+                    if (user.LastFmConnected())
+                    {
+                        logger.LogInformation("Last.fm username found ({0}), starting...", user.LastFmUsername);
+
+                        await new ScrobbleSaver(
+                            context.LastFmClient.User,
+                            new ScrobbleSaverConfig()
+                            {
+                                User = user,
+                                InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay),
+                                From = from,
+                                To = to,
+                                PageSize = page,
+                                DontAdd = noAdd,
+                                DontRemove = noRemove
+                            },
+                            db,
+                            context.Logger.CreateLogger<ScrobbleSaver>())
+                                .Execute(token);
+                    }
+                    else
+                    {
+                        logger.LogError("{0} doesn't have a Last.fm username", username);
+                    }
+                }
+                else
+                {
+                    logger.LogError("{0} not found", username);
+                }
+
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex);
+                return 1;
+            }
+
+            return 0;
+        }
+    }
+}

Selector.CLI/Extensions/CommandContextExtensions.cs

@@ -0,0 +1,62 @@
+using IF.Lastfm.Core.Api;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using Selector.Model;
+
+namespace Selector.CLI.Extensions
+{
+    public static class CommandContextExtensions
+    {
+        public static CommandContext WithConfig(this CommandContext context)
+        {
+            var configBuild = new ConfigurationBuilder();
+            configBuild.AddJsonFile("appsettings.json", optional: true)
+                       .AddJsonFile("appsettings.Development.json", optional: true)
+                       .AddJsonFile("appsettings.Production.json", optional: true);
+            context.Config = configBuild.Build().ConfigureOptions();
+
+            return context;
+        }
+
+        public static CommandContext WithLogger(this CommandContext context)
+        {
+            context.Logger = LoggerFactory.Create(builder =>
+            {
+                //builder.AddConsole(a => a.);
+                builder.AddSimpleConsole(options =>
+                {
+                    options.SingleLine = true;
+                });
+                builder.SetMinimumLevel(LogLevel.Trace);
+            });
+
+            return context;
+        }
+
+        public static CommandContext WithDb(this CommandContext context)
+        {
+            if (context.Config is null)
+            {
+                context.WithConfig();
+            }
+
+            context.DatabaseConfig = new DbContextOptionsBuilder<ApplicationDbContext>();
+            context.DatabaseConfig.UseNpgsql(context.Config.DatabaseOptions.ConnectionString);
+
+            return context;
+        }
+
+        public static CommandContext WithLastfmApi(this CommandContext context)
+        {
+            if (context.Config is null)
+            {
+                context.WithConfig();
+            }
+
+            context.LastFmClient = new LastfmClient(new LastAuth(context.Config.LastfmClient, context.Config.LastfmSecret));
+
+            return context;
+        }
+    }
+}

Selector.CLI/Extensions/ServiceExtensions.cs

@@ -0,0 +1,71 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using Selector.Cache.Extensions;
+using Selector.Extensions;
+using Selector.Model;
+using Selector.Model.Services;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Selector.CLI.Extensions
+{
+    public static class ServiceExtensions
+    {
+        public static IServiceCollection ConfigureLastFm(this IServiceCollection services, RootOptions config)
+        {
+            if (config.LastfmClient is not null)
+            {
+                Console.WriteLine("> Adding Last.fm credentials...");
+
+                services.AddLastFm(config.LastfmClient, config.LastfmSecret);
+
+                if (config.RedisOptions.Enabled)
+                {
+                    Console.WriteLine("> Adding caching Last.fm consumers...");
+                    services.AddCachingLastFm();
+                }
+            }
+            else
+            {
+                Console.WriteLine("> No Last.fm credentials, skipping init...");
+            }
+
+            return services;
+        }
+
+        public static IServiceCollection ConfigureDb(this IServiceCollection services, RootOptions config)
+        {
+            if (config.DatabaseOptions.Enabled)
+            {
+                Console.WriteLine("> Adding Databse Context...");
+                services.AddDbContext<ApplicationDbContext>(options =>
+                    options.UseNpgsql(config.DatabaseOptions.ConnectionString)
+                );
+
+                services.AddHostedService<MigratorService>();
+            }
+
+            return services;
+        }
+
+        public static IServiceCollection ConfigureEqual(this IServiceCollection services, RootOptions config)
+        {
+            switch (config.Equality)
+            {
+                case EqualityChecker.Uri:
+                    Console.WriteLine("> Using Uri Equality");
+                    services.AddSingleton<IEqual, UriEqual>();
+                    break;
+                case EqualityChecker.String:
+                    Console.WriteLine("> Using String Equality");
+                    services.AddSingleton<IEqual, StringEqual>();
+                    break;
+            }
+
+            return services;
+        }
+    }    
+}

Selector.CLI/Options.cs

@@ -14,7 +14,7 @@ namespace Selector.CLI
             config.GetSection(FormatKeys( new[] { RootOptions.Key, ScrobbleMonitorOptions.Key})).Bind(options.ScrobbleOptions);
         }  
 
-        public static RootOptions ConfigureOptions(IConfiguration config)
+        public static RootOptions ConfigureOptions(this IConfiguration config)
         {
             var options = config.GetSection(RootOptions.Key).Get<RootOptions>();
             ConfigureOptions(options, config);

Selector.CLI/Program.cs

@@ -1,177 +1,15 @@
-using System;
+using System.CommandLine;
-using System.Threading.Tasks;
-using System.CommandLine;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using Microsoft.EntityFrameworkCore;
-using NLog.Extensions.Logging;
-
-using Selector.Extensions;
-using Selector.Model;
-using Selector.Cache;
-using Selector.Cache.Extensions;
-using Selector.Events;
-using Selector.Model.Services;
-using Selector.CLI.Services;
 
 namespace Selector.CLI
 {
     public static class Program
     {
-        public static async Task Main(string[] args)
+        public static void Main(string[] args)
         {
-            var cmd = new RootCommand {
+            var cmd = new HostRootCommand();
-                new Command("start") {
+            cmd.AddCommand(new ScrobbleCommand("scrobble", "Manipulate scrobbles"));
 
-                }
+            cmd.Invoke(args);
-            };
-
-            CreateHostBuilder(args, ConfigureDefault, ConfigureDefaultNlog).Build().Run();
-        }
-
-        public static RootOptions ConfigureOptions(HostBuilderContext context, IServiceCollection services)
-        {
-            services.Configure<RootOptions>(options =>
-            {
-                OptionsHelper.ConfigureOptions(options, context.Configuration);
-            });
-
-            var config = OptionsHelper.ConfigureOptions(context.Configuration);
-
-            services.Configure<SpotifyAppCredentials>(options =>
-            {
-                options.ClientId = config.ClientId;
-                options.ClientSecret = config.ClientSecret;
-            });            
-
-            return config;
-        }
-
-        public static void ConfigureLastFm(RootOptions config, IServiceCollection services)
-        {
-            if(config.LastfmClient is not null)
-            {
-                Console.WriteLine("> Adding Last.fm credentials...");
-
-                services.AddLastFm(config.LastfmClient, config.LastfmSecret);
-                
-                if(config.RedisOptions.Enabled)
-                {
-                    Console.WriteLine("> Adding caching Last.fm consumers...");
-                    services.AddCachingLastFm();
         }
     }
-            else 
-            {
-                Console.WriteLine("> No Last.fm credentials, skipping init...");
-            }
-        }
-
-        public static void ConfigureDb(RootOptions config, IServiceCollection services)
-        {
-            if (config.DatabaseOptions.Enabled)
-            {
-                Console.WriteLine("> Adding Databse Context...");
-                services.AddDbContext<ApplicationDbContext>(options =>
-                    options.UseNpgsql(config.DatabaseOptions.ConnectionString)
-                );
-
-                services.AddHostedService<MigratorService>();
-            }
-        }
-
-        public static void ConfigureEqual(RootOptions config, IServiceCollection services)
-        {
-            switch (config.Equality)
-            {
-                case EqualityChecker.Uri:
-                    Console.WriteLine("> Using Uri Equality");
-                    services.AddSingleton<IEqual, UriEqual>();
-                    break;
-                case EqualityChecker.String:
-                    Console.WriteLine("> Using String Equality");
-                    services.AddSingleton<IEqual, StringEqual>();
-                    break;
-            }
-        }
-
-        public static void ConfigureDefault(HostBuilderContext context, IServiceCollection services)
-        {
-            Console.WriteLine("~~~ Selector CLI ~~~");
-            Console.WriteLine();
-
-            Console.WriteLine("> Configuring...");
-            // CONFIG
-            var config = ConfigureOptions(context, services);
-            services.AddHttpClient();
-
-            Console.WriteLine("> Adding Services...");
-            // SERVICES
-            services.AddConsumerFactories();
-            if (config.RedisOptions.Enabled)
-            {
-                Console.WriteLine("> Adding caching consumers...");
-                services.AddCachingConsumerFactories();
-            }
-            services.AddWatcher();
-
-            services.AddEvents();
-
-            services.AddSpotify();
-            ConfigureLastFm(config, services);
-            ConfigureDb(config, services);
-            ConfigureEqual(config, services);
-
-            if (config.RedisOptions.Enabled)
-            {
-                Console.WriteLine("> Adding Redis...");
-                services.AddRedisServices(config.RedisOptions.ConnectionString);
-
-                Console.WriteLine("> Adding cache event maps...");
-                services.AddTransient<IEventMapping, FromPubSub.SpotifyLink>();
-                services.AddTransient<IEventMapping, FromPubSub.Lastfm>();
-
-                Console.WriteLine("> Adding caching Spotify consumers...");
-                services.AddCachingSpotify();
-            }
-
-            // HOSTED SERVICES
-            if (config.WatcherOptions.Enabled)
-            {
-                if(config.WatcherOptions.LocalEnabled)
-                {
-                    Console.WriteLine("> Adding Local Watcher Service");
-                    services.AddHostedService<LocalWatcherService>();
-                }
-
-                if(config.DatabaseOptions.Enabled)
-                {
-                    Console.WriteLine("> Adding Db Watcher Service");
-                    services.AddHostedService<DbWatcherService>();
-                }
-            }
-
-            if (config.ScrobbleOptions.Enabled)
-            {
-                Console.WriteLine("> Adding Scrobble Monitor Service");
-
-                services.AddHostedService<ScrobbleMonitor>();
-            }
-        }
-
-        public static void ConfigureDefaultNlog(HostBuilderContext context, ILoggingBuilder builder)
-        {
-            builder.ClearProviders();
-            builder.SetMinimumLevel(LogLevel.Trace);
-            builder.AddNLog(context.Configuration);
-        }
-
-        static IHostBuilder CreateHostBuilder(string[] args, Action<HostBuilderContext, IServiceCollection> BuildServices, Action<HostBuilderContext, ILoggingBuilder> BuildLogs)
-            => Host.CreateDefaultBuilder(args)
-                .UseWindowsService()
-                .UseSystemd()
-                .ConfigureServices((context, services) => BuildServices(context, services))
-                .ConfigureLogging((context, builder) => BuildLogs(context, builder));
-    }
 }

Selector.CLI/Properties/launchSettings.json

@@ -6,6 +6,22 @@
         "DOTNET_ENVIRONMENT": "Development"
       },
       "nativeDebugging": true
+    },
+    "Selector.CLI.Scrobble": {
+      "commandName": "Project",
+      "commandLineArgs": "scrobble save -u sarsoo --from \"2022/01/01\"",
+      "environmentVariables": {
+        "DOTNET_ENVIRONMENT": "Development"
+      },
+      "nativeDebugging": true
+    },
+    "Selector.CLI.Scrobble.All": {
+      "commandName": "Project",
+      "commandLineArgs": "scrobble save -u sarsoo --from \"2017/01/01\" -p 200 -d 75 --no-remove",
+      "environmentVariables": {
+        "DOTNET_ENVIRONMENT": "Development"
+      },
+      "nativeDebugging": true
     }
   }
 }

Selector.CLI/ScrobbleSaver.cs

@@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging;
 using Selector.Model;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -19,6 +20,9 @@ namespace Selector
         public DateTime? From { get; set; }
         public DateTime? To { get; set; }
         public int PageSize { get; set; } = 100;
+        public int Retries { get; set; } = 5;
+        public bool DontAdd { get; set; } = false;
+        public bool DontRemove { get; set; } = false;
     }
 
     public class ScrobbleSaver
@@ -27,22 +31,19 @@ namespace Selector
 
         private readonly IUserApi userClient;
         private readonly ScrobbleSaverConfig config;
-        private readonly IServiceScopeFactory serviceScopeFactory;
+        private readonly ApplicationDbContext db;
 
-        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IServiceScopeFactory _serviceScopeFactory, ILogger<ScrobbleSaver> _logger)
+        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, ApplicationDbContext _db, ILogger<ScrobbleSaver> _logger)
         {
             userClient = _userClient;
             config = _config;
-            serviceScopeFactory = _serviceScopeFactory;
+            db = _db;
             logger = _logger;
         }
 
         public async Task Execute(CancellationToken token)
         {
-            logger.LogInformation("Saving all scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
+            logger.LogInformation("Saving scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
-            
-            using var scope = serviceScopeFactory.CreateScope();
-            using var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
 
             var page1 = await userClient.GetRecentScrobbles(config.User.LastFmUsername, count: config.PageSize, from: config.From, to: config.To);
 
@@ -52,7 +53,7 @@ namespace Selector
 
                 if (page1.TotalPages > 1)
                 {
-                    var tasks = await GetScrobblesFromPageNumbers(Enumerable.Range(2, page1.TotalPages - 1), token);
+                    var tasks = await GetScrobblesFromPageNumbers(2, page1.TotalPages, token);
                     var taskResults = await Task.WhenAll(tasks);
 
                     foreach (var result in taskResults)
@@ -63,14 +64,15 @@ namespace Selector
                         }
                         else
                         {
-                            logger.LogInformation("Failed to get a subset of scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
+                            logger.LogWarning("Failed to get a subset of scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
                         }
                     }                  
                 }
 
+                logger.LogDebug("Ordering and filtering pulled scrobbles");
+
                 var nativeScrobbles = scrobbles
-                    .DistinctBy(s => s.TimePlayed)
+                    .DistinctBy(s => s.TimePlayed?.UtcDateTime)
-                    .OrderBy(s => s.TimePlayed)
                     .Select(s =>
                     {
                         var nativeScrobble = (UserScrobble) s;
@@ -78,9 +80,10 @@ namespace Selector
                         return nativeScrobble;
                     });
 
+                logger.LogDebug("Pulling currently stored scrobbles");
+
                 var currentScrobbles = db.Scrobble
                     .AsEnumerable()
-                    .OrderBy(s => s.Timestamp)
                     .Where(s => s.UserId == config.User.Id);
 
                 if (config.From is not null)
@@ -93,11 +96,43 @@ namespace Selector
                     currentScrobbles = currentScrobbles.Where(s => s.Timestamp < config.To);
                 }
 
-                (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffsContains(currentScrobbles, nativeScrobbles);
+                logger.LogInformation("Completed scrobble pulling for {0}, pulled {1:n0}", config.User.UserName, nativeScrobbles.Count());
 
-                await db.Scrobble.AddRangeAsync(toAdd.Cast<UserScrobble>());
+                logger.LogDebug("Identifying difference sets");
-                db.Scrobble.RemoveRange(toRemove.Cast<UserScrobble>());
+                var time = Stopwatch.StartNew();
+
+                (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
+
+                var toAddUser = toAdd.Cast<UserScrobble>().ToList();
+                var toRemoveUser = toRemove.Cast<UserScrobble>().ToList();
+
+                time.Stop();
+                logger.LogTrace("Finished diffing: {0:n}ms", time.ElapsedMilliseconds);
+
+                var timeDbOps = Stopwatch.StartNew();
+
+                if(!config.DontAdd)
+                {
+                    await db.Scrobble.AddRangeAsync(toAddUser);
+                }
+                else
+                {
+                    logger.LogInformation("Skipping adding of {0} scrobbles", toAddUser.Count);
+                }
+                if (!config.DontRemove)
+                {
+                    db.Scrobble.RemoveRange(toRemoveUser);
+                }
+                else
+                {
+                    logger.LogInformation("Skipping removal of {0} scrobbles", toRemoveUser.Count);
+                }
                 await db.SaveChangesAsync();
+
+                timeDbOps.Stop();
+                logger.LogTrace("DB ops: {0:n}ms", timeDbOps.ElapsedMilliseconds);
+
+                logger.LogInformation("Completed scrobble pulling for {0}, +{1:n0}, -{2:n0}", config.User.UserName, toAddUser.Count(), toRemoveUser.Count());
             }
             else
             {
@@ -105,13 +140,13 @@ namespace Selector
             }
         }
 
-        private async Task<List<Task<PageResponse<LastTrack>>>> GetScrobblesFromPageNumbers(IEnumerable<int> pageNumbers, CancellationToken token)
+        private async Task<List<Task<PageResponse<LastTrack>>>> GetScrobblesFromPageNumbers(int start, int totalPages, CancellationToken token)
         {
             var tasks = new List<Task<PageResponse<LastTrack>>>();
 
-            foreach (var pageNumber in pageNumbers)
+            foreach (var pageNumber in Enumerable.Range(start, totalPages - 1))
             {
-                logger.LogInformation("Pulling page {2} for {0}/{1}", config.User.UserName, config.User.LastFmUsername, pageNumber);
+                logger.LogInformation("Pulling page {2:n0}/{3:n0} for {0}/{1}", config.User.UserName, config.User.LastFmUsername, pageNumber, totalPages);
                 
                 tasks.Add(userClient.GetRecentScrobbles(config.User.LastFmUsername, pagenumber: pageNumber, count: config.PageSize, from: config.From, to: config.To));
                 await Task.Delay(config.InterRequestDelay, token);

Selector.CLI/Services/ScrobbleMonitor.cs

@@ -20,7 +20,6 @@ namespace Selector.CLI.Services
         private readonly ScrobbleMonitorOptions config;
         private readonly IUserApi userApi;
         private readonly IServiceScopeFactory serviceScopeFactory;
-        private Task task;
 
         public ScrobbleMonitor(ILogger<ScrobbleMonitor> _logger, IOptions<ScrobbleMonitorOptions> _options, IUserApi _userApi, IServiceScopeFactory _serviceScopeFactory, ILoggerFactory _loggerFactory)
         {
@@ -43,19 +42,14 @@ namespace Selector.CLI.Services
 
         public async Task RunScrobbleSavers(ApplicationDbContext db, CancellationToken token)
         {
+            using var scope = serviceScopeFactory.CreateScope();
+
             foreach (var user in db.Users
                 .AsNoTracking()
                 .AsEnumerable()
                 .Where(u => u.ScrobbleSavingEnabled()))
             {
-                logger.LogInformation("Starting scrobble saver for {0}/{1}", user.UserName, user.LastFmUsername);
+                //TODO
-
-                await new ScrobbleSaver(userApi, new ScrobbleSaverConfig()
-                {
-                    User = user,
-                    InterRequestDelay = config.InterRequestDelay,
-                    From = DateTime.UtcNow.AddDays(-3)
-                }, serviceScopeFactory, loggerFactory.CreateLogger<ScrobbleSaver>()).Execute(token);
             }
         }
 

Selector.CLI/appsettings.json

@@ -4,7 +4,6 @@
         "ClientSecret": "",
         "Equality": "uri",
         "Watcher": {
-            "enabled": false,
             "localenabled": false,
             "Instances": [
                 {
@@ -19,7 +18,7 @@
             "enabled": true
         },
         "Redis": {
-            "enabled": false
+            "enabled": true
         }
     },
     "Logging": {

Selector.Cache/Extensions/ServiceExtensions.cs

@@ -9,7 +9,7 @@ namespace Selector.Cache.Extensions
 {
     public static class ServiceExtensions
     {
-        public static void AddRedisServices(this IServiceCollection services, string connectionStr)
+        public static IServiceCollection AddRedisServices(this IServiceCollection services, string connectionStr)
         {
             Console.WriteLine("> Configuring Redis...");
 
@@ -23,9 +23,11 @@ namespace Selector.Cache.Extensions
             services.AddSingleton(connMulti);
             services.AddTransient<IDatabaseAsync>(services => services.GetService<ConnectionMultiplexer>().GetDatabase());
             services.AddTransient<ISubscriber>(services => services.GetService<ConnectionMultiplexer>().GetSubscriber());
+
+            return services;
         }
 
-        public static void AddCachingConsumerFactories(this IServiceCollection services)
+        public static IServiceCollection AddCachingConsumerFactories(this IServiceCollection services)
         {
             services.AddTransient<IAudioFeatureInjectorFactory, CachingAudioFeatureInjectorFactory>();
             services.AddTransient<CachingAudioFeatureInjectorFactory>();
@@ -36,16 +38,22 @@ namespace Selector.Cache.Extensions
             services.AddTransient<CacheWriterFactory>();
             services.AddTransient<IPublisherFactory, PublisherFactory>();
             services.AddTransient<PublisherFactory>();
+
+            return services;
         }
 
-        public static void AddCachingSpotify(this IServiceCollection services)
+        public static IServiceCollection AddCachingSpotify(this IServiceCollection services)
         {
             services.AddSingleton<AudioFeaturePuller>();
+
+            return services;
         }
 
-        public static void AddCachingLastFm(this IServiceCollection services)
+        public static IServiceCollection AddCachingLastFm(this IServiceCollection services)
         {
             services.AddSingleton<PlayCountPuller>();
+
+            return services;
         }
     }
 }

Selector.Event/Extensions/ServiceExtensions.cs

@@ -4,24 +4,30 @@ namespace Selector.Events
 {
     public static class ServiceExtensions
     {
-        public static void AddEvents(this IServiceCollection services)
+        public static IServiceCollection AddEvents(this IServiceCollection services)
         {
             services.AddEventBus();
             services.AddEventMappingAgent();
 
             services.AddTransient<IUserEventFirerFactory, UserEventFirerFactory>();
             services.AddTransient<UserEventFirerFactory>();
+
+            return services;
         }
 
-        public static void AddEventBus(this IServiceCollection services)
+        public static IServiceCollection AddEventBus(this IServiceCollection services)
         {
             services.AddSingleton<UserEventBus>();
             services.AddSingleton<IEventBus, UserEventBus>(sp => sp.GetRequiredService<UserEventBus>());
+
+            return services;
         }
 
-        public static void AddEventMappingAgent(this IServiceCollection services)
+        public static IServiceCollection AddEventMappingAgent(this IServiceCollection services)
         {
             services.AddHostedService<EventMappingService>();
+
+            return services;
         }
     }
 }

Selector/Extensions/ServiceExtensions.cs

@@ -9,7 +9,7 @@ namespace Selector.Extensions
 {
     public static class ServiceExtensions
     {
-        public static void AddConsumerFactories(this IServiceCollection services)
+        public static IServiceCollection AddConsumerFactories(this IServiceCollection services)
         {
             services.AddTransient<IAudioFeatureInjectorFactory, AudioFeatureInjectorFactory>();
             services.AddTransient<AudioFeatureInjectorFactory>();
@@ -19,15 +19,19 @@ namespace Selector.Extensions
 
             services.AddTransient<IWebHookFactory, WebHookFactory>();
             services.AddTransient<WebHookFactory>();
+
+            return services;
         }
 
-        public static void AddSpotify(this IServiceCollection services)
+        public static IServiceCollection AddSpotify(this IServiceCollection services)
         {
             services.AddSingleton<IRefreshTokenFactoryProvider, RefreshTokenFactoryProvider>();
             services.AddSingleton<IRefreshTokenFactoryProvider, CachingRefreshTokenFactoryProvider>();
+
+            return services;
         }
 
-        public static void AddLastFm(this IServiceCollection services, string client, string secret)
+        public static IServiceCollection AddLastFm(this IServiceCollection services, string client, string secret)
         {
             var lastAuth = new LastAuth(client, secret);
             services.AddSingleton(lastAuth);
@@ -42,12 +46,16 @@ namespace Selector.Extensions
             services.AddTransient<IChartApi>(sp => sp.GetService<LastfmClient>().Chart);
             services.AddTransient<ILibraryApi>(sp => sp.GetService<LastfmClient>().Library);
             services.AddTransient<ITagApi>(sp => sp.GetService<LastfmClient>().Tag);
+
+            return services;
         }
 
-        public static void AddWatcher(this IServiceCollection services)
+        public static IServiceCollection AddWatcher(this IServiceCollection services)
         {
             services.AddSingleton<IWatcherFactory, WatcherFactory>();
             services.AddSingleton<IWatcherCollectionFactory, WatcherCollectionFactory>();
+
+            return services;
         }
     }
 }

Selector/Scrobble/Scrobble.cs

@@ -21,5 +21,7 @@ namespace Selector
             AlbumName = track.AlbumName,
             ArtistName = track.ArtistName,
         };
+
+        public override string ToString() => $"({Timestamp}) {TrackName}, {AlbumName}, {ArtistName}";
     }
 }

Selector/Scrobble/ScrobbleMatcher.cs

@@ -11,7 +11,7 @@ namespace Selector
             => serviceScrobble.TimePlayed.Equals(nativeScrobble);
 
         public static bool MatchTime(Scrobble nativeScrobble, Scrobble serviceScrobble)
-            => serviceScrobble.Timestamp.Equals(nativeScrobble);
+            => serviceScrobble.Timestamp.Equals(nativeScrobble.Timestamp);
 
         public static (IEnumerable<Scrobble>, IEnumerable<Scrobble>) IdentifyDiffs(IEnumerable<Scrobble> existing, IEnumerable<Scrobble> toApply)
         {
@@ -22,6 +22,8 @@ namespace Selector
             var toAdd = new List<Scrobble>();
             var toRemove = new List<Scrobble>();
 
+            if (toApplyIter.MoveNext())
+            {
                 if (existing.Any())
                 {
                     foreach (var currentExisting in existing)
@@ -30,10 +32,14 @@ namespace Selector
                         {
                             toAdd.Add(toApplyIter.Current);
 
-                        if (!toApplyIter.MoveNext()) break;
+                            toApplyIter.MoveNext();
                         }
 
-                    if (!MatchTime(currentExisting, toApplyIter.Current))
+                        if (MatchTime(currentExisting, toApplyIter.Current))
+                        {
+                            toApplyIter.MoveNext();
+                        }
+                        else
                         {
                             toRemove.Add(currentExisting);
                         }
@@ -43,6 +49,7 @@ namespace Selector
                 {
                     toAdd.AddRange(toApply);
                 }
+            }
 
             return (toAdd, toRemove);
         }