adding scrobbles and lastfm/spotify mappings

Selector.CLI/Options.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using Microsoft.Extensions.Configuration;
 
 namespace Selector.CLI
@@ -10,6 +11,7 @@ namespace Selector.CLI
             config.GetSection(FormatKeys( new[] { RootOptions.Key, WatcherOptions.Key})).Bind(options.WatcherOptions);
             config.GetSection(FormatKeys( new[] { RootOptions.Key, DatabaseOptions.Key})).Bind(options.DatabaseOptions);
             config.GetSection(FormatKeys( new[] { RootOptions.Key, RedisOptions.Key})).Bind(options.RedisOptions);
+            config.GetSection(FormatKeys( new[] { RootOptions.Key, ScrobbleMonitorOptions.Key})).Bind(options.ScrobbleOptions);
         }  
 
         public static RootOptions ConfigureOptions(IConfiguration config)
@@ -37,6 +39,7 @@ namespace Selector.CLI
         public string LastfmClient { get; set; }
         public string LastfmSecret { get; set; }
         public WatcherOptions WatcherOptions { get; set; } = new();
+        public ScrobbleMonitorOptions ScrobbleOptions { get; set; } = new();
         public DatabaseOptions DatabaseOptions { get; set; } = new();
         public RedisOptions RedisOptions { get; set; } = new();
         public EqualityChecker Equality { get; set; } = EqualityChecker.Uri;
@@ -85,4 +88,12 @@ namespace Selector.CLI
         public bool Enabled { get; set; } = false;
         public string ConnectionString { get; set; }
     }
+
+    public class ScrobbleMonitorOptions
+    {
+        public const string Key = "Scrobble";
+
+        public bool Enabled { get; set; } = true;
+        public TimeSpan InterRequestDelay { get; set; } = new(0, 0, 0, 1, 0);
+    }
 }

Selector.CLI/Program.cs

@@ -13,6 +13,7 @@ using Selector.Cache;
 using Selector.Cache.Extensions;
 using Selector.Events;
 using Selector.Model.Services;
+using Selector.CLI.Services;
 
 namespace Selector.CLI
 {
@@ -150,6 +151,13 @@ namespace Selector.CLI
                     services.AddHostedService<DbWatcherService>();
                 }
             }
+
+            if (config.ScrobbleOptions.Enabled)
+            {
+                Console.WriteLine("> Adding Scrobble Monitor Service");
+
+                services.AddHostedService<ScrobbleMonitor>();
+            }
         }
 
         public static void ConfigureDefaultNlog(HostBuilderContext context, ILoggingBuilder builder)

Selector.CLI/ScrobbleSaver.cs

@@ -0,0 +1,123 @@
+using IF.Lastfm.Core.Api;
+using IF.Lastfm.Core.Api.Helpers;
+using IF.Lastfm.Core.Objects;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Selector.Model;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Selector
+{
+    public class ScrobbleSaverConfig
+    {
+        public ApplicationUser User { get; set; }
+        public TimeSpan InterRequestDelay { get; set; }
+        public DateTime? From { get; set; }
+        public DateTime? To { get; set; }
+        public int PageSize { get; set; } = 100;
+    }
+
+    public class ScrobbleSaver
+    {
+        private readonly ILogger<ScrobbleSaver> logger;
+
+        private readonly IUserApi userClient;
+        private readonly ScrobbleSaverConfig config;
+        private readonly IServiceScopeFactory serviceScopeFactory;
+
+        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IServiceScopeFactory _serviceScopeFactory, ILogger<ScrobbleSaver> _logger)
+        {
+            userClient = _userClient;
+            config = _config;
+            serviceScopeFactory = _serviceScopeFactory;
+            logger = _logger;
+        }
+
+        public async Task Execute(CancellationToken token)
+        {
+            logger.LogInformation("Saving all scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
+            
+            using var scope = serviceScopeFactory.CreateScope();
+            using var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
+
+            var page1 = await userClient.GetRecentScrobbles(config.User.LastFmUsername, count: config.PageSize, from: config.From, to: config.To);
+
+            if(page1.Success)
+            {
+                var scrobbles = page1.Content.ToList();
+
+                if(page1.TotalPages > 1)
+                {
+                    var tasks = await GetScrobblesFromPageNumbers(Enumerable.Range(2, page1.TotalPages - 1), token);
+                    var taskResults = await Task.WhenAll(tasks);
+
+                    foreach (var result in taskResults)
+                    {
+                        if (result.Success)
+                        {
+                            scrobbles.AddRange(result.Content);
+                        }
+                        else
+                        {
+                            logger.LogInformation("Failed to get a subset of scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
+                        }
+                    }
+                }
+
+                var nativeScrobbles = scrobbles
+                    .DistinctBy(s => s.TimePlayed)
+                    .OrderBy(s => s.TimePlayed)
+                    .Select(s =>
+                    {
+                        var nativeScrobble = (UserScrobble) s;
+                        nativeScrobble.UserId = config.User.Id;
+                        return nativeScrobble;
+                    });
+                
+                var currentScrobbles = db.Scrobble
+                    .AsEnumerable()
+                    .OrderBy(s => s.Timestamp)
+                    .Where(s => s.UserId == config.User.Id);
+
+                if (config.From is not null)
+                {
+                    currentScrobbles = currentScrobbles.Where(s => s.Timestamp > config.From);
+                }
+
+                if (config.To is not null)
+                {
+                    currentScrobbles = currentScrobbles.Where(s => s.Timestamp < config.To);
+                }
+
+                (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffsContains(currentScrobbles, nativeScrobbles);
+
+                await db.Scrobble.AddRangeAsync(toAdd.Cast<UserScrobble>());
+                db.Scrobble.RemoveRange(toRemove.Cast<UserScrobble>());
+                await db.SaveChangesAsync();
+            }
+            else
+            {
+                logger.LogError("Failed to pull first scrobble page for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
+            }
+        }
+
+        private async Task<List<Task<PageResponse<LastTrack>>>> GetScrobblesFromPageNumbers(IEnumerable<int> pageNumbers, CancellationToken token)
+        {
+            var tasks = new List<Task<PageResponse<LastTrack>>>();
+
+            foreach (var pageNumber in pageNumbers)
+            {
+                logger.LogInformation("Pulling page {2} for {0}/{1}", config.User.UserName, config.User.LastFmUsername, pageNumber);
+                
+                tasks.Add(userClient.GetRecentScrobbles(config.User.LastFmUsername, pagenumber: pageNumber, count: config.PageSize, from: config.From, to: config.To));
+                await Task.Delay(config.InterRequestDelay, token);
+            }
+
+            return tasks;
+        }
+    }
+}

Selector.CLI/Services/DbWatcherService.cs

@@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging;
 
 using Selector.Cache;
 using Selector.Model;
+using Selector.Model.Extensions;
 using Selector.Events;
 using System.Collections.Concurrent;
 
@@ -131,7 +132,7 @@ namespace Selector.CLI
 
                     if (UserEventFirerFactory is not null) consumers.Add(await UserEventFirerFactory.Get());
 
-                    if (!string.IsNullOrWhiteSpace(dbWatcher.User.LastFmUsername))
+                    if (dbWatcher.User.LastFmConnected())
                     {
                         consumers.Add(await PlayCounterFactory.Get(creds: new() { Username = dbWatcher.User.LastFmUsername }));
                     }

Selector.CLI/Services/ScrobbleMonitor.cs

@@ -0,0 +1,67 @@
+using IF.Lastfm.Core.Api;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Selector.Model;
+using Selector.Model.Extensions;
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Selector.CLI.Services
+{
+    public class ScrobbleMonitor : IHostedService
+    {
+        private readonly ILogger<ScrobbleMonitor> logger;
+        private readonly ILoggerFactory loggerFactory;
+        private readonly ScrobbleMonitorOptions config;
+        private readonly IUserApi userApi;
+        private readonly IServiceScopeFactory serviceScopeFactory;
+        private Task task;
+
+        public ScrobbleMonitor(ILogger<ScrobbleMonitor> _logger, IOptions<ScrobbleMonitorOptions> _options, IUserApi _userApi, IServiceScopeFactory _serviceScopeFactory, ILoggerFactory _loggerFactory)
+        {
+            logger = _logger;
+            userApi = _userApi;
+            config = _options.Value;
+            serviceScopeFactory = _serviceScopeFactory;
+            loggerFactory = _loggerFactory;
+        }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            logger.LogInformation("Starting scrobble monitor");
+
+            using var scope = serviceScopeFactory.CreateScope();
+            using var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
+
+            await RunScrobbleSavers(db, cancellationToken);
+        }
+
+        public async Task RunScrobbleSavers(ApplicationDbContext db, CancellationToken token)
+        {
+            foreach (var user in db.Users
+                .AsNoTracking()
+                .AsEnumerable()
+                .Where(u => u.ScrobbleSavingEnabled()))
+            {
+                logger.LogInformation("Starting scrobble saver for {0}/{1}", user.UserName, user.LastFmUsername);
+
+                await new ScrobbleSaver(userApi, new ScrobbleSaverConfig()
+                {
+                    User = user,
+                    InterRequestDelay = config.InterRequestDelay,
+                    From = DateTime.UtcNow.AddDays(-3)
+                }, serviceScopeFactory, loggerFactory.CreateLogger<ScrobbleSaver>()).Execute(token);
+            }
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            return Task.CompletedTask;
+        }
+    }
+}

Selector.CLI/appsettings.json

@@ -4,6 +4,7 @@
         "ClientSecret": "",
         "Equality": "uri",
         "Watcher": {
+            "enabled": false,
             "localenabled": false,
             "Instances": [
                 {
@@ -18,7 +19,7 @@
             "enabled": true
         },
         "Redis": {
-            "enabled": true
+            "enabled": false
         }
     },
     "Logging": {

Selector.Model/ApplicationDbContext.cs

@@ -19,6 +19,10 @@ namespace Selector.Model
         private readonly ILogger<ApplicationDbContext> Logger;
 
         public DbSet<Watcher> Watcher { get; set; }
+        public DbSet<UserScrobble> Scrobble { get; set; }
+        public DbSet<TrackLastfmSpotifyMapping> TrackMapping { get; set; }
+        public DbSet<AlbumLastfmSpotifyMapping> AlbumMapping { get; set; }
+        public DbSet<ArtistLastfmSpotifyMapping> ArtistMapping { get; set; }
 
         public ApplicationDbContext(
             DbContextOptions<ApplicationDbContext> options, 
@@ -37,11 +41,56 @@ namespace Selector.Model
         {
             base.OnModelCreating(modelBuilder);
 
+            modelBuilder.HasCollation("case_insensitive", locale: "en-u-ks-primary", provider: "icu", deterministic: false);
+
+            modelBuilder.Entity<ApplicationUser>()
+                .Property(u => u.SpotifyIsLinked)
+                .IsRequired();
+            modelBuilder.Entity<ApplicationUser>()
+                .Property(u => u.LastFmUsername)
+                .UseCollation("case_insensitive");
+
             modelBuilder.Entity<Watcher>()
                 .HasOne(w => w.User)
                 .WithMany(u => u.Watchers)
                 .HasForeignKey(w => w.UserId);
 
+            modelBuilder.Entity<UserScrobble>().HasKey(s => new { s.UserId, s.Timestamp });
+            modelBuilder.Entity<UserScrobble>()
+                .HasOne(w => w.User)
+                .WithMany(u => u.Scrobbles)
+                .HasForeignKey(w => w.UserId);
+            modelBuilder.Entity<UserScrobble>()
+                .Property(s => s.TrackName)
+                .UseCollation("case_insensitive");
+            modelBuilder.Entity<UserScrobble>()
+                .Property(s => s.AlbumName)
+                .UseCollation("case_insensitive");
+            modelBuilder.Entity<UserScrobble>()
+                .Property(s => s.ArtistName)
+                .UseCollation("case_insensitive");
+
+            modelBuilder.Entity<TrackLastfmSpotifyMapping>().HasKey(s => s.Id);
+            modelBuilder.Entity<TrackLastfmSpotifyMapping>()
+                .Property(s => s.LastfmTrackName)
+                .UseCollation("case_insensitive");
+            modelBuilder.Entity<TrackLastfmSpotifyMapping>()
+                .Property(s => s.LastfmArtistName)
+                .UseCollation("case_insensitive");
+
+            modelBuilder.Entity<AlbumLastfmSpotifyMapping>().HasKey(s => s.Id);
+            modelBuilder.Entity<AlbumLastfmSpotifyMapping>()
+                .Property(s => s.LastfmAlbumName)
+                .UseCollation("case_insensitive");
+            modelBuilder.Entity<AlbumLastfmSpotifyMapping>()
+                .Property(s => s.LastfmArtistName)
+                .UseCollation("case_insensitive");
+
+            modelBuilder.Entity<ArtistLastfmSpotifyMapping>().HasKey(s => s.Id);
+            modelBuilder.Entity<ArtistLastfmSpotifyMapping>()
+                .Property(s => s.LastfmArtistName)
+                .UseCollation("case_insensitive");
+
             SeedData.Seed(modelBuilder);
         }
 

Selector.Model/ApplicationUser.cs

@@ -8,7 +8,6 @@ namespace Selector.Model
 {
     public class ApplicationUser : IdentityUser
     {
-        [Required]
         public bool SpotifyIsLinked { get; set; }
         public DateTime SpotifyLastRefresh { get; set; }
         public int SpotifyTokenExpiry { get; set; }
@@ -16,8 +15,10 @@ namespace Selector.Model
         public string SpotifyRefreshToken { get; set; }
 
         public string LastFmUsername { get; set; }
+        public bool SaveScrobbles { get; set; }
 
         public List<Watcher> Watchers { get; set; }
+        public List<UserScrobble> Scrobbles { get; set; }
     }
 
     public class ApplicationUserDTO

Selector.Model/Extensions/ScrobbleExtensions.cs

@@ -0,0 +1,13 @@
+
+namespace Selector.Model.Extensions
+{
+    public static class ScrobbleExtensions
+    {
+        /// <summary>
+        /// Helper to get the <see cref="Scrobble.AlbumArtistName"/> if possible, if not fall back to artist name 
+        /// </summary>
+        /// <param name="scrobble"></param>
+        /// <returns></returns>
+        public static string AlbumArtist(this Scrobble scrobble) => scrobble.AlbumArtistName ?? scrobble.ArtistName;
+    }
+}

Selector.Model/Extensions/UserExtensions.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Selector.Model.Extensions
+{
+    public static class UserExtensions
+    {
+        public static bool LastFmConnected(this ApplicationUser user)
+            => !string.IsNullOrEmpty(user.LastFmUsername);
+
+        public static bool ScrobbleSavingEnabled(this ApplicationUser user)
+            => user.LastFmConnected() && user.SaveScrobbles;
+    }
+}

Selector.Model/LastfmSpotifyMapping.cs

@@ -0,0 +1,26 @@
+namespace Selector.Model
+{
+    public abstract class LastfmSpotifyMapping
+    {
+        public int Id { get; set; }
+
+        public string SpotifyUri { get; set; }
+    }
+
+    public class TrackLastfmSpotifyMapping: LastfmSpotifyMapping
+    {
+        public string LastfmTrackName { get; set; }
+        public string LastfmArtistName { get; set; }
+    }
+
+    public class AlbumLastfmSpotifyMapping : LastfmSpotifyMapping
+    {
+        public string LastfmAlbumName { get; set; }
+        public string LastfmArtistName { get; set; }
+    }
+
+    public class ArtistLastfmSpotifyMapping : LastfmSpotifyMapping
+    {
+        public string LastfmArtistName { get; set; }
+    }
+}

Selector.Model/Migrations/20220216214339_scrobble_and_lastfm_spotify_mappings.Designer.cs

@@ -0,0 +1,455 @@
+// <auto-generated />
+using System;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+using Selector.Model;
+
+#nullable disable
+
+namespace Selector.Model.Migrations
+{
+    [DbContext(typeof(ApplicationDbContext))]
+    [Migration("20220216214339_scrobble_and_lastfm_spotify_mappings")]
+    partial class scrobble_and_lastfm_spotify_mappings
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False")
+                .HasAnnotation("ProductVersion", "6.0.2")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b =>
+                {
+                    b.Property<string>("Id")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ConcurrencyStamp")
+                        .IsConcurrencyToken()
+                        .HasColumnType("text");
+
+                    b.Property<string>("Name")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<string>("NormalizedName")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("NormalizedName")
+                        .IsUnique()
+                        .HasDatabaseName("RoleNameIndex");
+
+                    b.ToTable("AspNetRoles", (string)null);
+
+                    b.HasData(
+                        new
+                        {
+                            Id = "00c64c0a-3387-4933-9575-83443fa9092b",
+                            ConcurrencyStamp = "b91c880d-e280-4a17-a528-d34fdc35f291",
+                            Name = "Admin",
+                            NormalizedName = "ADMIN"
+                        });
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("ClaimType")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ClaimValue")
+                        .HasColumnType("text");
+
+                    b.Property<string>("RoleId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("RoleId");
+
+                    b.ToTable("AspNetRoleClaims", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("ClaimType")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ClaimValue")
+                        .HasColumnType("text");
+
+                    b.Property<string>("UserId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("AspNetUserClaims", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
+                {
+                    b.Property<string>("LoginProvider")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ProviderKey")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ProviderDisplayName")
+                        .HasColumnType("text");
+
+                    b.Property<string>("UserId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("LoginProvider", "ProviderKey");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("AspNetUserLogins", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
+                {
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.Property<string>("RoleId")
+                        .HasColumnType("text");
+
+                    b.HasKey("UserId", "RoleId");
+
+                    b.HasIndex("RoleId");
+
+                    b.ToTable("AspNetUserRoles", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
+                {
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.Property<string>("LoginProvider")
+                        .HasColumnType("text");
+
+                    b.Property<string>("Name")
+                        .HasColumnType("text");
+
+                    b.Property<string>("Value")
+                        .HasColumnType("text");
+
+                    b.HasKey("UserId", "LoginProvider", "Name");
+
+                    b.ToTable("AspNetUserTokens", (string)null);
+                });
+
+            modelBuilder.Entity("Selector.Model.AlbumLastfmSpotifyMapping", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("LastfmAlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("AlbumMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
+                {
+                    b.Property<string>("Id")
+                        .HasColumnType("text");
+
+                    b.Property<int>("AccessFailedCount")
+                        .HasColumnType("integer");
+
+                    b.Property<string>("ConcurrencyStamp")
+                        .IsConcurrencyToken()
+                        .HasColumnType("text");
+
+                    b.Property<string>("Email")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<bool>("EmailConfirmed")
+                        .HasColumnType("boolean");
+
+                    b.Property<string>("LastFmUsername")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<bool>("LockoutEnabled")
+                        .HasColumnType("boolean");
+
+                    b.Property<DateTimeOffset?>("LockoutEnd")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("NormalizedEmail")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<string>("NormalizedUserName")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<string>("PasswordHash")
+                        .HasColumnType("text");
+
+                    b.Property<string>("PhoneNumber")
+                        .HasColumnType("text");
+
+                    b.Property<bool>("PhoneNumberConfirmed")
+                        .HasColumnType("boolean");
+
+                    b.Property<bool>("SaveScrobbles")
+                        .HasColumnType("boolean");
+
+                    b.Property<string>("SecurityStamp")
+                        .HasColumnType("text");
+
+                    b.Property<string>("SpotifyAccessToken")
+                        .HasColumnType("text");
+
+                    b.Property<bool>("SpotifyIsLinked")
+                        .HasColumnType("boolean");
+
+                    b.Property<DateTime>("SpotifyLastRefresh")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("SpotifyRefreshToken")
+                        .HasColumnType("text");
+
+                    b.Property<int>("SpotifyTokenExpiry")
+                        .HasColumnType("integer");
+
+                    b.Property<bool>("TwoFactorEnabled")
+                        .HasColumnType("boolean");
+
+                    b.Property<string>("UserName")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("NormalizedEmail")
+                        .HasDatabaseName("EmailIndex");
+
+                    b.HasIndex("NormalizedUserName")
+                        .IsUnique()
+                        .HasDatabaseName("UserNameIndex");
+
+                    b.ToTable("AspNetUsers", (string)null);
+                });
+
+            modelBuilder.Entity("Selector.Model.ArtistLastfmSpotifyMapping", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("ArtistMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("LastfmTrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("TrackMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.UserScrobble", b =>
+                {
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.Property<DateTime>("Timestamp")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("AlbumArtistName")
+                        .HasColumnType("text");
+
+                    b.Property<string>("AlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("ArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("TrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.HasKey("UserId", "Timestamp");
+
+                    b.ToTable("Scrobble");
+                });
+
+            modelBuilder.Entity("Selector.Model.Watcher", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<int>("Type")
+                        .HasColumnType("integer");
+
+                    b.Property<string>("UserId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("Watcher");
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
+                {
+                    b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
+                        .WithMany()
+                        .HasForeignKey("RoleId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
+                {
+                    b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
+                        .WithMany()
+                        .HasForeignKey("RoleId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Selector.Model.UserScrobble", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", "User")
+                        .WithMany("Scrobbles")
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("Selector.Model.Watcher", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", "User")
+                        .WithMany("Watchers")
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
+                {
+                    b.Navigation("Scrobbles");
+
+                    b.Navigation("Watchers");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

Selector.Model/Migrations/20220216214339_scrobble_and_lastfm_spotify_mappings.cs

@@ -0,0 +1,146 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace Selector.Model.Migrations
+{
+    public partial class scrobble_and_lastfm_spotify_mappings : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AlterDatabase()
+                .Annotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "LastFmUsername",
+                table: "AspNetUsers",
+                type: "text",
+                nullable: true,
+                collation: "case_insensitive",
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true);
+
+            migrationBuilder.AddColumn<bool>(
+                name: "SaveScrobbles",
+                table: "AspNetUsers",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false);
+
+            migrationBuilder.CreateTable(
+                name: "AlbumMapping",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
+                    LastfmAlbumName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    LastfmArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    SpotifyUri = table.Column<string>(type: "text", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_AlbumMapping", x => x.Id);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "ArtistMapping",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
+                    LastfmArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    SpotifyUri = table.Column<string>(type: "text", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_ArtistMapping", x => x.Id);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "Scrobble",
+                columns: table => new
+                {
+                    Timestamp = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
+                    UserId = table.Column<string>(type: "text", nullable: false),
+                    TrackName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    AlbumName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    AlbumArtistName = table.Column<string>(type: "text", nullable: true),
+                    ArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive")
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_Scrobble", x => new { x.UserId, x.Timestamp });
+                    table.ForeignKey(
+                        name: "FK_Scrobble_AspNetUsers_UserId",
+                        column: x => x.UserId,
+                        principalTable: "AspNetUsers",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "TrackMapping",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
+                    LastfmTrackName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    LastfmArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    SpotifyUri = table.Column<string>(type: "text", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_TrackMapping", x => x.Id);
+                });
+
+            migrationBuilder.UpdateData(
+                table: "AspNetRoles",
+                keyColumn: "Id",
+                keyValue: "00c64c0a-3387-4933-9575-83443fa9092b",
+                column: "ConcurrencyStamp",
+                value: "b91c880d-e280-4a17-a528-d34fdc35f291");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "AlbumMapping");
+
+            migrationBuilder.DropTable(
+                name: "ArtistMapping");
+
+            migrationBuilder.DropTable(
+                name: "Scrobble");
+
+            migrationBuilder.DropTable(
+                name: "TrackMapping");
+
+            migrationBuilder.DropColumn(
+                name: "SaveScrobbles",
+                table: "AspNetUsers");
+
+            migrationBuilder.AlterDatabase()
+                .OldAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "LastFmUsername",
+                table: "AspNetUsers",
+                type: "text",
+                nullable: true,
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true,
+                oldCollation: "case_insensitive");
+
+            migrationBuilder.UpdateData(
+                table: "AspNetRoles",
+                keyColumn: "Id",
+                keyValue: "00c64c0a-3387-4933-9575-83443fa9092b",
+                column: "ConcurrencyStamp",
+                value: "0801d9f2-0f90-4ca7-bb85-eaa36046fc86");
+        }
+    }
+}

Selector.Model/Migrations/ApplicationDbContextModelSnapshot.cs

@@ -17,7 +17,8 @@ namespace Selector.Model.Migrations
         {
 #pragma warning disable 612, 618
             modelBuilder
-                .HasAnnotation("ProductVersion", "6.0.0")
+                .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False")
+                .HasAnnotation("ProductVersion", "6.0.2")
                 .HasAnnotation("Relational:MaxIdentifierLength", 63);
 
             NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@@ -51,7 +52,7 @@ namespace Selector.Model.Migrations
                         new
                         {
                             Id = "00c64c0a-3387-4933-9575-83443fa9092b",
-                            ConcurrencyStamp = "0801d9f2-0f90-4ca7-bb85-eaa36046fc86",
+                            ConcurrencyStamp = "b91c880d-e280-4a17-a528-d34fdc35f291",
                             Name = "Admin",
                             NormalizedName = "ADMIN"
                         });
@@ -163,6 +164,30 @@ namespace Selector.Model.Migrations
                     b.ToTable("AspNetUserTokens", (string)null);
                 });
 
+            modelBuilder.Entity("Selector.Model.AlbumLastfmSpotifyMapping", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("LastfmAlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("AlbumMapping");
+                });
+
             modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
                 {
                     b.Property<string>("Id")
@@ -183,7 +208,8 @@ namespace Selector.Model.Migrations
                         .HasColumnType("boolean");
 
                     b.Property<string>("LastFmUsername")
-                        .HasColumnType("text");
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
 
                     b.Property<bool>("LockoutEnabled")
                         .HasColumnType("boolean");
@@ -208,6 +234,9 @@ namespace Selector.Model.Migrations
                     b.Property<bool>("PhoneNumberConfirmed")
                         .HasColumnType("boolean");
 
+                    b.Property<bool>("SaveScrobbles")
+                        .HasColumnType("boolean");
+
                     b.Property<string>("SecurityStamp")
                         .HasColumnType("text");
 
@@ -245,6 +274,78 @@ namespace Selector.Model.Migrations
                     b.ToTable("AspNetUsers", (string)null);
                 });
 
+            modelBuilder.Entity("Selector.Model.ArtistLastfmSpotifyMapping", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("ArtistMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("LastfmTrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("TrackMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.UserScrobble", b =>
+                {
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.Property<DateTime>("Timestamp")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("AlbumArtistName")
+                        .HasColumnType("text");
+
+                    b.Property<string>("AlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("ArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("TrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.HasKey("UserId", "Timestamp");
+
+                    b.ToTable("Scrobble");
+                });
+
             modelBuilder.Entity("Selector.Model.Watcher", b =>
                 {
                     b.Property<int>("Id")
@@ -318,6 +419,17 @@ namespace Selector.Model.Migrations
                         .IsRequired();
                 });
 
+            modelBuilder.Entity("Selector.Model.UserScrobble", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", "User")
+                        .WithMany("Scrobbles")
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("User");
+                });
+
             modelBuilder.Entity("Selector.Model.Watcher", b =>
                 {
                     b.HasOne("Selector.Model.ApplicationUser", "User")
@@ -331,6 +443,8 @@ namespace Selector.Model.Migrations
 
             modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
                 {
+                    b.Navigation("Scrobbles");
+
                     b.Navigation("Watchers");
                 });
 #pragma warning restore 612, 618

Selector.Model/Services/MigratorService.cs

@@ -1,7 +1,9 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -9,13 +11,13 @@ namespace Selector.Model.Services
 {
     public class MigratorService : IHostedService
     {
-        private readonly ApplicationDbContext context;
+        private readonly IServiceScopeFactory scopeProvider;
         private readonly DatabaseOptions options;
         private readonly ILogger<MigratorService> logger;
 
-        public MigratorService(ApplicationDbContext _context, IOptions<DatabaseOptions> _options, ILogger<MigratorService> _logger)
+        public MigratorService(IServiceScopeFactory _scopeProvider, IOptions<DatabaseOptions> _options, ILogger<MigratorService> _logger)
         {
-            context = _context;
+            scopeProvider = _scopeProvider;
             options = _options.Value;
             logger = _logger;
         }
@@ -24,8 +26,10 @@ namespace Selector.Model.Services
         {
             if(options.Migrate)
             {
+                using var scope = scopeProvider.CreateScope();
+                
                 logger.LogInformation("Applying migrations");
-                context.Database.Migrate();
+                scope.ServiceProvider.GetRequiredService<ApplicationDbContext>().Database.Migrate();
             }
 
             return Task.CompletedTask;

Selector.Model/UserScrobble.cs

@@ -0,0 +1,20 @@
+using System;
+
+namespace Selector.Model
+{
+    public class UserScrobble: Scrobble
+    {
+        public string UserId { get; set; }
+        public ApplicationUser User { get; set; }
+
+        public static explicit operator UserScrobble(IF.Lastfm.Core.Objects.LastTrack track) => new()
+        {
+            Timestamp = track.TimePlayed?.UtcDateTime ?? DateTime.MinValue,
+
+            TrackName = track.Name,
+            AlbumName = track.AlbumName,
+            ArtistName = track.ArtistName,
+        };
+
+    }
+}

Selector.Web/Hubs/NowPlayingHub.cs

@@ -11,6 +11,7 @@ using StackExchange.Redis;
 
 using Selector.Cache;
 using Selector.Model;
+using Selector.Model.Extensions;
 
 namespace Selector.Web.Hubs
 {
@@ -88,7 +89,7 @@ namespace Selector.Web.Hubs
                         .SingleOrDefault()
                             ?? throw new SqlNullValueException("No user returned");
 
-                if(!string.IsNullOrWhiteSpace(user.LastFmUsername))
+                if(user.LastFmConnected())
                 {
                     var playCount = await PlayCountPuller.Get(user.LastFmUsername, track, artist, album, albumArtist);
 

Selector/Scrobble/Scrobble.cs

@@ -0,0 +1,25 @@
+using System;
+
+namespace Selector
+{
+    public class Scrobble
+    {
+        public DateTime Timestamp { get; set; }
+        public string TrackName { get; set; }
+        public string AlbumName { get; set; }
+        /// <summary>
+        /// Not populated by default from the service, where not the same as <see cref="ArtistName"/> these have been manually entered
+        /// </summary>
+        public string AlbumArtistName { get; set; }
+        public string ArtistName { get; set; }
+        
+        public static explicit operator Scrobble(IF.Lastfm.Core.Objects.LastTrack track) => new()
+        {
+            Timestamp = track.TimePlayed?.UtcDateTime ?? DateTime.MinValue,
+
+            TrackName = track.Name,
+            AlbumName = track.AlbumName,
+            ArtistName = track.ArtistName,
+        };
+    }
+}

Selector/Scrobble/ScrobbleMatcher.cs

@@ -0,0 +1,65 @@
+using IF.Lastfm.Core.Objects;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+namespace Selector
+{
+    public static class ScrobbleMatcher
+    {
+        public static bool MatchTime(Scrobble nativeScrobble, LastTrack serviceScrobble) 
+            => serviceScrobble.TimePlayed.Equals(nativeScrobble);
+
+        public static bool MatchTime(Scrobble nativeScrobble, Scrobble serviceScrobble)
+            => serviceScrobble.Timestamp.Equals(nativeScrobble);
+
+        public static (IEnumerable<Scrobble>, IEnumerable<Scrobble>) IdentifyDiffs(IEnumerable<Scrobble> existing, IEnumerable<Scrobble> toApply)
+        {
+            existing = existing.OrderBy(s => s.Timestamp);
+            toApply = toApply.OrderBy(s => s.Timestamp);
+            var toApplyIter = toApply.GetEnumerator();
+
+            var toAdd = new List<Scrobble>();
+            var toRemove = new List<Scrobble>();
+
+            if(existing.Any())
+            {
+                foreach (var currentExisting in existing)
+                {
+                    while (toApplyIter.Current.Timestamp < currentExisting.Timestamp)
+                    {
+                        toAdd.Add(toApplyIter.Current);
+
+                        if (!toApplyIter.MoveNext()) break;
+                    }
+
+                    if (!MatchTime(currentExisting, toApplyIter.Current))
+                    {
+                        toRemove.Add(currentExisting);
+                    }
+                }
+            }
+            else
+            {
+                toAdd.AddRange(toApply);
+            }
+
+            return (toAdd, toRemove);
+        }
+
+        public static (IEnumerable<Scrobble>, IEnumerable<Scrobble>) IdentifyDiffsContains(IEnumerable<Scrobble> existing, IEnumerable<Scrobble> toApply)
+        {
+            var toAdd = toApply.Where(s => !existing.Contains(s, new ScrobbleComp()));
+            var toRemove = existing.Where(s => !toApply.Contains(s, new ScrobbleComp()));
+
+            return (toAdd, toRemove);
+        }
+
+        public class ScrobbleComp : IEqualityComparer<Scrobble>
+        {
+            public bool Equals(Scrobble x, Scrobble y) => x.Timestamp == y.Timestamp;
+
+            public int GetHashCode([DisallowNull] Scrobble obj) => obj.Timestamp.GetHashCode();
+        }
+    }
+}