working scrobble mapping command

added scrobble mapping repo

Selector.CLI/Command/CommandContext.cs

@@ -2,6 +2,7 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 using Selector.Model;
+using SpotifyAPI.Web;
 
 namespace Selector.CLI
 {
@@ -9,6 +10,7 @@ namespace Selector.CLI
     {
         public RootOptions Config { get; set; }
         public ILoggerFactory Logger { get; set; }
+        public ISpotifyClient Spotify{ get; set; }
 
         public DbContextOptionsBuilder<ApplicationDbContext> DatabaseConfig { get; set; }        
         public LastfmClient LastFmClient { get; set; }

Selector.CLI/Command/Scrobble/Scrobble.cs

@@ -11,6 +11,9 @@ namespace Selector.CLI
 
             var clearCommand = new ScrobbleClearCommand("clear", "clear user scrobbles");
             AddCommand(clearCommand);
+
+            var mapCommand = new ScrobbleMapCommand("map", "map last.fm data to spotify uris");
+            AddCommand(mapCommand);
         }
     }
 }

Selector.CLI/Command/Scrobble/ScrobbleMap.cs

@@ -0,0 +1,81 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Selector.CLI.Extensions;
+using Selector.Model;
+using Selector.Model.Extensions;
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Selector.CLI
+{
+    public class ScrobbleMapCommand : Command
+    {
+        public ScrobbleMapCommand(string name, string description = null) : base(name, description)
+        {
+            var delayOption = new Option<int>("--delay", getDefaultValue: () => 100, "milliseconds to delay");
+            delayOption.AddAlias("-d");
+            AddOption(delayOption);
+
+            var simulOption = new Option<int>("--simultaneous", getDefaultValue: () => 3, "simultaneous connections when pulling");
+            simulOption.AddAlias("-s");
+            AddOption(simulOption);
+
+            var limitOption = new Option<int?>("--limit", "limit number of objects to poll");
+            limitOption.AddAlias("-l");
+            AddOption(limitOption);
+
+            var artists = new Option("--artist", "map scrobble artists to spotify");
+            artists.AddAlias("-ar");
+            AddOption(artists);
+
+            var albums = new Option("--album", "map scrobble albums to spotify");
+            albums.AddAlias("-al");
+            AddOption(albums);
+
+            var tracks = new Option("--track", "map scrobble tracks to spotify");
+            tracks.AddAlias("-tr");
+            AddOption(tracks);
+
+            Handler = CommandHandler.Create(async (int delay, int simultaneous, int? limit, bool artist, bool album, bool track, CancellationToken token) => await Execute(delay, simultaneous, limit, artist, album, track, token));
+        }
+
+        public static async Task<int> Execute(int delay, int simultaneous, int? limit, bool artists, bool albums, bool tracks, CancellationToken token)
+        {
+            try
+            {
+                var context = new CommandContext().WithLogger().WithDb().WithSpotify();
+                var logger = context.Logger.CreateLogger("Scrobble");
+
+                using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger<ApplicationDbContext>());
+
+                await new ScrobbleMapper(
+                    context.Spotify.Search,
+                    new ()
+                    {
+                        InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay),
+                        SimultaneousConnections = simultaneous,
+                        Limit = limit,
+                        Artists = artists,
+                        Albums = albums,
+                        Tracks = tracks
+                    }, 
+                    new ScrobbleRepository(db), 
+                    new ScrobbleMappingRepository(db),
+                    context.Logger.CreateLogger<ScrobbleMapper>(), 
+                    context.Logger)
+                        .Execute(token);
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex);
+                return 1;
+            }
+
+            return 0;
+        }
+    }
+}

Selector.CLI/Command/Scrobble/ScrobbleSave.cs

@@ -57,21 +57,32 @@ namespace Selector.CLI
                 var logger = context.Logger.CreateLogger("Scrobble");
 
                 using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger<ApplicationDbContext>());
+                var repo = new ScrobbleRepository(db);
 
-                logger.LogInformation("Running from {0} to {1}", from, to);
+                logger.LogInformation("Running from {} to {}", from, to);
 
-                logger.LogInformation("Searching for {0}", username);
+                logger.LogInformation("Searching for {}", username);
                 var user = db.Users.AsNoTracking().FirstOrDefault(u => u.UserName == username);
 
                 if (user is not null)
                 {
                     if (user.LastFmConnected())
                     {
-                        logger.LogInformation("Last.fm username found ({0}), starting...", user.LastFmUsername);
+                        logger.LogInformation("Last.fm username found ({}), starting...", user.LastFmUsername);
+
+                        if(from.Kind != DateTimeKind.Utc)
+                        {
+                            from = from.ToUniversalTime();
+                        }
+
+                        if (to.Kind != DateTimeKind.Utc)
+                        {
+                            to = to.ToUniversalTime();
+                        }
 
                         await new ScrobbleSaver(
                             context.LastFmClient.User,
-                            new ScrobbleSaverConfig()
+                            new ()
                             {
                                 User = user,
                                 InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay),
@@ -82,19 +93,19 @@ namespace Selector.CLI
                                 DontRemove = noRemove,
                                 SimultaneousConnections = simul
                             },
-                            db,
+                            repo,
                             context.Logger.CreateLogger<ScrobbleSaver>(), 
                             context.Logger)
                                 .Execute(token);
                     }
                     else
                     {
-                        logger.LogError("{0} doesn't have a Last.fm username", username);
+                        logger.LogError("{} doesn't have a Last.fm username", username);
                     }
                 }
                 else
                 {
-                    logger.LogError("{0} not found", username);
+                    logger.LogError("{} not found", username);
                 }
 
             }

Selector.CLI/Extensions/CommandContextExtensions.cs

@@ -2,7 +2,10 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 using Selector.Model;
+using SpotifyAPI.Web;
+using System.Linq;
 
 namespace Selector.CLI.Extensions
 {
@@ -58,5 +61,33 @@ namespace Selector.CLI.Extensions
 
             return context;
         }
+
+        public static CommandContext WithSpotify(this CommandContext context)
+        {
+            if (context.Config is null)
+            {
+                context.WithConfig();
+            }
+
+            var refreshToken = context.Config.RefreshToken;
+
+            if(string.IsNullOrWhiteSpace(refreshToken))
+            {
+                if (context.DatabaseConfig is null)
+                {
+                    context.WithDb();
+                }
+
+                using var db = new ApplicationDbContext(context.DatabaseConfig.Options, NullLogger<ApplicationDbContext>.Instance);
+
+                refreshToken = db.Users.FirstOrDefault(u => u.UserName == "sarsoo")?.SpotifyRefreshToken;
+            }
+
+            var configFactory = new RefreshTokenFactory(context.Config.ClientId, context.Config.ClientSecret, refreshToken);
+
+            context.Spotify = new SpotifyClient(configFactory.GetConfig().Result);
+
+            return context;
+        }
     }
 }

Selector.CLI/Extensions/ServiceExtensions.cs

@@ -45,6 +45,8 @@ namespace Selector.CLI.Extensions
                     options.UseNpgsql(config.DatabaseOptions.ConnectionString)
                 );
 
+                services.AddTransient<IScrobbleRepository, ScrobbleRepository>();
+
                 services.AddHostedService<MigratorService>();
             }
 

Selector.CLI/Options.cs

@@ -36,6 +36,10 @@ namespace Selector.CLI
         /// Spotify app secret
         /// </summary>
         public string ClientSecret { get; set; }
+        /// <summary>
+        /// Service account refresh token for tool spotify usage
+        /// </summary>
+        public string RefreshToken { get; set; }
         public string LastfmClient { get; set; }
         public string LastfmSecret { get; set; }
         public WatcherOptions WatcherOptions { get; set; } = new();

Selector.CLI/ScrobbleMapper.cs

@@ -0,0 +1,108 @@
+using IF.Lastfm.Core.Api;
+using IF.Lastfm.Core.Api.Helpers;
+using IF.Lastfm.Core.Objects;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Selector.Model;
+using Selector.Operations;
+using SpotifyAPI.Web;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Selector
+{
+    public class ScrobbleMapperConfig
+    {
+        public TimeSpan InterRequestDelay { get; set; }
+        public TimeSpan Timeout { get; set; } = new TimeSpan(0, 20, 0);
+        public int SimultaneousConnections { get; set; } = 3;
+        public int? Limit { get; set; } = null;
+        public bool Tracks { get; set; } = false;
+        public bool Albums { get; set; } = false;
+        public bool Artists { get; set; } = false;
+    }
+
+    public class ScrobbleMapper
+    {
+        private readonly ILogger<ScrobbleMapper> logger;
+        private readonly ILoggerFactory loggerFactory;
+        private readonly ISearchClient searchClient;
+
+        private readonly ScrobbleMapperConfig config;
+
+        private readonly IScrobbleRepository scrobbleRepo;
+        private readonly IScrobbleMappingRepository mappingRepo;
+
+        public ScrobbleMapper(ISearchClient _searchClient, ScrobbleMapperConfig _config, IScrobbleRepository _scrobbleRepository, IScrobbleMappingRepository _scrobbleMappingRepository, ILogger<ScrobbleMapper> _logger, ILoggerFactory _loggerFactory = null)
+        {
+            searchClient = _searchClient;
+            config = _config;
+            scrobbleRepo = _scrobbleRepository;
+            mappingRepo = _scrobbleMappingRepository;
+            logger = _logger;
+            loggerFactory = _loggerFactory;
+        }
+
+        public async Task Execute(CancellationToken token)
+        {
+            if (config.Artists)
+            {
+                logger.LogInformation("Mapping scrobble artists");
+
+                var currentArtists = mappingRepo.GetArtists();
+                var scrobbleArtists = scrobbleRepo.GetAll()
+                    .GroupBy(x => x.ArtistName)
+                    .Select(x => (x.Key, x.Count()))
+                    .OrderByDescending(x => x.Item2)
+                    .Select(x => x.Key);
+
+                if(config.Limit is not null)
+                {
+                    scrobbleArtists = scrobbleArtists.Take(config.Limit.Value);
+                }
+
+                var artistsToPull = scrobbleArtists
+                    .ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a);
+
+
+                var requests = artistsToPull.Select(a => new ScrobbleArtistMapping(
+                     searchClient, 
+                     loggerFactory.CreateLogger<ScrobbleArtistMapping>() ?? NullLogger<ScrobbleArtistMapping>.Instance,
+                     a)
+                ).ToList();
+
+                logger.LogInformation("Found {} artists to map, starting", requests.Count);
+
+                var batchRequest = new BatchingOperation<ScrobbleArtistMapping>(
+                    config.InterRequestDelay, 
+                    config.Timeout, 
+                    config.SimultaneousConnections, 
+                    requests
+                );
+
+                await batchRequest.TriggerRequests(token);
+
+                logger.LogInformation("Finished mapping artists");
+
+                var newArtists = batchRequest.DoneRequests
+                    .Where(a => a is not null)
+                    .Select(a => (FullArtist) a.Result)
+                    .Where(a => a is not null);
+                var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping() { 
+                    LastfmArtistName = a.Name,
+                    SpotifyUri = a.Uri
+                });
+
+                mappingRepo.AddRange(newMappings);
+            }
+
+            await mappingRepo.Save();
+        }
+    }
+}

Selector.CLI/ScrobbleSaver.cs

@@ -1,9 +1,9 @@
 using IF.Lastfm.Core.Api;
-using IF.Lastfm.Core.Api.Helpers;
 using IF.Lastfm.Core.Objects;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Selector.Model;
+using Selector.Operations;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
@@ -36,27 +36,23 @@ namespace Selector
 
         private readonly IUserApi userClient;
         private readonly ScrobbleSaverConfig config;
-        private CancellationToken _token;
+        private Task batchTask;
-        private Task aggregateNetworkTask;
+        private BatchingOperation<ScrobbleRequest> batchOperation;
 
-        private readonly ApplicationDbContext db;
+        private readonly IScrobbleRepository scrobbleRepo;
 
-        private ConcurrentQueue<ScrobbleRequest> waitingRequests = new();
+        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null)
-        private ConcurrentQueue<ScrobbleRequest> runRequests = new();
-
-        public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, ApplicationDbContext _db, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null)
         {
             userClient = _userClient;
             config = _config;
-            db = _db;
+            scrobbleRepo = _scrobbleRepository;
             logger = _logger;
             loggerFactory = _loggerFactory;
         }
 
         public async Task Execute(CancellationToken token)
         {
-            logger.LogInformation("Saving scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
+            logger.LogInformation("Saving scrobbles for {}/{}", config.User.UserName, config.User.LastFmUsername);
-            _token = token;
 
             var page1 = new ScrobbleRequest(userClient, 
                 loggerFactory?.CreateLogger<ScrobbleRequest>() ?? NullLogger<ScrobbleRequest>.Instance, 
@@ -66,21 +62,37 @@ namespace Selector
                 config.From, config.To);
 
             await page1.Execute();
-            runRequests.Enqueue(page1);
 
             if (page1.Succeeded)
             {
                 if (page1.TotalPages > 1)
                 {
-                    TriggerNetworkRequests(page1.TotalPages, token);
+                    batchOperation = new BatchingOperation<ScrobbleRequest>(
+                        config.InterRequestDelay, 
+                        config.Timeout,
+                        config.SimultaneousConnections, 
+                        GetRequestsFromPageNumbers(2, page1.TotalPages)
+                    );
+
+                    batchTask = batchOperation.TriggerRequests(token);
                 }
 
+
                 logger.LogDebug("Pulling currently stored scrobbles");
 
-                var currentScrobbles = GetDbScrobbles();
+                var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To);
 
-                await aggregateNetworkTask;
+                if(batchTask is not null)
-                var scrobbles = runRequests.SelectMany(r => r.Scrobbles);
+                {
+                    await batchTask;
+                }
+
+                var scrobbles = page1.Scrobbles;
+
+                if(batchOperation is not null)
+                {
+                    scrobbles.AddRange(batchOperation.DoneRequests.SelectMany(r => r.Scrobbles));
+                }
 
                 IdentifyDuplicates(scrobbles);
 
@@ -97,7 +109,7 @@ namespace Selector
                         return nativeScrobble;
                     });
 
-                logger.LogInformation("Completed database scrobble pulling for {0}, pulled {1:n0}", config.User.UserName, nativeScrobbles.Count());
+                logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Count());
 
                 logger.LogDebug("Identifying difference sets");
                 var time = Stopwatch.StartNew();
@@ -105,99 +117,39 @@ namespace Selector
                 (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
 
                 time.Stop();
-                logger.LogTrace("Finished diffing: {0:n}ms", time.ElapsedMilliseconds);
+                logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds);
 
                 var timeDbOps = Stopwatch.StartNew();
 
                 if(!config.DontAdd)
                 {
-                    await db.Scrobble.AddRangeAsync(toAdd.Cast<UserScrobble>());
+                    scrobbleRepo.AddRange(toAdd.Cast<UserScrobble>());
                 }
                 else
                 {
-                    logger.LogInformation("Skipping adding of {0} scrobbles", toAdd.Count());
+                    logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count());
                 }
                 if (!config.DontRemove)
                 {
-                    db.Scrobble.RemoveRange(toRemove.Cast<UserScrobble>());
+                    scrobbleRepo.RemoveRange(toRemove.Cast<UserScrobble>());
                 }
                 else
                 {
-                    logger.LogInformation("Skipping removal of {0} scrobbles", toRemove.Count());
+                    logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count());
                 }
-                await db.SaveChangesAsync();
+                await scrobbleRepo.Save();
 
                 timeDbOps.Stop();
-                logger.LogTrace("DB ops: {0:n}ms", timeDbOps.ElapsedMilliseconds);
+                logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds);
 
-                logger.LogInformation("Completed scrobble pulling for {0}, +{1:n0}, -{2:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
+                logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
             }
             else
             {
-                logger.LogError("Failed to pull first scrobble page for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
+                logger.LogError("Failed to pull first scrobble page for {}/{}", config.User.UserName, config.User.LastFmUsername);
             }
         }
 
-        private async void HandleSuccessfulRequest(object o, EventArgs e)
-        {
-            await Task.Delay(config.InterRequestDelay, _token);
-            TransitionRequest();
-        }
-
-        private void TransitionRequest()
-        {
-            if (waitingRequests.TryDequeue(out var request))
-            {
-                request.Success += HandleSuccessfulRequest;
-                _ = request.Execute();
-                runRequests.Enqueue(request);
-            }
-        }
-
-        private void TriggerNetworkRequests(int totalPages, CancellationToken token)
-        {
-            foreach (var req in GetRequestsFromPageNumbers(2, totalPages))
-            {
-                waitingRequests.Enqueue(req);
-            }
-
-            foreach (var _ in Enumerable.Range(1, config.SimultaneousConnections))
-            {
-                TransitionRequest();
-            }
-
-            var timeoutTask = Task.Delay(config.Timeout, token);
-            var allTasks = waitingRequests.Union(runRequests).Select(r => r.Task).ToList();
-
-            aggregateNetworkTask = Task.WhenAny(timeoutTask, Task.WhenAll(allTasks));
-
-            aggregateNetworkTask.ContinueWith(t =>
-            {
-                if (timeoutTask.IsCompleted)
-                {
-                    throw new TimeoutException($"Timed-out pulling scrobbles, took {config.Timeout}");
-                }
-            });            
-        }
-
-        private IEnumerable<UserScrobble> GetDbScrobbles()
-        {
-            var currentScrobbles = db.Scrobble.AsEnumerable()
-                    .Where(s => s.UserId == config.User.Id);
-
-            if (config.From is not null)
-            {
-                currentScrobbles = currentScrobbles.Where(s => s.Timestamp > config.From);
-            }
-
-            if (config.To is not null)
-            {
-                currentScrobbles = currentScrobbles.Where(s => s.Timestamp < config.To);
-            }
-
-            return currentScrobbles;
-        }
-
         private IEnumerable<ScrobbleRequest> GetRequestsFromPageNumbers(int start, int totalPages)
             => Enumerable.Range(start, totalPages - 1)
                          .Select(n => new ScrobbleRequest(
@@ -207,7 +159,8 @@ namespace Selector
                              n, 
                              config.PageSize, 
                              config.From, 
-                             config.To));
+                             config.To,
+                             config.Retries));
 
         private void IdentifyDuplicates(IEnumerable<LastTrack> tracks)
         {
@@ -223,18 +176,16 @@ namespace Selector
 
                 foreach(var scrobble in dupe)
                 {
-                    dupeString.Append("(");
+                    dupeString.Append('(');
                     dupeString.Append(scrobble.Name);
                     dupeString.Append(", ");
                     dupeString.Append(scrobble.AlbumName); 
                     dupeString.Append(", ");
                     dupeString.Append(scrobble.ArtistName);
                     dupeString.Append(") ");
-                    
-                    dupeString.Append(" ");
                 }
 
-                logger.LogInformation("Duplicate at {0}: {1}", dupe.Key, dupeString.ToString());
+                logger.LogInformation("Duplicate at {}: {}", dupe.Key, dupeString.ToString());
             }
         }
 

Selector.Model/Scrobble/IScrobbleMappingRepository.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Selector.Model
+{
+    public interface IScrobbleMappingRepository
+    {
+        void Add(TrackLastfmSpotifyMapping item);
+        void Add(AlbumLastfmSpotifyMapping item);
+        void Add(ArtistLastfmSpotifyMapping item);
+        void AddRange(IEnumerable<TrackLastfmSpotifyMapping> item);
+        void AddRange(IEnumerable<AlbumLastfmSpotifyMapping> item);
+        void AddRange(IEnumerable<ArtistLastfmSpotifyMapping> item);
+        IEnumerable<TrackLastfmSpotifyMapping> GetTracks(string include = null, string trackName = null, string albumName = null, string artistName = null);
+        IEnumerable<AlbumLastfmSpotifyMapping> GetAlbums(string include = null, string albumName = null, string artistName = null);
+        IEnumerable<ArtistLastfmSpotifyMapping> GetArtists(string include = null, string artistName = null);
+        
+        public void Remove(TrackLastfmSpotifyMapping mapping);
+        public void Remove(AlbumLastfmSpotifyMapping mapping);
+        public void Remove(ArtistLastfmSpotifyMapping mapping);
+        public void RemoveRange(IEnumerable<TrackLastfmSpotifyMapping> mappings);
+        public void RemoveRange(IEnumerable<AlbumLastfmSpotifyMapping> mappings);
+        public void RemoveRange(IEnumerable<ArtistLastfmSpotifyMapping> mappings);
+        Task<int> Save();
+    }
+}

Selector.Model/Scrobble/IScrobbleRepository.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Selector.Model
+{
+    public interface IScrobbleRepository
+    {
+        void Add(UserScrobble item);
+        void AddRange(IEnumerable<UserScrobble> item);
+        IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+        UserScrobble Find(int key, string include = null);
+        void Remove(int key);
+        public void Remove(UserScrobble scrobble);
+        public void RemoveRange(IEnumerable<UserScrobble> scrobbles);
+        void Update(UserScrobble item);
+        Task<int> Save();
+    }
+}

Selector.Model/Scrobble/ScrobbleMappingRepository.cs

@@ -0,0 +1,146 @@
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+
+namespace Selector.Model
+{
+    public class ScrobbleMappingRepository : IScrobbleMappingRepository
+    {
+        private readonly ApplicationDbContext db;
+
+        public ScrobbleMappingRepository(ApplicationDbContext context)
+        {
+            db = context;
+        }
+
+        public void Add(TrackLastfmSpotifyMapping item)
+        {
+            db.TrackMapping.Add(item);
+        }
+
+        public void Add(AlbumLastfmSpotifyMapping item)
+        {
+            db.AlbumMapping.Add(item);
+        }
+
+        public void Add(ArtistLastfmSpotifyMapping item)
+        {
+            db.ArtistMapping.Add(item);
+        }
+
+        public void AddRange(IEnumerable<TrackLastfmSpotifyMapping> item)
+        {
+            db.TrackMapping.AddRange(item);
+        }
+
+        public void AddRange(IEnumerable<AlbumLastfmSpotifyMapping> item)
+        {
+            db.AlbumMapping.AddRange(item);
+        }
+
+        public void AddRange(IEnumerable<ArtistLastfmSpotifyMapping> item)
+        {
+            db.ArtistMapping.AddRange(item);
+        }
+
+        public IEnumerable<AlbumLastfmSpotifyMapping> GetAlbums(string include = null, string albumName = null, string artistName = null)
+        {
+            var mappings = db.AlbumMapping.AsQueryable();
+
+            if (!string.IsNullOrWhiteSpace(include))
+            {
+                mappings = mappings.Include(include);
+            }
+
+            if (!string.IsNullOrWhiteSpace(albumName))
+            {
+                mappings = mappings.Where(s => s.LastfmAlbumName == albumName);
+            }
+
+            if (!string.IsNullOrWhiteSpace(artistName))
+            {
+                mappings = mappings.Where(s => s.LastfmArtistName == artistName);
+            }
+
+            return mappings.AsEnumerable();
+        }
+
+        public IEnumerable<ArtistLastfmSpotifyMapping> GetArtists(string include = null, string artistName = null)
+        {
+            var mappings = db.ArtistMapping.AsQueryable();
+
+            if (!string.IsNullOrWhiteSpace(include))
+            {
+                mappings = mappings.Include(include);
+            }
+
+            if (!string.IsNullOrWhiteSpace(artistName))
+            {
+                mappings = mappings.Where(s => s.LastfmArtistName == artistName);
+            }
+
+            return mappings.AsEnumerable();
+        }
+
+        public IEnumerable<TrackLastfmSpotifyMapping> GetTracks(string include = null, string trackName = null, string albumName = null, string artistName = null)
+        {
+            var mappings = db.TrackMapping.AsQueryable();
+
+            if (!string.IsNullOrWhiteSpace(include))
+            {
+                mappings = mappings.Include(include);
+            }
+
+            if (!string.IsNullOrWhiteSpace(trackName))
+            {
+                mappings = mappings.Where(s => s.LastfmTrackName == trackName);
+            }
+
+            if (!string.IsNullOrWhiteSpace(artistName))
+            {
+                mappings = mappings.Where(s => s.LastfmArtistName == artistName);
+            }
+
+            return mappings.AsEnumerable();
+        }
+
+        public void Remove(TrackLastfmSpotifyMapping mapping)
+        {
+            db.TrackMapping.Remove(mapping);
+        }
+
+        public void Remove(AlbumLastfmSpotifyMapping mapping)
+        {
+            db.AlbumMapping.Remove(mapping);
+        }
+
+        public void Remove(ArtistLastfmSpotifyMapping mapping)
+        {
+            db.ArtistMapping.Remove(mapping);
+        }
+
+        public void RemoveRange(IEnumerable<TrackLastfmSpotifyMapping> mappings)
+        {
+            db.TrackMapping.RemoveRange(mappings);
+        }
+
+        public void RemoveRange(IEnumerable<AlbumLastfmSpotifyMapping> mappings)
+        {
+            db.AlbumMapping.RemoveRange(mappings);
+        }
+
+        public void RemoveRange(IEnumerable<ArtistLastfmSpotifyMapping> mappings)
+        {
+            db.ArtistMapping.RemoveRange(mappings);
+        }
+
+        public Task<int> Save()
+        {
+            return db.SaveChangesAsync();
+        }
+    }
+}

Selector.Model/Scrobble/ScrobbleRepository.cs

@@ -0,0 +1,122 @@
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+
+namespace Selector.Model
+{
+    public class ScrobbleRepository : IScrobbleRepository
+    {
+        private readonly ApplicationDbContext db;
+
+        public ScrobbleRepository(ApplicationDbContext context)
+        {
+            db = context;
+        }
+
+        public void Add(UserScrobble item)
+        {
+            db.Scrobble.Add(item);
+        }
+
+        public void AddRange(IEnumerable<UserScrobble> item)
+        {
+            db.Scrobble.AddRange(item);
+        }
+
+        public UserScrobble Find(int key, string include = null)
+        {
+            var scrobbles = db.Scrobble.Where(s => s.Id == key);
+            
+            if (!string.IsNullOrWhiteSpace(include))
+            {
+                scrobbles = scrobbles.Include(include);
+            }
+                
+            return scrobbles.FirstOrDefault();
+        }
+
+        public IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        {
+            var scrobbles = db.Scrobble.AsQueryable();
+
+            if (!string.IsNullOrWhiteSpace(include))
+            {
+                scrobbles = scrobbles.Include(include);
+            }
+
+            if (!string.IsNullOrWhiteSpace(userId))
+            {
+                scrobbles = scrobbles.Where(s => s.UserId == userId);
+            }
+
+            if (!string.IsNullOrWhiteSpace(username))
+            {
+                var user = db.Users.AsNoTracking().Where(u => u.UserName.Equals(username, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
+                if (user is not null)
+                {
+                    scrobbles = scrobbles.Where(s => s.UserId == user.Id);
+                }
+                else
+                {
+                    scrobbles = Enumerable.Empty<UserScrobble>().AsQueryable();
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(trackName))
+            {
+                scrobbles = scrobbles.Where(s => s.TrackName == trackName);
+            }
+
+            if (!string.IsNullOrWhiteSpace(albumName))
+            {
+                scrobbles = scrobbles.Where(s => s.AlbumName == albumName);
+            }
+
+            if (!string.IsNullOrWhiteSpace(artistName))
+            {
+                scrobbles = scrobbles.Where(s => s.ArtistName == artistName);
+            }
+
+            if (from is not null)
+            {
+                scrobbles = scrobbles.Where(u => u.Timestamp >= from.Value);
+            }
+
+            if (to is not null)
+            {
+                scrobbles = scrobbles.Where(u => u.Timestamp < to.Value);
+            }
+
+            return scrobbles.AsEnumerable();
+        }
+
+        public void Remove(int key)
+        {
+            Remove(Find(key));
+        }
+
+        public void Remove(UserScrobble scrobble)
+        {
+            db.Scrobble.Remove(scrobble);
+        }
+
+        public void RemoveRange(IEnumerable<UserScrobble> scrobbles)
+        {
+            db.Scrobble.RemoveRange(scrobbles);
+        }
+
+        public void Update(UserScrobble item)
+        {
+            db.Scrobble.Update(item);
+        }
+
+        public Task<int> Save()
+        {
+            return db.SaveChangesAsync();
+        }
+    }
+}

Selector/Operations/BatchingOperation.cs

@@ -0,0 +1,74 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Selector.Operations
+{ 
+    public class BatchingOperation<T> where T : IOperation
+    {
+        protected ILogger<BatchingOperation<T>> logger;
+        protected CancellationToken _token;
+        protected Task aggregateNetworkTask;
+
+        public ConcurrentQueue<T> WaitingRequests { get; private set; } = new();
+        public ConcurrentQueue<T> DoneRequests { get; private set; } = new();
+
+        private TimeSpan interRequestDelay;
+        private TimeSpan timeout;
+        private int simultaneousRequests;
+
+        public BatchingOperation(TimeSpan _interRequestDelay, TimeSpan _timeout, int _simultaneous, IEnumerable<T> requests, ILogger<BatchingOperation<T>> _logger = null)
+        {
+            interRequestDelay = _interRequestDelay;
+            timeout = _timeout;
+            simultaneousRequests = _simultaneous;
+            logger = _logger ?? NullLogger<BatchingOperation<T>>.Instance;
+
+            foreach(var request in requests)
+            {
+                WaitingRequests.Enqueue(request);
+            }
+        }
+
+        public bool TimedOut { get; private set; } = false;
+
+        private async void HandleSuccessfulRequest(object o, EventArgs e)
+        {
+            await Task.Delay(interRequestDelay, _token);
+            TransitionRequest();
+        }
+
+        private void TransitionRequest()
+        {
+            if (WaitingRequests.TryDequeue(out var request))
+            {
+                request.Success += HandleSuccessfulRequest;
+                _ = request.Execute();
+                DoneRequests.Enqueue(request);
+
+                logger.LogInformation("Executing request {} of {}", DoneRequests.Count, WaitingRequests.Count + DoneRequests.Count);
+            }
+        }
+
+        public async Task TriggerRequests(CancellationToken token)
+        {
+            foreach (var _ in Enumerable.Range(1, simultaneousRequests))
+            {
+                TransitionRequest();
+            }
+
+            var timeoutTask = Task.Delay(timeout, token);
+            var allTasks = WaitingRequests.Union(DoneRequests).Select(r => r.Task).ToList();
+
+            var firstToFinish = await Task.WhenAny(timeoutTask, Task.WhenAll(allTasks));
+
+            TimedOut = firstToFinish == timeoutTask;
+        }
+    }
+}

Selector/Operations/IOperation.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Selector.Operations
+{
+    public interface IOperation
+    {
+        event EventHandler Success;
+        Task Execute();
+        Task Task { get; }
+    }
+}

Selector/Scrobble/ScrobbleAlbumMapping.cs

@@ -0,0 +1,39 @@
+using Microsoft.Extensions.Logging;
+using SpotifyAPI.Web;
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace Selector
+{
+    public class ScrobbleAlbumMapping : ScrobbleMapping
+    {
+        public string AlbumName { get; set; }
+        public string ArtistName { get; set; }
+
+        public ScrobbleAlbumMapping(ISearchClient _searchClient, ILogger<ScrobbleAlbumMapping> _logger, string albumName, string artistName) : base(_searchClient, _logger)
+        {
+            AlbumName = albumName;
+            ArtistName = artistName;
+        }
+
+        private SimpleAlbum result;
+        public override object Result => result;
+
+        public override string Query => $"{AlbumName} {ArtistName}";
+
+        public override SearchRequest.Types QueryType => SearchRequest.Types.Album;
+
+        public override void HandleResponse(Task<SearchResponse> response)
+        {
+            var topResult = response.Result.Albums.Items.FirstOrDefault();
+
+            if (topResult is not null
+                && topResult.Name.Equals(AlbumName, StringComparison.InvariantCultureIgnoreCase)
+                && topResult.Artists.First().Name.Equals(ArtistName, StringComparison.InvariantCultureIgnoreCase))
+            {
+                result = topResult;
+            }
+        }
+    }
+}

Selector/Scrobble/ScrobbleArtistMapping.cs

@@ -0,0 +1,38 @@
+using Microsoft.Extensions.Logging;
+using SpotifyAPI.Web;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Selector
+{
+    public class ScrobbleArtistMapping : ScrobbleMapping
+    {
+        public string ArtistName { get; set; }
+
+        public ScrobbleArtistMapping(ISearchClient _searchClient, ILogger<ScrobbleArtistMapping> _logger, string artistName) : base(_searchClient, _logger)
+        {
+            ArtistName = artistName;
+        }
+
+        private FullArtist result;
+        public override object Result => result;
+
+        public override string Query => ArtistName;
+
+        public override SearchRequest.Types QueryType => SearchRequest.Types.Artist;
+
+        public override void HandleResponse(Task<SearchResponse> response)
+        {
+            var topResult = response.Result.Artists.Items.FirstOrDefault();
+
+            if (topResult is not null
+                && topResult.Name.Equals(ArtistName, StringComparison.InvariantCultureIgnoreCase))
+            {
+                result = topResult;
+            }
+        }
+    }
+}

Selector/Scrobble/ScrobbleMapping.cs

@@ -0,0 +1,80 @@
+using Microsoft.Extensions.Logging;
+using Selector.Operations;
+using SpotifyAPI.Web;
+using System;
+using System.Threading.Tasks;
+
+namespace Selector
+{
+    public enum LastfmObject{
+        Track, Album, Artist
+    }
+
+    public abstract class ScrobbleMapping : IOperation
+    {
+        private readonly ILogger<ScrobbleMapping> logger;
+        private readonly ISearchClient searchClient;
+
+        public event EventHandler Success;
+
+        public int MaxAttempts { get; private set; } = 5;
+        public int Attempts { get; private set; }
+
+        private Task<SearchResponse> currentTask { get; set; }
+        public bool Succeeded { get; private set; } = false;
+
+        public abstract object Result { get; }
+        public abstract string Query { get; }
+        public abstract SearchRequest.Types QueryType { get; }
+
+        private TaskCompletionSource AggregateTaskSource { get; set; } = new();
+        public Task Task => AggregateTaskSource.Task;
+
+        public ScrobbleMapping(ISearchClient _searchClient, ILogger<ScrobbleMapping> _logger)
+        {
+            logger = _logger;
+            searchClient = _searchClient;
+        }
+
+        public Task Execute()
+        {
+            logger.LogInformation("Mapping Last.fm {} ({}) to Spotify", Query, QueryType);
+
+            currentTask = searchClient.Item(new (QueryType, Query));
+            currentTask.ContinueWith(async t =>
+            {
+                if (t.IsCompletedSuccessfully)
+                {
+                    HandleResponse(t);
+                    OnSuccess();
+                    AggregateTaskSource.SetResult();
+                }
+                else
+                {
+                    if(t.Exception.InnerException is APITooManyRequestsException ex)
+                    {
+                        logger.LogError("Spotify search request too many requests, waiting for {}", ex.RetryAfter);
+                        await Task.Delay(ex.RetryAfter);
+                        Execute();
+                    }
+                    else
+                    {
+                        logger.LogError("Spotify search request task faulted, {}", t.Exception);
+                        AggregateTaskSource.SetException(t.Exception);
+                    }
+                }
+            });
+
+            Attempts++;
+            return Task.CompletedTask;
+        }
+
+        public abstract void HandleResponse(Task<SearchResponse> response);
+
+        protected virtual void OnSuccess()
+        {
+            // Raise the event in a thread-safe manner using the ?. operator.
+            Success?.Invoke(this, new EventArgs());
+        }
+    }
+}

Selector/Scrobble/ScrobbleRequest.cs

@@ -2,6 +2,7 @@
 using IF.Lastfm.Core.Api.Helpers;
 using IF.Lastfm.Core.Objects;
 using Microsoft.Extensions.Logging;
+using Selector.Operations;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -9,14 +10,14 @@ using System.Threading.Tasks;
 
 namespace Selector
 {
-    public class ScrobbleRequest
+    public class ScrobbleRequest : IOperation
     {
         private readonly ILogger<ScrobbleRequest> logger;
         private readonly IUserApi userClient;
 
         public event EventHandler Success;
 
-        public int MaxAttempts { get; private set; } = 5;
+        public int MaxAttempts { get; private set; }
         public int Attempts { get; private set; }
         public List<LastTrack> Scrobbles { get; private set; }
         public int TotalPages { get; private set; }
@@ -32,7 +33,7 @@ namespace Selector
         private TaskCompletionSource AggregateTaskSource { get; set; } = new();
         public Task Task => AggregateTaskSource.Task;
 
-        public ScrobbleRequest(IUserApi _userClient, ILogger<ScrobbleRequest> _logger, string _username, int _pageNumber, int _pageSize, DateTime? _from, DateTime? _to)
+        public ScrobbleRequest(IUserApi _userClient, ILogger<ScrobbleRequest> _logger, string _username, int _pageNumber, int _pageSize, DateTime? _from, DateTime? _to, int maxRetries = 5)
         {
             userClient = _userClient;
             logger = _logger;
@@ -42,12 +43,8 @@ namespace Selector
             pageSize = _pageSize;
             from = _from;
             to = _to;
-        }
 
-        protected virtual void RaiseSampleEvent()
+            MaxAttempts = maxRetries;
-        {
-            // Raise the event in a thread-safe manner using the ?. operator.
-            Success?.Invoke(this, new EventArgs());
         }
 
         public Task Execute()
@@ -95,7 +92,6 @@ namespace Selector
 
         protected virtual void OnSuccess()
         {
-            // Raise the event in a thread-safe manner using the ?. operator.
             Success?.Invoke(this, new EventArgs());
         }
     }

Selector/Scrobble/ScrobbleTrackMapping.cs

@@ -0,0 +1,41 @@
+using Microsoft.Extensions.Logging;
+using SpotifyAPI.Web;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Selector
+{
+    public class ScrobbleTrackMapping : ScrobbleMapping
+    {
+        public string TrackName { get; set; }
+        public string ArtistName { get; set; }
+
+        public ScrobbleTrackMapping(ISearchClient _searchClient, ILogger<ScrobbleTrackMapping> _logger, string trackName, string artistName) : base(_searchClient, _logger)
+        {
+            TrackName = trackName;
+            ArtistName = artistName;
+        }
+
+        private FullTrack result;
+        public override object Result => result;
+
+        public override string Query => $"{TrackName} {ArtistName}";
+
+        public override SearchRequest.Types QueryType => SearchRequest.Types.Track;
+
+        public override void HandleResponse(Task<SearchResponse> response)
+        {
+            var topResult = response.Result.Tracks.Items.FirstOrDefault();
+
+            if(topResult is not null 
+                && topResult.Name.Equals(TrackName, StringComparison.InvariantCultureIgnoreCase) 
+                && topResult.Artists.First().Name.Equals(ArtistName, StringComparison.InvariantCultureIgnoreCase))
+            {
+                result = topResult;
+            }
+        }
+    }
+}