more efficient linq, adding album and track last fm mapping

Selector.CLI/Command/Scrobble/ScrobbleSave.cs

@@ -46,10 +46,10 @@ namespace Selector.CLI
             dontRemove.AddAlias("-nr");
             AddOption(dontRemove);
 
-            Handler = CommandHandler.Create(async (DateTime from, DateTime to, int page, int delay, int simul, string username, bool noAdd, bool noRemove, CancellationToken token) => await Execute(from, to, page, delay, simul, username, noAdd, noRemove, token));
+            Handler = CommandHandler.Create(async (DateTime from, DateTime to, int page, int delay, int simultaneous, string username, bool noAdd, bool noRemove, CancellationToken token) => await Execute(from, to, page, delay, simultaneous, username, noAdd, noRemove, token));
         }
 
-        public static async Task<int> Execute(DateTime from, DateTime to, int page, int delay, int simul, string username, bool noAdd, bool noRemove, CancellationToken token)
+        public static async Task<int> Execute(DateTime from, DateTime to, int page, int delay, int simultaneous, string username, bool noAdd, bool noRemove, CancellationToken token)
         {
             try
             {
@@ -91,7 +91,7 @@ namespace Selector.CLI
                                 PageSize = page,
                                 DontAdd = noAdd,
                                 DontRemove = noRemove,
-                                SimultaneousConnections = simul
+                                SimultaneousConnections = simultaneous
                             },
                             repo,
                             context.Logger.CreateLogger<ScrobbleSaver>(), 

Selector.CLI/ScrobbleMapper.cs

@@ -1,17 +1,10 @@
-using IF.Lastfm.Core.Api;
+using Microsoft.Extensions.Logging;
-using IF.Lastfm.Core.Api.Helpers;
-using IF.Lastfm.Core.Objects;
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Selector.Model;
 using Selector.Operations;
 using SpotifyAPI.Web;
 using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -53,56 +46,175 @@ namespace Selector
         {
             if (config.Artists)
             {
-                logger.LogInformation("Mapping scrobble artists");
+                await MapArtists(token);
+            }
 
-                var currentArtists = mappingRepo.GetArtists();
+            if (config.Albums)
-                var scrobbleArtists = scrobbleRepo.GetAll()
+            {
-                    .GroupBy(x => x.ArtistName)
+                await MapAlbums(token);
-                    .Select(x => (x.Key, x.Count()))
+            }
-                    .OrderByDescending(x => x.Item2)
-                    .Select(x => x.Key);
 
-                if(config.Limit is not null)
+            if (config.Tracks)
-                {
+            {
-                    scrobbleArtists = scrobbleArtists.Take(config.Limit.Value);
+                await MapTracks(token);
-                }
-
-                var artistsToPull = scrobbleArtists
-                    .ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a);
-
-
-                var requests = artistsToPull.Select(a => new ScrobbleArtistMapping(
-                     searchClient, 
-                     loggerFactory.CreateLogger<ScrobbleArtistMapping>() ?? NullLogger<ScrobbleArtistMapping>.Instance,
-                     a)
-                ).ToList();
-
-                logger.LogInformation("Found {} artists to map, starting", requests.Count);
-
-                var batchRequest = new BatchingOperation<ScrobbleArtistMapping>(
-                    config.InterRequestDelay, 
-                    config.Timeout, 
-                    config.SimultaneousConnections, 
-                    requests
-                );
-
-                await batchRequest.TriggerRequests(token);
-
-                logger.LogInformation("Finished mapping artists");
-
-                var newArtists = batchRequest.DoneRequests
-                    .Where(a => a is not null)
-                    .Select(a => (FullArtist) a.Result)
-                    .Where(a => a is not null);
-                var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping() { 
-                    LastfmArtistName = a.Name,
-                    SpotifyUri = a.Uri
-                });
-
-                mappingRepo.AddRange(newMappings);
             }
 
             await mappingRepo.Save();
         }
+
+        private async Task MapArtists(CancellationToken token)
+        {
+            logger.LogInformation("Mapping scrobble artists");
+
+            var currentArtists = mappingRepo.GetArtists();
+            var scrobbleArtists = scrobbleRepo.GetAll()
+                .GroupBy(x => x.ArtistName)
+                .Select(x => (x.Key, x.Count()))
+                .OrderByDescending(x => x.Item2)
+                .Select(x => x.Key);
+
+            var artistsToPull = scrobbleArtists
+                .ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a);
+
+            if (config.Limit is not null)
+            {
+                artistsToPull = artistsToPull.Take(config.Limit.Value);
+            }
+
+            var requests = artistsToPull.Select(a => new ScrobbleArtistMapping(
+                 searchClient,
+                 loggerFactory.CreateLogger<ScrobbleArtistMapping>() ?? NullLogger<ScrobbleArtistMapping>.Instance,
+                 a)
+            ).ToArray();
+
+            logger.LogInformation("Found {} artists to map, starting", requests.Length);
+
+            var batchRequest = new BatchingOperation<ScrobbleArtistMapping>(
+                config.InterRequestDelay,
+                config.Timeout,
+                config.SimultaneousConnections,
+                requests
+            );
+
+            await batchRequest.TriggerRequests(token);
+
+            logger.LogInformation("Finished mapping artists");
+
+            var newArtists = batchRequest.DoneRequests
+                .Select(a => a.Result)
+                .Cast<FullArtist>()
+                .Where(a => a is not null);
+            var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping()
+            {
+                LastfmArtistName = a.Name,
+                SpotifyUri = a.Uri
+            });
+
+            mappingRepo.AddRange(newMappings);
+        }
+
+        private async Task MapAlbums(CancellationToken token)
+        {
+            logger.LogInformation("Mapping scrobble albums");
+
+            var currentAlbums = mappingRepo.GetAlbums();
+            var scrobbleAlbums = scrobbleRepo.GetAll()
+                .GroupBy(x => (x.ArtistName, x.AlbumName))
+                .Select(x => (x.Key, x.Count()))
+                .OrderByDescending(x => x.Item2)
+                .Select(x => x.Key);
+
+            var albumsToPull = scrobbleAlbums
+                .ExceptBy(currentAlbums.Select(a => (a.LastfmArtistName, a.LastfmAlbumName)), a => a);
+
+            if (config.Limit is not null)
+            {
+                albumsToPull = albumsToPull.Take(config.Limit.Value);
+            }
+
+            var requests = albumsToPull.Select(a => new ScrobbleAlbumMapping(
+                 searchClient,
+                 loggerFactory.CreateLogger<ScrobbleAlbumMapping>() ?? NullLogger<ScrobbleAlbumMapping>.Instance,
+                 a.AlbumName, a.ArtistName)
+            ).ToArray();
+
+            logger.LogInformation("Found {} albums to map, starting", requests.Length);
+
+            var batchRequest = new BatchingOperation<ScrobbleAlbumMapping>(
+                config.InterRequestDelay,
+                config.Timeout,
+                config.SimultaneousConnections,
+                requests
+            );
+
+            await batchRequest.TriggerRequests(token);
+
+            logger.LogInformation("Finished mapping albums");
+
+            var newArtists = batchRequest.DoneRequests
+                .Select(a => a.Result)
+                .Cast<SimpleAlbum>()
+                .Where(a => a is not null);
+            var newMappings = newArtists.Select(a => new AlbumLastfmSpotifyMapping()
+            {
+                LastfmAlbumName = a.Name,
+                LastfmArtistName = a.Artists.FirstOrDefault()?.Name,
+                SpotifyUri = a.Uri
+            });
+
+            mappingRepo.AddRange(newMappings);
+        }
+
+        private async Task MapTracks(CancellationToken token)
+        {
+            logger.LogInformation("Mapping scrobble tracks");
+
+            var currentTracks = mappingRepo.GetTracks();
+            var scrobbleTracks = scrobbleRepo.GetAll()
+                .GroupBy(x => (x.ArtistName, x.TrackName))
+                .Select(x => (x.Key, x.Count()))
+                .OrderByDescending(x => x.Item2)
+                .Select(x => x.Key);
+
+            if (config.Limit is not null)
+            {
+                scrobbleTracks = scrobbleTracks.Take(config.Limit.Value);
+            }
+
+            var tracksToPull = scrobbleTracks
+                .ExceptBy(currentTracks.Select(a => (a.LastfmArtistName, a.LastfmTrackName)), a => a);
+
+            var requests = tracksToPull.Select(a => new ScrobbleTrackMapping(
+                 searchClient,
+                 loggerFactory.CreateLogger<ScrobbleTrackMapping>() ?? NullLogger<ScrobbleTrackMapping>.Instance,
+                 a.TrackName, a.ArtistName)
+            ).ToArray();
+
+            logger.LogInformation("Found {} tracks to map, starting", requests.Length);
+
+            var batchRequest = new BatchingOperation<ScrobbleTrackMapping>(
+                config.InterRequestDelay,
+                config.Timeout,
+                config.SimultaneousConnections,
+                requests
+            );
+
+            await batchRequest.TriggerRequests(token);
+
+            logger.LogInformation("Finished mapping tracks");
+
+            var newArtists = batchRequest.DoneRequests
+                .Select(a => a.Result)
+                .Cast<FullTrack>()
+                .Where(a => a is not null);
+            var newMappings = newArtists.Select(a => new TrackLastfmSpotifyMapping()
+            {
+                LastfmTrackName = a.Name,
+                LastfmArtistName = a.Artists.FirstOrDefault()?.Name,
+                SpotifyUri = a.Uri
+            });
+
+            mappingRepo.AddRange(newMappings);
+        }
     }
 }

Selector.CLI/ScrobbleSaver.cs

@@ -87,29 +87,28 @@ namespace Selector
                     await batchTask;
                 }
 
-                var scrobbles = page1.Scrobbles;
+                IEnumerable<LastTrack> scrobbles;
-
                 if(batchOperation is not null)
                 {
-                    scrobbles.AddRange(batchOperation.DoneRequests.SelectMany(r => r.Scrobbles));
+                    scrobbles = page1.Scrobbles.Union(batchOperation.DoneRequests.SelectMany(r => r.Scrobbles));
+                }
+                else
+                {
+                    scrobbles = page1.Scrobbles;
                 }
 
                 IdentifyDuplicates(scrobbles);
 
                 logger.LogDebug("Ordering and filtering pulled scrobbles");
 
-                RemoveNowPlaying(scrobbles.ToList());
+                scrobbles = RemoveNowPlaying(scrobbles);
 
                 var nativeScrobbles = scrobbles
                     .DistinctBy(s => new { s.TimePlayed?.UtcDateTime, s.Name, s.ArtistName })
-                    .Select(s =>
+                    .Select(s => (UserScrobble) s)
-                    {
+                    .ToArray();
-                        var nativeScrobble = (UserScrobble)s;
-                        nativeScrobble.UserId = config.User.Id;
-                        return nativeScrobble;
-                    });
 
-                logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Count());
+                logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
 
                 logger.LogDebug("Identifying difference sets");
                 var time = Stopwatch.StartNew();
@@ -123,7 +122,12 @@ namespace Selector
 
                 if(!config.DontAdd)
                 {
-                    scrobbleRepo.AddRange(toAdd.Cast<UserScrobble>());
+                    foreach(var add in toAdd)
+                    {
+                        var scrobble = (UserScrobble) add;
+                        scrobble.UserId = config.User.Id;
+                        scrobbleRepo.Add(scrobble);
+                    }
                 }
                 else
                 {
@@ -131,7 +135,12 @@ namespace Selector
                 }
                 if (!config.DontRemove)
                 {
-                    scrobbleRepo.RemoveRange(toRemove.Cast<UserScrobble>());
+                    foreach (var remove in toRemove)
+                    {
+                        var scrobble = (UserScrobble) remove;
+                        scrobble.UserId = config.User.Id;
+                        scrobbleRepo.Remove(scrobble);
+                    }
                 }
                 else
                 {
@@ -189,19 +198,18 @@ namespace Selector
             }
         }
 
-        private bool RemoveNowPlaying(List<LastTrack> scrobbles)
+        private IEnumerable<LastTrack> RemoveNowPlaying(IEnumerable<LastTrack> scrobbles)
         {
             var newestScrobble = scrobbles.FirstOrDefault();
             if (newestScrobble is not null)
             {
                 if (newestScrobble.IsNowPlaying is bool playing && playing)
                 {
-                    scrobbles.Remove(newestScrobble);
+                    scrobbles = scrobbles.Skip(1);
-                    return true;
                 }
             }
 
-            return false;
+            return scrobbles;
         }
     }
 }

Selector/Operations/BatchingOperation.cs

@@ -64,7 +64,7 @@ namespace Selector.Operations
             }
 
             var timeoutTask = Task.Delay(timeout, token);
-            var allTasks = WaitingRequests.Union(DoneRequests).Select(r => r.Task).ToList();
+            var allTasks = WaitingRequests.Union(DoneRequests).Select(r => r.Task);
 
             var firstToFinish = await Task.WhenAny(timeoutTask, Task.WhenAll(allTasks));
 

Selector/Scrobble/ScrobbleRequest.cs

@@ -19,7 +19,7 @@ namespace Selector
 
         public int MaxAttempts { get; private set; }
         public int Attempts { get; private set; }
-        public List<LastTrack> Scrobbles { get; private set; }
+        public IEnumerable<LastTrack> Scrobbles { get; private set; }
         public int TotalPages { get; private set; }
         private Task<PageResponse<LastTrack>> currentTask { get; set; }
         public bool Succeeded { get; private set; } = false;
@@ -60,7 +60,7 @@ namespace Selector
 
                     if (Succeeded)
                     {
-                        Scrobbles = result.Content.ToList();
+                        Scrobbles = result.Content.ToArray();
                         TotalPages = result.TotalPages;
                         OnSuccess();
                         AggregateTaskSource.SetResult();