Merge branch 'master' into past

.github/workflows/ci.yml

@@ -9,12 +9,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet-version: [ '6.0.x' ]
+        dotnet-version: [ '7.0.x' ]
 
     steps:
       - uses: actions/checkout@v2
       - name: Setup .NET Core SDK ${{ matrix.dotnet-version }}
-        uses: actions/setup-dotnet@v1.7.2
+        uses: actions/setup-dotnet@v3.0.3
         with:
           dotnet-version: ${{ matrix.dotnet-version }}
       - name: Install Dependencies

Dockerfile.CLI

@@ -1,8 +1,9 @@
-FROM mcr.microsoft.com/dotnet/sdk:6.0 AS base
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS base
 
 COPY *.sln .
 COPY Selector/*.csproj ./Selector/
 COPY Selector.Cache/*.csproj ./Selector.Cache/
+COPY Selector.Data/*.csproj ./Selector.Data/
 COPY Selector.Event/*.csproj ./Selector.Event/
 COPY Selector.Model/*.csproj ./Selector.Model/
 COPY Selector.CLI/*.csproj ./Selector.CLI/
@@ -14,7 +15,7 @@ COPY . ./
 FROM base as publish
 RUN dotnet publish Selector.CLI/Selector.CLI.csproj -c Release -o /app --no-restore
 
-FROM mcr.microsoft.com/dotnet/aspnet:6.0
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
 WORKDIR /app
 COPY --from=publish /app ./
 ENV DOTNET_EnableDiagnostics=0

Dockerfile.Web

@@ -6,11 +6,12 @@ RUN npm ci
 COPY ./Selector.Web ./
 RUN npm run build
 
-FROM mcr.microsoft.com/dotnet/sdk:6.0 AS base
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS base
 
 COPY *.sln .
 COPY Selector/*.csproj ./Selector/
 COPY Selector.Cache/*.csproj ./Selector.Cache/
+COPY Selector.Data/*.csproj ./Selector.Data/
 COPY Selector.Event/*.csproj ./Selector.Event/
 COPY Selector.Model/*.csproj ./Selector.Model/
 COPY Selector.Web/*.csproj ./Selector.Web/
@@ -24,7 +25,7 @@ COPY --from=frontend /Selector.Web/wwwroot Selector.Web/wwwroot/
 COPY --from=frontend /Selector.Web/wwwroot Selector.Web/wwwroot/
 RUN dotnet publish Selector.Web/Selector.Web.csproj -c Release -o /app --no-restore
 
-FROM mcr.microsoft.com/dotnet/aspnet:6.0
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
 EXPOSE 80
 WORKDIR /app
 COPY --from=publish /app ./

Selector.CLI/Command/CommandContext.cs

@@ -3,6 +3,7 @@ using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 using Selector.Model;
 using SpotifyAPI.Web;
+using StackExchange.Redis;
 
 namespace Selector.CLI
 {
@@ -10,7 +11,8 @@ namespace Selector.CLI
     {
         public RootOptions Config { get; set; }
         public ILoggerFactory Logger { get; set; }
-        public ISpotifyClient Spotify{ get; set; }
+        public ISpotifyClient Spotify { get; set; }
+        public ConnectionMultiplexer RedisMux { get; set; }
 
         public DbContextOptionsBuilder<ApplicationDbContext> DatabaseConfig { get; set; }        
         public LastfmClient LastFmClient { get; set; }

Selector.CLI/Command/SpotHistoryCommand.cs

@@ -9,6 +9,7 @@ using System.CommandLine;
 using System.CommandLine.Invocation;
 using System.Linq;
 using System.Collections.Generic;
+using Selector.Cache;
 
 namespace Selector.CLI
 {
@@ -28,24 +29,29 @@ namespace Selector.CLI
             username.AddAlias("-u");
             AddOption(username);
 
-            Handler = CommandHandler.Create((string connectionString, string path, string username) => Execute(connectionString, path, username));
+            Handler = CommandHandler.Create((string connection, string path, string username) => Execute(connection, path, username));
         }
 
-        public static int Execute(string connectionString, string path, string username)
+        public static int Execute(string connection, string path, string username)
         {
             var streams = new List<FileStream>();
 
             try
             {
-                var context = new CommandContext().WithLogger().WithDb(connectionString).WithLastfmApi();
+                var context = new CommandContext().WithLogger().WithDb(connection).WithSpotify().WithRedis();
                 var logger = context.Logger.CreateLogger("Scrobble");
 
                 using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger<ApplicationDbContext>());
 
                 var historyPersister = new HistoryPersister(db, new DataJsonContext(), new()
                 {
-                    Username = username
+                    Username = username,
-                }, context.Logger.CreateLogger<HistoryPersister>());
+                    Apply50PercentRule = true
+                },
+                durationPuller: new(context.Logger.CreateLogger<DurationPuller>(),
+                                    context.Spotify.Tracks,
+                                    cache: context.RedisMux.GetDatabase()),
+                logger: context.Logger.CreateLogger<HistoryPersister>());
 
                 logger.LogInformation("Preparing to parse from {} for {}", path, username);
 

Selector.CLI/Extensions/CommandContextExtensions.cs

@@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Selector.Model;
 using SpotifyAPI.Web;
+using StackExchange.Redis;
 using System.Linq;
 
 namespace Selector.CLI.Extensions
@@ -91,5 +92,21 @@ namespace Selector.CLI.Extensions
 
             return context;
         }
+
+        public static CommandContext WithRedis(this CommandContext context)
+        {
+            if (context.Config is null)
+            {
+                context.WithConfig();
+            }
+
+            var connectionString = context.Config.RedisOptions.ConnectionString;
+
+            var connMulti = ConnectionMultiplexer.Connect(connectionString);
+
+            context.RedisMux = connMulti;
+
+            return context;
+        }
     }
 }

Selector.CLI/Selector.CLI.csproj

@@ -2,22 +2,22 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <StartupObject>Selector.CLI.Program</StartupObject>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.9">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="7.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
     <PackageReference Include="NLog" Version="5.0.4" />
     <PackageReference Include="NLog.Extensions.Logging" Version="5.0.4" />
     <PackageReference Include="Quartz" Version="3.4.0" />

Selector.CLI/nlog.config

@@ -31,6 +31,9 @@
 
 	<!-- rules to map from logger name to target -->
 	<rules>
+		<logger name="Selector.*" minlevel="Debug" writeTo="logconsole" />
+		<logger name="Microsoft.*" minlevel="Warning" writeTo="logconsole" />
+
 		<!--<logger name="*" minlevel="Debug" writeTo="logfile" />-->
 		<logger name="Selector.*" minlevel="Debug" writeTo="logfile" />
 		<logger name="Microsoft.*" minlevel="Warning" writeTo="logfile" />
@@ -38,8 +41,5 @@
 		<!--<logger name="*" minlevel="Trace" writeTo="tracefile" />-->
 		<logger name="Selector.*" minlevel="Debug" writeTo="tracefile" />
 		<logger name="Microsoft.*" minlevel="Warning" writeTo="tracefile" />
-
-		<logger name="Selector.*" minlevel="Debug" writeTo="logconsole" />
-		<logger name="Microsoft.*" minlevel="Warning" writeTo="logconsole" />
 	</rules>
 </nlog>

Selector.Cache/Extensions/CacheExtensions.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Threading.Tasks;
+using StackExchange.Redis;
+
+namespace Selector.Cache;
+
+public static class CacheExtensions
+{
+    public static async Task<int?> GetTrackDuration(this IDatabaseAsync cache, string trackId)
+    {
+        return (int?) await cache?.HashGetAsync(Key.Track(trackId), Key.Duration);
+    }
+
+    public static async Task SetTrackDuration(this IDatabaseAsync cache, string trackId, int duration, TimeSpan? expiry = null)
+    {
+        var trackCacheKey = Key.Track(trackId);
+
+        await cache?.HashSetAsync(trackCacheKey, Key.Duration, duration);
+
+        if(expiry is not null)
+        {
+            await cache?.KeyExpireAsync(trackCacheKey, expiry);
+        }
+    }
+}
+

Selector.Cache/Key.cs

@@ -20,6 +20,7 @@ namespace Selector.Cache
 
         public const string AudioFeatureName = "AUDIO_FEATURE";
         public const string PlayCountName = "PLAY_COUNT";
+        public const string Duration = "DURATION";
 
         public const string SpotifyName = "SPOTIFY";
         public const string LastfmName = "LASTFM";
@@ -34,6 +35,9 @@ namespace Selector.Cache
         public static string CurrentlyPlaying(string user) => MajorNamespace(MinorNamespace(UserName, CurrentlyPlayingName), user);
         public static readonly string AllCurrentlyPlaying = CurrentlyPlaying(All);
 
+        public static string Track(string trackId) => MajorNamespace(TrackName, trackId);
+        public static readonly string AllTracks = Track(All);
+
         public static string AudioFeature(string trackId) => MajorNamespace(MinorNamespace(TrackName, AudioFeatureName), trackId);
         public static readonly string AllAudioFeatures = AudioFeature(All);
 

Selector.Cache/Selector.Cache.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <EnableDefaultCompileItems>true</EnableDefaultCompileItems>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="StackExchange.Redis" Version="2.6.66" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
     <PackageReference Include="SpotifyAPI.Web" Version="6.3.0" />
   </ItemGroup>
 

Selector.Cache/Services/DurationPuller.cs

@@ -0,0 +1,183 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+using SpotifyAPI.Web;
+using StackExchange.Redis;
+
+namespace Selector.Cache
+{
+    public class DurationPuller
+    {
+        private readonly IDatabaseAsync Cache;
+        private readonly ILogger<DurationPuller> Logger;
+
+        protected readonly ITracksClient SpotifyClient;
+
+        private int _retries = 0;
+
+
+        public DurationPuller(
+            ILogger<DurationPuller> logger,
+
+            ITracksClient spotifyClient,
+            IDatabaseAsync cache = null
+        )
+        {
+            Cache = cache;
+            Logger = logger;
+
+            SpotifyClient = spotifyClient;
+        }
+
+        public async Task<int?> Get(string uri)
+        {
+            if (string.IsNullOrWhiteSpace(uri)) throw new ArgumentNullException("No uri provided");
+
+            var trackId = uri.Split(":").Last();
+
+            var cachedVal = await Cache?.HashGetAsync(Key.Track(trackId), Key.Duration);
+            if (Cache is null || cachedVal == RedisValue.Null || cachedVal.IsNullOrEmpty)
+            {
+                try {
+                    Logger.LogDebug("Missed cache, pulling");
+
+                    var info = await SpotifyClient.Get(trackId);
+
+                    await Cache?.SetTrackDuration(trackId, info.DurationMs, TimeSpan.FromDays(7));
+
+                    _retries = 0;
+
+                    return info.DurationMs;
+                }
+                catch (APIUnauthorizedException e)
+                {
+                    Logger.LogError("Unauthorised error: [{message}] (should be refreshed and retried?)", e.Message);
+                    throw e;
+                }
+                catch (APITooManyRequestsException e)
+                {
+                    if(_retries <= 3)
+                    {
+                        Logger.LogWarning("Too many requests error, retrying ({}): [{message}]", e.RetryAfter, e.Message);
+                        _retries++;
+                        await Task.Delay(e.RetryAfter);
+                        return await Get(uri);
+                    }
+                    else
+                    {
+                        Logger.LogError("Too many requests error, done retrying: [{message}]", e.Message);
+                        throw e;
+                    }
+                }
+                catch (APIException e)
+                {
+                    if (_retries <= 3)
+                    {
+                        Logger.LogWarning("API error, retrying: [{message}]", e.Message);
+                        _retries++;
+                        await Task.Delay(TimeSpan.FromSeconds(2));
+                        return await Get(uri);
+                    }
+                    else
+                    {
+                        Logger.LogError("API error, done retrying: [{message}]", e.Message);
+                        throw e;
+                    }
+                }
+            }
+            else
+            {
+                return (int?) cachedVal;
+            }
+        }
+
+        public async Task<IDictionary<string, int>> Get(IEnumerable<string> uri)
+        {
+            if (!uri.Any()) throw new ArgumentNullException("No URIs provided");
+
+            var ret = new Dictionary<string, int>();
+            var toPullFromSpotify = new List<string>();
+
+            foreach (var input in uri.Select(x => x.Split(":").Last()))
+            {
+                var cachedVal = await Cache?.HashGetAsync(Key.Track(input), Key.Duration);
+
+                if (Cache is null || cachedVal == RedisValue.Null || cachedVal.IsNullOrEmpty)
+                {
+                    toPullFromSpotify.Add(input);
+                }
+                else
+                {
+                    ret[input] = (int) cachedVal;
+                }
+            }
+
+            var retries = new List<string>();
+
+            foreach(var chunk in toPullFromSpotify.Chunk(50))
+            {
+                await PullChunk(chunk, ret);
+                await Task.Delay(TimeSpan.FromMilliseconds(500));
+            }            
+
+            return ret;
+        }
+
+        private async Task PullChunk(IList<string> toPull, IDictionary<string, int> ret)
+        {
+            try
+            {
+                var info = await SpotifyClient.GetSeveral(new(toPull));
+
+                foreach (var resp in info.Tracks)
+                {
+                    await Cache?.SetTrackDuration(resp.Id, resp.DurationMs, TimeSpan.FromDays(7));
+
+                    ret[resp.Id] = (int)resp.DurationMs;
+                }
+
+                _retries = 0;
+            }
+            catch (APIUnauthorizedException e)
+            {
+                Logger.LogError("Unauthorised error: [{message}] (should be refreshed and retried?)", e.Message);
+                throw e;
+            }
+            catch (APITooManyRequestsException e)
+            {
+                if (_retries <= 3)
+                {
+                    Logger.LogWarning("Too many requests error, retrying ({}): [{message}]", e.RetryAfter, e.Message);
+                    _retries++;
+                    await Task.Delay(e.RetryAfter);
+
+                    await PullChunk(toPull, ret);
+                }
+                else
+                {
+                    Logger.LogError("Too many requests error, done retrying: [{message}]", e.Message);
+                    throw e;
+                }
+            }
+            catch (APIException e)
+            {
+                if (_retries <= 3)
+                {
+                    Logger.LogWarning("API error, retrying: [{message}]", e.Message);
+                    _retries++;
+                    await Task.Delay(TimeSpan.FromSeconds(5));
+
+                    await PullChunk(toPull, ret);
+                }
+                else
+                {
+                    Logger.LogError("API error, done retrying: [{message}]", e.Message);
+                    throw e;
+                }
+            }
+        }
+    }
+}

Selector.Data/HistoryPersister.cs

@@ -1,6 +1,8 @@
 using System.Text.Json;
 using Microsoft.Extensions.Logging;
+using Selector.Cache;
 using Selector.Model;
+using static SpotifyAPI.Web.PlaylistRemoveItemsRequest;
 
 namespace Selector.Data;
 
@@ -17,14 +19,31 @@ public class HistoryPersister
     private ApplicationDbContext Db { get; set; }
     private DataJsonContext Json { get; set; }
 
+    private DurationPuller DurationPuller { get; set; }
+
     private ILogger<HistoryPersister> Logger { get; set; }
 
-    public HistoryPersister(ApplicationDbContext db, DataJsonContext json, HistoryPersisterConfig config, ILogger<HistoryPersister> logger = null)
+    private readonly Dictionary<string, int> Durations;
+
+    public HistoryPersister(
+        ApplicationDbContext db,
+        DataJsonContext json,
+        HistoryPersisterConfig config,
+        DurationPuller durationPuller = null,
+        ILogger<HistoryPersister> logger = null)
     {
         Config = config;
         Db = db;
         Json = json;
+        DurationPuller = durationPuller;
         Logger = logger;
+
+        if (config.Apply50PercentRule && DurationPuller is null)
+        {
+            throw new ArgumentNullException(nameof(DurationPuller));
+        }
+
+        Durations = new();
     }
 
     public void Process(string input)
@@ -48,7 +67,7 @@ public class HistoryPersister
             var parsed = await JsonSerializer.DeserializeAsync(singleInput, Json.EndSongArray);
             songs = songs.Concat(parsed);
 
-            Logger?.LogDebug("Parsed {.2f} items for {}", parsed.Length, Config.Username);
+            Logger?.LogDebug("Parsed {:n0} items for {}", parsed.Length, Config.Username);
         }
 
         await Process(songs);
@@ -56,47 +75,136 @@ public class HistoryPersister
 
     public async Task Process(IEnumerable<EndSong> input)
     {
+        var user = Db.Users.Single(u => u.UserName == Config.Username);
+
         if (Config.InitialClear)
         {
             var latestTime = input.OrderBy(x => x.ts).Last().ts;
             var time = DateTime.Parse(latestTime).ToUniversalTime();
-            Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.User.UserName == Config.Username && x.Timestamp <= time));
+            Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.UserId == user.Id && x.Timestamp <= time));
         }
 
-        var user = Db.Users.Single(u => u.UserName == Config.Username);
+        var filtered = input.Where(x => x.ms_played > 30000
+                                     && !string.IsNullOrWhiteSpace(x.master_metadata_track_name))
+                            .DistinctBy(x => (x.offline_timestamp, x.ts, x.spotify_track_uri))
+                            .ToArray();
 
-        var counter = 0;
+        Logger.LogInformation("{:n0} items after filtering", filtered.Length);
 
-        var filtered = input.Where(x => x.ms_played > 30000)
+        var processedCounter = 0;
-            .DistinctBy(x => (x.offline_timestamp, x.ts, x.spotify_track_uri))
+        foreach (var item in filtered.Chunk(1000))
-            .ToArray();
-
-        Logger.LogInformation("{.2f} items after filtering", filtered.Length);
-
-        foreach (var item in filtered)
         {
-            if(!string.IsNullOrWhiteSpace(item.master_metadata_track_name))
+            IEnumerable<EndSong> toPopulate = item;
+
+            if (Config.Apply50PercentRule)
             {
-                Db.SpotifyListen.Add(new()
+                Logger.LogDebug("Validating tracks {:n0}/{:n0}", processedCounter + 1, filtered.Length);
-                {
-                    TrackName = item.master_metadata_track_name,
-                    AlbumName = item.master_metadata_album_album_name,
-                    ArtistName = item.master_metadata_album_artist_name,
 
-                    Timestamp = DateTime.Parse(item.ts).ToUniversalTime(),
+                toPopulate = Passes50PcRule(toPopulate);
-                    PlayedDuration = item.ms_played,
+            }
 
-                    TrackUri = item.spotify_track_uri,
+            Db.SpotifyListen.AddRange(toPopulate.Select(x => new SpotifyListen()
-                    UserId = user.Id
+            {
-                });
+                TrackName = x.master_metadata_track_name,
+                AlbumName = x.master_metadata_album_album_name,
+                ArtistName = x.master_metadata_album_artist_name,
 
-                counter++;
+                Timestamp = DateTime.Parse(x.ts).ToUniversalTime(),
+                PlayedDuration = x.ms_played,
+
+                TrackUri = x.spotify_track_uri,
+                UserId = user.Id
+            }));
+
+            processedCounter += item.Length;
+        }
+
+        Logger?.LogInformation("Saving {:n0} historical items for {}", processedCounter, user.UserName);
+
+        await Db.SaveChangesAsync();
+
+        Logger?.LogInformation("Added {:n0} historical items for {}", processedCounter, user.UserName);
+    }
+
+    private const int FOUR_MINUTES = 4 * 60 * 1000;
+
+    public async Task<bool> Passes50PcRule(EndSong song)
+    {
+        if (string.IsNullOrWhiteSpace(song.spotify_track_uri)) return true;
+
+        int duration;
+
+        if (Durations.TryGetValue(song.spotify_track_uri, out duration))
+        {
+
+        }
+        else
+        {
+            var pulledDuration = await DurationPuller.Get(song.spotify_track_uri);
+
+            if (pulledDuration is int d)
+            {
+                duration = d;
+
+                Durations.Add(song.spotify_track_uri, duration);
+            }
+            else
+            {
+                Logger.LogDebug("No duration returned for {}/{}", song.master_metadata_track_name, song.master_metadata_album_artist_name);
+                return true; // if can't get duration, just pass
             }
         }
 
-        Logger?.LogInformation("Added {} historical items for {}", counter, user.UserName);
+        return CheckDuration(song, duration);
-
-        await Db.SaveChangesAsync();
     }
+
+    public IEnumerable<EndSong> Passes50PcRule(IEnumerable<EndSong> inputTracks)
+    {
+        var toPullOverWire = new List<EndSong>();
+
+        // quick return items from local cache
+        foreach(var track in inputTracks)
+        {
+            if (string.IsNullOrWhiteSpace(track.spotify_track_uri)) yield return track;
+
+            if (Durations.TryGetValue(track.spotify_track_uri, out var duration))
+            {
+                if (CheckDuration(track, duration))
+                {
+                    yield return track;
+                }
+            }
+            else
+            {
+                toPullOverWire.Add(track);
+            }
+        }
+
+        var pulledDuration = DurationPuller.Get(toPullOverWire.Select(x => x.spotify_track_uri)).Result;
+
+        // apply results to cache
+        foreach((var uri, var dur) in pulledDuration)
+        {
+            Durations[uri] = dur;
+        }
+
+        // check return acceptable tracks from pulled
+        foreach(var track in toPullOverWire)
+        {
+            if(pulledDuration.TryGetValue(track.spotify_track_uri, out var duration))
+            {
+                if(CheckDuration(track, duration))
+                {
+                    yield return track;
+                }
+            }
+            else
+            {
+                yield return track;
+            }
+        }
+    }
+
+    public bool CheckDuration(EndSong song, int duration) => song.ms_played >= duration / 2 || song.ms_played >= FOUR_MINUTES;
 }
 

Selector.Data/Selector.Data.csproj

@@ -1,16 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Selector.Model\Selector.Model.csproj" />
+    <ProjectReference Include="..\Selector.Cache\Selector.Cache.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.9" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.0" />
   </ItemGroup>
 
 </Project>

Selector.Event/Selector.Event.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 

Selector.Model/ApplicationDbContext.cs

@@ -70,6 +70,8 @@ namespace Selector.Model
             modelBuilder.Entity<UserScrobble>()
                 .Property(s => s.ArtistName)
                 .UseCollation("case_insensitive");
+            //modelBuilder.Entity<UserScrobble>()
+            //    .HasIndex(x => new { x.UserId, x.ArtistName, x.TrackName });
 
             modelBuilder.Entity<TrackLastfmSpotifyMapping>().HasKey(s => s.SpotifyUri);
             modelBuilder.Entity<TrackLastfmSpotifyMapping>()
@@ -102,6 +104,8 @@ namespace Selector.Model
             modelBuilder.Entity<SpotifyListen>()
                 .Property(s => s.ArtistName)
                 .UseCollation("case_insensitive");
+            //modelBuilder.Entity<SpotifyListen>()
+            //    .HasIndex(x => new { x.UserId, x.ArtistName, x.TrackName });
 
             SeedData.Seed(modelBuilder);
         }

Selector.Model/Listen/IListenRepository.cs

@@ -5,7 +5,7 @@ namespace Selector.Model
 {
     public interface IListenRepository
     {
-        IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+        IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false);
         int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
     }
 }

Selector.Model/Listen/MetaListenRepository.cs

@@ -28,7 +28,8 @@ public class MetaListenRepository: IListenRepository
             albumName: albumName,
             artistName: artistName,
             from: from,
-            to:to).Count();
+            to:to,
+            tracking: false).Count();
 
     public IEnumerable<IListen> GetAll(
         string includes = null,
@@ -38,7 +39,9 @@ public class MetaListenRepository: IListenRepository
         string albumName = null,
         string artistName = null,
         DateTime? from = null,
-        DateTime? to = null)
+        DateTime? to = null,
+        bool tracking = true,
+        bool orderTime = false)
     {
         var scrobbles = scrobbleRepository.GetAll(
             include: includes,
@@ -48,9 +51,9 @@ public class MetaListenRepository: IListenRepository
             albumName: albumName,
             artistName: artistName,
             from: from,
-            to: to)
+            to: to,
-            .OrderBy(x => x.Timestamp)
+            tracking: tracking,
-            .ToArray();
+            orderTime: true).ToArray();
 
         var spotListens = spotifyRepository.GetAll(
             include: includes,
@@ -60,10 +63,9 @@ public class MetaListenRepository: IListenRepository
             albumName: albumName,
             artistName: artistName,
             from: from,
-            to: scrobbles.FirstOrDefault()?.Timestamp)
+            to: scrobbles.FirstOrDefault()?.Timestamp,
-            .OrderBy(x => x.Timestamp)
+            tracking: tracking,
-            .ToArray();
+            orderTime: orderTime);
-        
 
         return spotListens.Concat(scrobbles);
     }

Selector.Model/Listen/SpotifyListenRepository.cs

@@ -39,10 +39,15 @@ namespace Selector.Model
             return listens.FirstOrDefault();
         }
 
-        private IQueryable<SpotifyListen> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        private IQueryable<SpotifyListen> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
         {
             var listens = db.SpotifyListen.AsQueryable();
 
+            if (!tracking)
+            {
+                listens = listens.AsNoTracking();
+            }
+
             if (!string.IsNullOrWhiteSpace(include))
             {
                 listens = listens.Include(include);
@@ -92,11 +97,16 @@ namespace Selector.Model
                 listens = listens.Where(u => u.Timestamp < to.Value);
             }
 
+            if (orderTime)
+            {
+                listens = listens.OrderBy(x => x.Timestamp);
+            }
+
             return listens;
         }
 
-        public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
-            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
+            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: tracking, orderTime: orderTime).AsEnumerable();
 
         public void Remove(DateTime key)
         {
@@ -124,6 +134,6 @@ namespace Selector.Model
         }
 
         public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
-            => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
+            => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: false).Count();
     }
 }

Selector.Model/Scrobble/DBPlayCountPuller.cs

@@ -27,7 +27,7 @@ namespace Selector.Cache
 
             var userScrobbleCount = ScrobbleRepository.Count(username: username);
 
-            var artistScrobbles = ScrobbleRepository.GetAll(username: username, artistName: artist).ToArray();
+            var artistScrobbles = ScrobbleRepository.GetAll(username: username, artistName: artist, tracking: false, orderTime: true).ToArray();
             var albumScrobbles = artistScrobbles.Where(
                 s => s.AlbumName.Equals(album, StringComparison.CurrentCultureIgnoreCase)).ToArray();
             var trackScrobbles = artistScrobbles.Where(

Selector.Model/Scrobble/ScrobbleRepository.cs

@@ -39,10 +39,15 @@ namespace Selector.Model
             return scrobbles.FirstOrDefault();
         }
 
-        private IQueryable<UserScrobble> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        private IQueryable<UserScrobble> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
         {
             var scrobbles = db.Scrobble.AsQueryable();
 
+            if (!tracking)
+            {
+                scrobbles = scrobbles.AsNoTracking();
+            }
+
             if (!string.IsNullOrWhiteSpace(include))
             {
                 scrobbles = scrobbles.Include(include);
@@ -92,11 +97,16 @@ namespace Selector.Model
                 scrobbles = scrobbles.Where(u => u.Timestamp < to.Value);
             }
 
+            if (orderTime)
+            {
+                scrobbles = scrobbles.OrderBy(x => x.Timestamp);
+            }
+
             return scrobbles;
         }
 
-        public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
-            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
+            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: tracking, orderTime: orderTime).AsEnumerable();
 
         public void Remove(int key)
         {
@@ -124,6 +134,6 @@ namespace Selector.Model
         }
 
         public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
-            => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
+            => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: false).Count();
     }
 }

Selector.Model/Selector.Model.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <EnableDefaultCompileItems>true</EnableDefaultCompileItems>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
@@ -12,20 +12,20 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNetCore.Identity" Version="2.2.0" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.9" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.0" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.9" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="7.0.0" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.9" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.0" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.9">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.0">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.9">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="7.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="6.0.9" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="7.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
-    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.7" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>

Selector.Tests/Selector.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
     <LangVersion>latest</LangVersion>

Selector.Web/Selector.Web.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <LangVersion>latest</LangVersion>
     <TypeScriptCompileBlocked>true</TypeScriptCompileBlocked>
   </PropertyGroup>
@@ -14,21 +14,21 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.9" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.0" />
-    <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.9" />
+    <PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="7.0.0" />
 
-    <PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="6.0.9" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="7.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
 
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.9">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.0">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
 
-    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
 
-    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="7.0.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.9" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.0" />
     <PackageReference Include="NLog" Version="5.0.4" />
     <PackageReference Include="NLog.Extensions.Logging" Version="5.0.4" />
     <PackageReference Include="NLog.Web.AspNetCore" Version="5.1.4" />

Selector.Web/nlog.config

@@ -46,5 +46,7 @@
 		<logger name="System.Net.Http.*" maxlevel="Info" final="true" />
 
 		<!--<logger name="*" minlevel="Debug" writeTo="logfile" />-->
+		<logger name="Selector.*" minlevel="Info" writeTo="logfile" />
+		<logger name="Microsoft.*" minlevel="Warning" writeTo="logfile" />
 	</rules>
 </nlog>

Selector.Web/scripts/now.ts

@@ -10,6 +10,7 @@ import BaseInfoCard from "./Now/BaseInfoCard";
 
 const connection = new signalR.HubConnectionBuilder()
     .withUrl("/nowhub")
+    .withAutomaticReconnect()
     .build();
 
 connection.start()

Selector/Selector.csproj

@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	  <EnableDefaultCompileItems>true</EnableDefaultCompileItems>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
     <PackageReference Include="SpotifyAPI.Web" Version="6.3.0" />
     <PackageReference Include="Inflatable.Lastfm" Version="1.2.0" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />

docker-compose.build.yml

@@ -27,11 +27,11 @@ services:
       DOTNET_ENVIRONMENT: Production
 
   redis:
-    image: redis:alpine
+    image: redis:7
     ports:
       - "6379:6379"
   database:
-    image: postgres
+    image: postgres:14
     ports:
       - "5432:5432"
     env_file: .env # set POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB

docker-compose.yml

@@ -25,12 +25,12 @@ services:
       DOTNET_ENVIRONMENT: Production
 
   redis:
-    image: redis:alpine
+    image: redis:7
     restart: unless-stopped
     ports:
       - "6379:6379"
   database:
-    image: postgres
+    image: postgres:14
     restart: unless-stopped
     ports:
       - "5432:5432"