tweaking resampler, db count puller

Selector.Model/Extensions/ServiceExtensions.cs

@@ -1,6 +1,6 @@
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.Extensions.DependencyInjection;
-
+using Selector.Cache;
 using Selector.Model.Authorisation;
 
 namespace Selector.Model.Extensions
@@ -21,5 +21,12 @@ namespace Selector.Model.Extensions
             services.AddScoped<IAuthorizationHandler, UserIsSelfAuthHandler>();
             services.AddSingleton<IAuthorizationHandler, UserIsAdminAuthHandler>();
         }
+
+        public static IServiceCollection AddDBPlayCountPuller(this IServiceCollection services)
+        {
+            services.AddTransient<DBPlayCountPuller>();
+
+            return services;
+        }
     }
 }

Selector.Model/Scrobble/DBPlayCountPuller.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+using Selector.Model;
+
+namespace Selector.Cache
+{
+    public class DBPlayCountPuller
+    {
+        protected readonly IScrobbleRepository ScrobbleRepository;
+        private readonly IOptions<NowPlayingOptions> nowOptions;
+
+        public DBPlayCountPuller(
+            IOptions<NowPlayingOptions> options,
+            IScrobbleRepository scrobbleRepository
+        )
+        {
+            ScrobbleRepository = scrobbleRepository;
+            nowOptions = options;
+        }
+
+        public Task<PlayCount> Get(string username, string track, string artist, string album, string albumArtist)
+        {
+            if (string.IsNullOrWhiteSpace(username)) throw new ArgumentNullException("No username provided");
+
+            var userScrobbleCount = ScrobbleRepository.Count(username: username);
+
+            var artistScrobbles = ScrobbleRepository.GetAll(username: username, artistName: artist).ToArray();
+            var albumScrobbles = artistScrobbles.Where(
+                s => s.AlbumName.Equals(album, StringComparison.CurrentCultureIgnoreCase)).ToArray();
+            var trackScrobbles = artistScrobbles.Where(
+                s => s.TrackName.Equals(track, StringComparison.CurrentCultureIgnoreCase)).ToArray();
+
+            var postCalc = artistScrobbles.Resample(nowOptions.Value.ArtistResampleWindow).Select(s => s.Value).Sum();
+            //var postCalc = playCount.ArtistCountData.Select(s => s.Value).Sum();
+            Debug.Assert(postCalc == artistScrobbles.Count());
+
+            PlayCount playCount = new()
+            {
+                Username = username,
+                Artist = artistScrobbles.Count(),
+                Album = albumScrobbles.Count(),
+                Track = trackScrobbles.Count(),
+                User = userScrobbleCount,
+
+                ArtistCountData = artistScrobbles
+                    .Resample(nowOptions.Value.ArtistResampleWindow)
+                    //.ResampleByMonth()
+                    .CumulativeSum()
+                    .ToArray(),
+
+                AlbumCountData = albumScrobbles
+                    .Resample(nowOptions.Value.AlbumResampleWindow)
+                    //.ResampleByMonth()
+                    .CumulativeSum()
+                    .ToArray(),
+
+                TrackCountData = trackScrobbles
+                    .Resample(nowOptions.Value.TrackResampleWindow)
+                    //.ResampleByMonth()
+                    .CumulativeSum()
+                    .ToArray()
+            };
+
+            return Task.FromResult(playCount);
+        }
+    }
+}

Selector.Model/Scrobble/IScrobbleRepository.cs

@@ -17,5 +17,6 @@ namespace Selector.Model
         public void RemoveRange(IEnumerable<UserScrobble> scrobbles);
         void Update(UserScrobble item);
         Task<int> Save();
+        int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
     }
 }

Selector.Model/Scrobble/ScrobbleRepository.cs

@@ -39,7 +39,7 @@ namespace Selector.Model
             return scrobbles.FirstOrDefault();
         }
 
-        public IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        private IQueryable<UserScrobble> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
         {
             var scrobbles = db.Scrobble.AsQueryable();
 
@@ -55,7 +55,8 @@ namespace Selector.Model
 
             if (!string.IsNullOrWhiteSpace(username))
             {
-                var user = db.Users.AsNoTracking().Where(u => u.UserName.Equals(username, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
+                var normalUsername = username.ToUpperInvariant();
+                var user = db.Users.AsNoTracking().Where(u => u.NormalizedUserName == normalUsername).FirstOrDefault();
                 if (user is not null)
                 {
                     scrobbles = scrobbles.Where(s => s.UserId == user.Id);
@@ -91,9 +92,12 @@ namespace Selector.Model
                 scrobbles = scrobbles.Where(u => u.Timestamp < to.Value);
             }
 
-            return scrobbles.AsEnumerable();
+            return scrobbles;
         }
 
+        public IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
+
         public void Remove(int key)
         {
             Remove(Find(key));
@@ -118,5 +122,8 @@ namespace Selector.Model
         {
             return db.SaveChangesAsync();
         }
+
+        public int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
     }
 }

Selector.Web/Hubs/NowPlayingHub.cs

@@ -30,6 +30,7 @@ namespace Selector.Web.Hubs
         private readonly IDatabaseAsync Cache;
         private readonly AudioFeaturePuller AudioFeaturePuller;
         private readonly PlayCountPuller PlayCountPuller;
+        private readonly DBPlayCountPuller DBPlayCountPuller;
         private readonly ApplicationDbContext Db;
         private readonly IScrobbleRepository ScrobbleRepository;
 
@@ -41,12 +42,14 @@ namespace Selector.Web.Hubs
             ApplicationDbContext db,
             IScrobbleRepository scrobbleRepository,
             IOptions<NowPlayingOptions> options,
+            DBPlayCountPuller dbPlayCountPuller,
             PlayCountPuller playCountPuller = null
         )
         {
             Cache = cache;
             AudioFeaturePuller = featurePuller;
             PlayCountPuller = playCountPuller;
+            DBPlayCountPuller = dbPlayCountPuller;
             Db = db;
             ScrobbleRepository = scrobbleRepository;
             nowOptions = options;
@@ -103,36 +106,16 @@ namespace Selector.Web.Hubs
 
                 if (user.LastFmConnected())
                 {
-                    var playCount = await PlayCountPuller.Get(user.LastFmUsername, track, artist, album, albumArtist);
+                    PlayCount playCount;
 
                     if (user.ScrobbleSavingEnabled())
                     {
-                        var artistScrobbles = ScrobbleRepository.GetAll(userId: user.Id, artistName: artist).ToArray();
+                        playCount = await DBPlayCountPuller.Get(user.UserName, track, artist, album, albumArtist);
-
+                    }
-                        playCount.Artist = artistScrobbles.Length;
+                    else
-
+                    {
-                        playCount.ArtistCountData = artistScrobbles
+                        playCount = await PlayCountPuller.Get(user.LastFmUsername, track, artist, album, albumArtist);
-                            //.Resample(nowOptions.Value.ArtistResampleWindow)
-                            .ResampleByMonth()
-                            .ToArray();
-
-                        var postCalc = playCount.ArtistCountData.Select(s => s.Value).Sum();
-                        Debug.Assert(postCalc == artistScrobbles.Count());
-
-                        playCount.AlbumCountData = artistScrobbles
-                            .Where(s => s.AlbumName.Equals(album, StringComparison.CurrentCultureIgnoreCase))
-                            //.Resample(nowOptions.Value.AlbumResampleWindow)
-                            .ResampleByMonth()
-                            .ToArray();
-
-                        playCount.TrackCountData = artistScrobbles
-                            .Where(s => s.TrackName.Equals(track, StringComparison.CurrentCultureIgnoreCase))
-                            //.Resample(nowOptions.Value.TrackResampleWindow)
-                            .ResampleByMonth()
-                            .ToArray();
                     }
-
-
 
                     await Clients.Caller.OnNewPlayCount(playCount);
                 }

Selector.Web/Options.cs

@@ -53,22 +53,4 @@ namespace Selector.Web
         public bool Enabled { get; set; } = false;
         public string ConnectionString { get; set; }
     }
-
-    public class NowPlayingOptions
-    {
-        public const string Key = "Now";
-
-        public TimeSpan ArtistResampleWindow { get; set; } = TimeSpan.FromDays(30);
-        public TimeSpan AlbumResampleWindow { get; set; } = TimeSpan.FromDays(30);
-        public TimeSpan TrackResampleWindow { get; set; } = TimeSpan.FromDays(30);
-
-        public TimeSpan ArtistDensityWindow { get; set; } = TimeSpan.FromDays(10);
-        public decimal ArtistDensityThreshold { get; set; } = 5;
-
-        public TimeSpan AlbumDensityWindow { get; set; } = TimeSpan.FromDays(10);
-        public decimal AlbumDensityThreshold { get; set; } = 5;
-
-        public TimeSpan TrackDensityWindow { get; set; } = TimeSpan.FromDays(10);
-        public decimal TrackDensityThreshold { get; set; } = 5;
-    }
 }

Selector.Web/Startup.cs

@@ -54,6 +54,7 @@ namespace Selector.Web
             services.AddDbContext<ApplicationDbContext>(options => 
                 options.UseNpgsql(Configuration.GetConnectionString("Default"))
             );
+            services.AddDBPlayCountPuller();
             services.AddTransient<IScrobbleRepository, ScrobbleRepository>();
 
             services.AddIdentity<ApplicationUser, IdentityRole>()

Selector/Options.cs

@@ -0,0 +1,22 @@
+using System;
+namespace Selector
+{
+    public class NowPlayingOptions
+    {
+        public const string Key = "Now";
+
+        public TimeSpan ArtistResampleWindow { get; set; } = TimeSpan.FromDays(30);
+        public TimeSpan AlbumResampleWindow { get; set; } = TimeSpan.FromDays(30);
+        public TimeSpan TrackResampleWindow { get; set; } = TimeSpan.FromDays(30);
+
+        public TimeSpan ArtistDensityWindow { get; set; } = TimeSpan.FromDays(10);
+        public decimal ArtistDensityThreshold { get; set; } = 5;
+
+        public TimeSpan AlbumDensityWindow { get; set; } = TimeSpan.FromDays(10);
+        public decimal AlbumDensityThreshold { get; set; } = 5;
+
+        public TimeSpan TrackDensityWindow { get; set; } = TimeSpan.FromDays(10);
+        public decimal TrackDensityThreshold { get; set; } = 5;
+    }
+}
+

Selector/Scrobble/Resampler.cs

@@ -26,25 +26,37 @@ namespace Selector
 			var earliest = sortedScrobbles.First().Timestamp;
 			var latest = sortedScrobbles.Last().Timestamp;
 
-			for (var counter = earliest; counter <= latest; counter += window)
+			var enumeratorExhausted = false;
+
+			for (var windowStart = earliest; windowStart <= latest; windowStart += window)
 			{
-				var windowEnd = counter + window;
+				var windowEnd = windowStart + window;
 
 				var count = 0;
+				var windowOverran = false;
 
-				if (sortedScrobblesIter.Current is not null)
+				while(!windowOverran && !enumeratorExhausted)
                 {
-					count++;
+					if (windowStart <= sortedScrobblesIter.Current.Timestamp)
-				}
+					{
-
+						if(sortedScrobblesIter.Current.Timestamp < windowEnd)
-				while (sortedScrobblesIter.MoveNext() && counter <= sortedScrobblesIter.Current.Timestamp && sortedScrobblesIter.Current.Timestamp < windowEnd)
+                        {
-				{
+							count++;
-					count++;
+							if (!sortedScrobblesIter.MoveNext())
+                            {
+								enumeratorExhausted = true;
+							}
+						}
+						else
+                        {
+							windowOverran = true;
+                        }
+					}
 				}
 
 				yield return new CountSample()
 				{
-					TimeStamp = counter + (window / 2),
+					TimeStamp = windowStart + (window / 2),
 					Value = count
 				};
 			}
@@ -92,6 +104,21 @@ namespace Selector
 				};
 			}
 		}
+
+		public static IEnumerable<CountSample> CumulativeSum(this IEnumerable<CountSample> samples)
+		{
+			var sum = 0;
+			foreach(var sample in samples)
+            {
+				sum += sample.Value;
+
+				yield return new CountSample
+				{
+					TimeStamp = sample.TimeStamp,
+					Value = sum
+                };
+            }
+		}
 	}
 }
 

Selector/Watcher/BaseWatcher.cs

@@ -1,7 +1,5 @@
 using System;
-using System.Collections.Generic;
+using System.Diagnostics;
-using System.Linq.Expressions;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -15,10 +13,12 @@ namespace Selector
         protected readonly ILogger<BaseWatcher> Logger;
         public string Id { get; set; }
         public string SpotifyUsername { get; set; }
+        private Stopwatch ExecutionTimer { get; set; }
 
         public BaseWatcher(ILogger<BaseWatcher> logger = null)
         {
             Logger = logger ?? NullLogger<BaseWatcher>.Instance;
+            ExecutionTimer = new Stopwatch();
         }
 
         public abstract Task WatchOne(CancellationToken token);
@@ -28,6 +28,9 @@ namespace Selector
         {
             Logger.LogDebug("Starting watcher");
             while (true) {
+
+                ExecutionTimer.Start();
+
                 cancelToken.ThrowIfCancellationRequested();
 
                 try
@@ -38,9 +41,13 @@ namespace Selector
                 {
                     Logger.LogError(ex, "Exception occured while conducting single poll operation");
                 }
-                
+
-                Logger.LogTrace($"Finished watch one, delaying {PollPeriod}ms...");
+                ExecutionTimer.Stop();
-                await Task.Delay(PollPeriod, cancelToken);
+                var waitTime = decimal.ToInt32(Math.Max(0, PollPeriod - ExecutionTimer.ElapsedMilliseconds));
+                ExecutionTimer.Reset();
+
+                Logger.LogTrace($"Finished watch one, delaying \"{PollPeriod}\"ms (\"{waitTime}\"ms)...");
+                await Task.Delay(waitTime, cancelToken);
             }
         }