sarsoo/Selector
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases Wiki Activity

tweaking resampler, db count puller

Browse Source
This commit is contained in:
andy 2022-06-22 08:01:59 +01:00
parent 0b621e00bc
commit 63df0ab0c2
10 changed files with 171 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Selector.Model/Extensions/ServiceExtensions.cs
View File

@ -1,6 +1,6 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.Extensions.DependencyInjection;
using Selector.Cache;
using Selector.Model.Authorisation;
namespace Selector.Model.Extensions
@ -21,5 +21,12 @@ namespace Selector.Model.Extensions
services.AddScoped<IAuthorizationHandler, UserIsSelfAuthHandler>();
services.AddSingleton<IAuthorizationHandler, UserIsAdminAuthHandler>();
}
public static IServiceCollection AddDBPlayCountPuller(this IServiceCollection services)
{
services.AddTransient<DBPlayCountPuller>();
return services;
}
}
}

70
Selector.Model/Scrobble/DBPlayCountPuller.cs Normal file
View File

@ -0,0 +1,70 @@
using System;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Options;
using Selector.Model;
namespace Selector.Cache
{
public class DBPlayCountPuller
{
protected readonly IScrobbleRepository ScrobbleRepository;
private readonly IOptions<NowPlayingOptions> nowOptions;
public DBPlayCountPuller(
IOptions<NowPlayingOptions> options,
IScrobbleRepository scrobbleRepository
)
{
ScrobbleRepository = scrobbleRepository;
nowOptions = options;
}
public Task<PlayCount> Get(string username, string track, string artist, string album, string albumArtist)
{
if (string.IsNullOrWhiteSpace(username)) throw new ArgumentNullException("No username provided");
var userScrobbleCount = ScrobbleRepository.Count(username: username);
var artistScrobbles = ScrobbleRepository.GetAll(username: username, artistName: artist).ToArray();
var albumScrobbles = artistScrobbles.Where(
s => s.AlbumName.Equals(album, StringComparison.CurrentCultureIgnoreCase)).ToArray();
var trackScrobbles = artistScrobbles.Where(
s => s.TrackName.Equals(track, StringComparison.CurrentCultureIgnoreCase)).ToArray();
var postCalc = artistScrobbles.Resample(nowOptions.Value.ArtistResampleWindow).Select(s => s.Value).Sum();
//var postCalc = playCount.ArtistCountData.Select(s => s.Value).Sum();
Debug.Assert(postCalc == artistScrobbles.Count());
PlayCount playCount = new()
{
Username = username,
Artist = artistScrobbles.Count(),
Album = albumScrobbles.Count(),
Track = trackScrobbles.Count(),
User = userScrobbleCount,
ArtistCountData = artistScrobbles
.Resample(nowOptions.Value.ArtistResampleWindow)
//.ResampleByMonth()
.CumulativeSum()
.ToArray(),
AlbumCountData = albumScrobbles
.Resample(nowOptions.Value.AlbumResampleWindow)
//.ResampleByMonth()
.CumulativeSum()
.ToArray(),
TrackCountData = trackScrobbles
.Resample(nowOptions.Value.TrackResampleWindow)
//.ResampleByMonth()
.CumulativeSum()
.ToArray()
};
return Task.FromResult(playCount);
}
}
}

1
Selector.Model/Scrobble/IScrobbleRepository.cs
View File

@ -17,5 +17,6 @@ namespace Selector.Model
public void RemoveRange(IEnumerable<UserScrobble> scrobbles);
void Update(UserScrobble item);
Task<int> Save();
int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
}
}

13
Selector.Model/Scrobble/ScrobbleRepository.cs
View File

@ -39,7 +39,7 @@ namespace Selector.Model
return scrobbles.FirstOrDefault();
}
public IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
private IQueryable<UserScrobble> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
{
var scrobbles = db.Scrobble.AsQueryable();
@ -55,7 +55,8 @@ namespace Selector.Model
if (!string.IsNullOrWhiteSpace(username))
{
var user = db.Users.AsNoTracking().Where(u => u.UserName.Equals(username, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
var normalUsername = username.ToUpperInvariant();
var user = db.Users.AsNoTracking().Where(u => u.NormalizedUserName == normalUsername).FirstOrDefault();
if (user is not null)
{
scrobbles = scrobbles.Where(s => s.UserId == user.Id);
@ -91,9 +92,12 @@ namespace Selector.Model
scrobbles = scrobbles.Where(u => u.Timestamp < to.Value);
}
return scrobbles.AsEnumerable();
return scrobbles;
}
public IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
=> GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
public void Remove(int key)
{
Remove(Find(key));
@ -118,5 +122,8 @@ namespace Selector.Model
{
return db.SaveChangesAsync();
}
public int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
=> GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
}
}

35
Selector.Web/Hubs/NowPlayingHub.cs
View File

@ -30,6 +30,7 @@ namespace Selector.Web.Hubs
private readonly IDatabaseAsync Cache;
private readonly AudioFeaturePuller AudioFeaturePuller;
private readonly PlayCountPuller PlayCountPuller;
private readonly DBPlayCountPuller DBPlayCountPuller;
private readonly ApplicationDbContext Db;
private readonly IScrobbleRepository ScrobbleRepository;
@ -41,12 +42,14 @@ namespace Selector.Web.Hubs
ApplicationDbContext db,
IScrobbleRepository scrobbleRepository,
IOptions<NowPlayingOptions> options,
DBPlayCountPuller dbPlayCountPuller,
PlayCountPuller playCountPuller = null
)
{
Cache = cache;
AudioFeaturePuller = featurePuller;
PlayCountPuller = playCountPuller;
DBPlayCountPuller = dbPlayCountPuller;
Db = db;
ScrobbleRepository = scrobbleRepository;
nowOptions = options;
@ -103,36 +106,16 @@ namespace Selector.Web.Hubs
if (user.LastFmConnected())
{
var playCount = await PlayCountPuller.Get(user.LastFmUsername, track, artist, album, albumArtist);
PlayCount playCount;
if (user.ScrobbleSavingEnabled())
{
var artistScrobbles = ScrobbleRepository.GetAll(userId: user.Id, artistName: artist).ToArray();
playCount.Artist = artistScrobbles.Length;
playCount.ArtistCountData = artistScrobbles
//.Resample(nowOptions.Value.ArtistResampleWindow)
.ResampleByMonth()
.ToArray();
var postCalc = playCount.ArtistCountData.Select(s => s.Value).Sum();
Debug.Assert(postCalc == artistScrobbles.Count());
playCount.AlbumCountData = artistScrobbles
.Where(s => s.AlbumName.Equals(album, StringComparison.CurrentCultureIgnoreCase))
//.Resample(nowOptions.Value.AlbumResampleWindow)
.ResampleByMonth()
.ToArray();
playCount.TrackCountData = artistScrobbles
.Where(s => s.TrackName.Equals(track, StringComparison.CurrentCultureIgnoreCase))
//.Resample(nowOptions.Value.TrackResampleWindow)
.ResampleByMonth()
.ToArray();
playCount = await DBPlayCountPuller.Get(user.UserName, track, artist, album, albumArtist);
}
else
{
playCount = await PlayCountPuller.Get(user.LastFmUsername, track, artist, album, albumArtist);
}
await Clients.Caller.OnNewPlayCount(playCount);
}

18
Selector.Web/Options.cs
View File

@ -53,22 +53,4 @@ namespace Selector.Web
public bool Enabled { get; set; } = false;
public string ConnectionString { get; set; }
}
public class NowPlayingOptions
{
public const string Key = "Now";
public TimeSpan ArtistResampleWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan AlbumResampleWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan TrackResampleWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan ArtistDensityWindow { get; set; } = TimeSpan.FromDays(10);
public decimal ArtistDensityThreshold { get; set; } = 5;
public TimeSpan AlbumDensityWindow { get; set; } = TimeSpan.FromDays(10);
public decimal AlbumDensityThreshold { get; set; } = 5;
public TimeSpan TrackDensityWindow { get; set; } = TimeSpan.FromDays(10);
public decimal TrackDensityThreshold { get; set; } = 5;
}
}

1
Selector.Web/Startup.cs
View File

@ -54,6 +54,7 @@ namespace Selector.Web
services.AddDbContext<ApplicationDbContext>(options =>
options.UseNpgsql(Configuration.GetConnectionString("Default"))
);
services.AddDBPlayCountPuller();
services.AddTransient<IScrobbleRepository, ScrobbleRepository>();
services.AddIdentity<ApplicationUser, IdentityRole>()

22
Selector/Options.cs Normal file
View File

@ -0,0 +1,22 @@
using System;
namespace Selector
{
public class NowPlayingOptions
{
public const string Key = "Now";
public TimeSpan ArtistResampleWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan AlbumResampleWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan TrackResampleWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan ArtistDensityWindow { get; set; } = TimeSpan.FromDays(10);
public decimal ArtistDensityThreshold { get; set; } = 5;
public TimeSpan AlbumDensityWindow { get; set; } = TimeSpan.FromDays(10);
public decimal AlbumDensityThreshold { get; set; } = 5;
public TimeSpan TrackDensityWindow { get; set; } = TimeSpan.FromDays(10);
public decimal TrackDensityThreshold { get; set; } = 5;
}
}

47
Selector/Scrobble/Resampler.cs
View File

@ -26,25 +26,37 @@ namespace Selector
var earliest = sortedScrobbles.First().Timestamp;
var latest = sortedScrobbles.Last().Timestamp;
for (var counter = earliest; counter <= latest; counter += window)
var enumeratorExhausted = false;
for (var windowStart = earliest; windowStart <= latest; windowStart += window)
{
var windowEnd = counter + window;
var windowEnd = windowStart + window;
var count = 0;
var windowOverran = false;
if (sortedScrobblesIter.Current is not null)
while(!windowOverran && !enumeratorExhausted)
{
count++;
}
while (sortedScrobblesIter.MoveNext() && counter <= sortedScrobblesIter.Current.Timestamp && sortedScrobblesIter.Current.Timestamp < windowEnd)
{
count++;
if (windowStart <= sortedScrobblesIter.Current.Timestamp)
{
if(sortedScrobblesIter.Current.Timestamp < windowEnd)
{
count++;
if (!sortedScrobblesIter.MoveNext())
{
enumeratorExhausted = true;
}
}
else
{
windowOverran = true;
}
}
}
yield return new CountSample()
{
TimeStamp = counter + (window / 2),
TimeStamp = windowStart + (window / 2),
Value = count
};
}
@ -92,6 +104,21 @@ namespace Selector
};
}
}
public static IEnumerable<CountSample> CumulativeSum(this IEnumerable<CountSample> samples)
{
var sum = 0;
foreach(var sample in samples)
{
sum += sample.Value;
yield return new CountSample
{
TimeStamp = sample.TimeStamp,
Value = sum
};
}
}
}
}

19
Selector/Watcher/BaseWatcher.cs
View File

@ -1,7 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Text;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
@ -15,10 +13,12 @@ namespace Selector
protected readonly ILogger<BaseWatcher> Logger;
public string Id { get; set; }
public string SpotifyUsername { get; set; }
private Stopwatch ExecutionTimer { get; set; }
public BaseWatcher(ILogger<BaseWatcher> logger = null)
{
Logger = logger ?? NullLogger<BaseWatcher>.Instance;
ExecutionTimer = new Stopwatch();
}
public abstract Task WatchOne(CancellationToken token);
@ -28,6 +28,9 @@ namespace Selector
{
Logger.LogDebug("Starting watcher");
while (true) {
ExecutionTimer.Start();
cancelToken.ThrowIfCancellationRequested();
try
@ -38,9 +41,13 @@ namespace Selector
{
Logger.LogError(ex, "Exception occured while conducting single poll operation");
}
Logger.LogTrace($"Finished watch one, delaying {PollPeriod}ms...");
await Task.Delay(PollPeriod, cancelToken);
ExecutionTimer.Stop();
var waitTime = decimal.ToInt32(Math.Max(0, PollPeriod - ExecutionTimer.ElapsedMilliseconds));
ExecutionTimer.Reset();
Logger.LogTrace($"Finished watch one, delaying \"{PollPeriod}\"ms (\"{waitTime}\"ms)...");
await Task.Delay(waitTime, cancelToken);
}
}