sarsoo/Selector
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into past

Browse Source
This commit is contained in:
andy 2022-11-13 22:10:19 +00:00 committed by GitHub
commit 2e726061ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 941 additions and 511 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View File

@ -9,12 +9,12 @@ jobs:
strategy:
fail-fast: false
matrix:
dotnet-version: [ '6.0.x' ]
dotnet-version: [ '7.0.x' ]
steps:
- uses: actions/checkout@v2
- name: Setup .NET Core SDK ${{ matrix.dotnet-version }}
uses: actions/setup-dotnet@v1.7.2
uses: actions/setup-dotnet@v3.0.3
with:
dotnet-version: ${{ matrix.dotnet-version }}
- name: Install Dependencies

5
Dockerfile.CLI
View File

@ -1,8 +1,9 @@
FROM mcr.microsoft.com/dotnet/sdk:6.0 AS base
FROM mcr.microsoft.com/dotnet/sdk:7.0 AS base
COPY *.sln .
COPY Selector/*.csproj ./Selector/
COPY Selector.Cache/*.csproj ./Selector.Cache/
COPY Selector.Data/*.csproj ./Selector.Data/
COPY Selector.Event/*.csproj ./Selector.Event/
COPY Selector.Model/*.csproj ./Selector.Model/
COPY Selector.CLI/*.csproj ./Selector.CLI/
@ -14,7 +15,7 @@ COPY . ./
FROM base as publish
RUN dotnet publish Selector.CLI/Selector.CLI.csproj -c Release -o /app --no-restore
FROM mcr.microsoft.com/dotnet/aspnet:6.0
FROM mcr.microsoft.com/dotnet/aspnet:7.0
WORKDIR /app
COPY --from=publish /app ./
ENV DOTNET_EnableDiagnostics=0

5
Dockerfile.Web
View File

@ -6,11 +6,12 @@ RUN npm ci
COPY ./Selector.Web ./
RUN npm run build
FROM mcr.microsoft.com/dotnet/sdk:6.0 AS base
FROM mcr.microsoft.com/dotnet/sdk:7.0 AS base
COPY *.sln .
COPY Selector/*.csproj ./Selector/
COPY Selector.Cache/*.csproj ./Selector.Cache/
COPY Selector.Data/*.csproj ./Selector.Data/
COPY Selector.Event/*.csproj ./Selector.Event/
COPY Selector.Model/*.csproj ./Selector.Model/
COPY Selector.Web/*.csproj ./Selector.Web/
@ -24,7 +25,7 @@ COPY --from=frontend /Selector.Web/wwwroot Selector.Web/wwwroot/
COPY --from=frontend /Selector.Web/wwwroot Selector.Web/wwwroot/
RUN dotnet publish Selector.Web/Selector.Web.csproj -c Release -o /app --no-restore
FROM mcr.microsoft.com/dotnet/aspnet:6.0
FROM mcr.microsoft.com/dotnet/aspnet:7.0
EXPOSE 80
WORKDIR /app
COPY --from=publish /app ./

2
Selector.CLI/Command/CommandContext.cs
View File

@ -3,6 +3,7 @@ using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Selector.Model;
using SpotifyAPI.Web;
using StackExchange.Redis;
namespace Selector.CLI
{
@ -11,6 +12,7 @@ namespace Selector.CLI
public RootOptions Config { get; set; }
public ILoggerFactory Logger { get; set; }
public ISpotifyClient Spotify { get; set; }
public ConnectionMultiplexer RedisMux { get; set; }
public DbContextOptionsBuilder<ApplicationDbContext> DatabaseConfig { get; set; }
public LastfmClient LastFmClient { get; set; }

16
Selector.CLI/Command/SpotHistoryCommand.cs
View File

@ -9,6 +9,7 @@ using System.CommandLine;
using System.CommandLine.Invocation;
using System.Linq;
using System.Collections.Generic;
using Selector.Cache;
namespace Selector.CLI
{
@ -28,24 +29,29 @@ namespace Selector.CLI
username.AddAlias("-u");
AddOption(username);
Handler = CommandHandler.Create((string connectionString, string path, string username) => Execute(connectionString, path, username));
Handler = CommandHandler.Create((string connection, string path, string username) => Execute(connection, path, username));
}
public static int Execute(string connectionString, string path, string username)
public static int Execute(string connection, string path, string username)
{
var streams = new List<FileStream>();
try
{
var context = new CommandContext().WithLogger().WithDb(connectionString).WithLastfmApi();
var context = new CommandContext().WithLogger().WithDb(connection).WithSpotify().WithRedis();
var logger = context.Logger.CreateLogger("Scrobble");
using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger<ApplicationDbContext>());
var historyPersister = new HistoryPersister(db, new DataJsonContext(), new()
{
Username = username
}, context.Logger.CreateLogger<HistoryPersister>());
Username = username,
Apply50PercentRule = true
},
durationPuller: new(context.Logger.CreateLogger<DurationPuller>(),
context.Spotify.Tracks,
cache: context.RedisMux.GetDatabase()),
logger: context.Logger.CreateLogger<HistoryPersister>());
logger.LogInformation("Preparing to parse from {} for {}", path, username);

17
Selector.CLI/Extensions/CommandContextExtensions.cs
View File

@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Selector.Model;
using SpotifyAPI.Web;
using StackExchange.Redis;
using System.Linq;
namespace Selector.CLI.Extensions
@ -91,5 +92,21 @@ namespace Selector.CLI.Extensions
return context;
}
public static CommandContext WithRedis(this CommandContext context)
{
if (context.Config is null)
{
context.WithConfig();
}
var connectionString = context.Config.RedisOptions.ConnectionString;
var connMulti = ConnectionMultiplexer.Connect(connectionString);
context.RedisMux = connMulti;
return context;
}
}
}

16
Selector.CLI/Selector.CLI.csproj
View File

@ -2,22 +2,22 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<StartupObject>Selector.CLI.Program</StartupObject>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.9">
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
<PackageReference Include="NLog" Version="5.0.4" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.0.4" />
<PackageReference Include="Quartz" Version="3.4.0" />

6
Selector.CLI/nlog.config
View File

@ -31,6 +31,9 @@
<!-- rules to map from logger name to target -->
<rules>
<logger name="Selector.*" minlevel="Debug" writeTo="logconsole" />
<logger name="Microsoft.*" minlevel="Warning" writeTo="logconsole" />
<!--<logger name="*" minlevel="Debug" writeTo="logfile" />-->
<logger name="Selector.*" minlevel="Debug" writeTo="logfile" />
<logger name="Microsoft.*" minlevel="Warning" writeTo="logfile" />
@ -38,8 +41,5 @@
<!--<logger name="*" minlevel="Trace" writeTo="tracefile" />-->
<logger name="Selector.*" minlevel="Debug" writeTo="tracefile" />
<logger name="Microsoft.*" minlevel="Warning" writeTo="tracefile" />
<logger name="Selector.*" minlevel="Debug" writeTo="logconsole" />
<logger name="Microsoft.*" minlevel="Warning" writeTo="logconsole" />
</rules>
</nlog>

26
Selector.Cache/Extensions/CacheExtensions.cs Normal file
View File

@ -0,0 +1,26 @@
using System;
using System.Threading.Tasks;
using StackExchange.Redis;
namespace Selector.Cache;
public static class CacheExtensions
{
public static async Task<int?> GetTrackDuration(this IDatabaseAsync cache, string trackId)
{
return (int?) await cache?.HashGetAsync(Key.Track(trackId), Key.Duration);
}
public static async Task SetTrackDuration(this IDatabaseAsync cache, string trackId, int duration, TimeSpan? expiry = null)
{
var trackCacheKey = Key.Track(trackId);
await cache?.HashSetAsync(trackCacheKey, Key.Duration, duration);
if(expiry is not null)
{
await cache?.KeyExpireAsync(trackCacheKey, expiry);
}
}
}

4
Selector.Cache/Key.cs
View File

@ -20,6 +20,7 @@ namespace Selector.Cache
public const string AudioFeatureName = "AUDIO_FEATURE";
public const string PlayCountName = "PLAY_COUNT";
public const string Duration = "DURATION";
public const string SpotifyName = "SPOTIFY";
public const string LastfmName = "LASTFM";
@ -34,6 +35,9 @@ namespace Selector.Cache
public static string CurrentlyPlaying(string user) => MajorNamespace(MinorNamespace(UserName, CurrentlyPlayingName), user);
public static readonly string AllCurrentlyPlaying = CurrentlyPlaying(All);
public static string Track(string trackId) => MajorNamespace(TrackName, trackId);
public static readonly string AllTracks = Track(All);
public static string AudioFeature(string trackId) => MajorNamespace(MinorNamespace(TrackName, AudioFeatureName), trackId);
public static readonly string AllAudioFeatures = AudioFeature(All);

4
Selector.Cache/Selector.Cache.csproj
View File

@ -1,14 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<EnableDefaultCompileItems>true</EnableDefaultCompileItems>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="StackExchange.Redis" Version="2.6.66" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
<PackageReference Include="SpotifyAPI.Web" Version="6.3.0" />
</ItemGroup>

183
Selector.Cache/Services/DurationPuller.cs Normal file
View File

@ -0,0 +1,183 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json.Linq;
using SpotifyAPI.Web;
using StackExchange.Redis;
namespace Selector.Cache
{
public class DurationPuller
{
private readonly IDatabaseAsync Cache;
private readonly ILogger<DurationPuller> Logger;
protected readonly ITracksClient SpotifyClient;
private int _retries = 0;
public DurationPuller(
ILogger<DurationPuller> logger,
ITracksClient spotifyClient,
IDatabaseAsync cache = null
)
{
Cache = cache;
Logger = logger;
SpotifyClient = spotifyClient;
}
public async Task<int?> Get(string uri)
{
if (string.IsNullOrWhiteSpace(uri)) throw new ArgumentNullException("No uri provided");
var trackId = uri.Split(":").Last();
var cachedVal = await Cache?.HashGetAsync(Key.Track(trackId), Key.Duration);
if (Cache is null || cachedVal == RedisValue.Null || cachedVal.IsNullOrEmpty)
{
try {
Logger.LogDebug("Missed cache, pulling");
var info = await SpotifyClient.Get(trackId);
await Cache?.SetTrackDuration(trackId, info.DurationMs, TimeSpan.FromDays(7));
_retries = 0;
return info.DurationMs;
}
catch (APIUnauthorizedException e)
{
Logger.LogError("Unauthorised error: [{message}] (should be refreshed and retried?)", e.Message);
throw e;
}
catch (APITooManyRequestsException e)
{
if(_retries <= 3)
{
Logger.LogWarning("Too many requests error, retrying ({}): [{message}]", e.RetryAfter, e.Message);
_retries++;
await Task.Delay(e.RetryAfter);
return await Get(uri);
}
else
{
Logger.LogError("Too many requests error, done retrying: [{message}]", e.Message);
throw e;
}
}
catch (APIException e)
{
if (_retries <= 3)
{
Logger.LogWarning("API error, retrying: [{message}]", e.Message);
_retries++;
await Task.Delay(TimeSpan.FromSeconds(2));
return await Get(uri);
}
else
{
Logger.LogError("API error, done retrying: [{message}]", e.Message);
throw e;
}
}
}
else
{
return (int?) cachedVal;
}
}
public async Task<IDictionary<string, int>> Get(IEnumerable<string> uri)
{
if (!uri.Any()) throw new ArgumentNullException("No URIs provided");
var ret = new Dictionary<string, int>();
var toPullFromSpotify = new List<string>();
foreach (var input in uri.Select(x => x.Split(":").Last()))
{
var cachedVal = await Cache?.HashGetAsync(Key.Track(input), Key.Duration);
if (Cache is null || cachedVal == RedisValue.Null || cachedVal.IsNullOrEmpty)
{
toPullFromSpotify.Add(input);
}
else
{
ret[input] = (int) cachedVal;
}
}
var retries = new List<string>();
foreach(var chunk in toPullFromSpotify.Chunk(50))
{
await PullChunk(chunk, ret);
await Task.Delay(TimeSpan.FromMilliseconds(500));
}
return ret;
}
private async Task PullChunk(IList<string> toPull, IDictionary<string, int> ret)
{
try
{
var info = await SpotifyClient.GetSeveral(new(toPull));
foreach (var resp in info.Tracks)
{
await Cache?.SetTrackDuration(resp.Id, resp.DurationMs, TimeSpan.FromDays(7));
ret[resp.Id] = (int)resp.DurationMs;
}
_retries = 0;
}
catch (APIUnauthorizedException e)
{
Logger.LogError("Unauthorised error: [{message}] (should be refreshed and retried?)", e.Message);
throw e;
}
catch (APITooManyRequestsException e)
{
if (_retries <= 3)
{
Logger.LogWarning("Too many requests error, retrying ({}): [{message}]", e.RetryAfter, e.Message);
_retries++;
await Task.Delay(e.RetryAfter);
await PullChunk(toPull, ret);
}
else
{
Logger.LogError("Too many requests error, done retrying: [{message}]", e.Message);
throw e;
}
}
catch (APIException e)
{
if (_retries <= 3)
{
Logger.LogWarning("API error, retrying: [{message}]", e.Message);
_retries++;
await Task.Delay(TimeSpan.FromSeconds(5));
await PullChunk(toPull, ret);
}
else
{
Logger.LogError("API error, done retrying: [{message}]", e.Message);
throw e;
}
}
}
}
}

156
Selector.Data/HistoryPersister.cs
View File

@ -1,6 +1,8 @@
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Selector.Cache;
using Selector.Model;
using static SpotifyAPI.Web.PlaylistRemoveItemsRequest;
namespace Selector.Data;
@ -17,14 +19,31 @@ public class HistoryPersister
private ApplicationDbContext Db { get; set; }
private DataJsonContext Json { get; set; }
private DurationPuller DurationPuller { get; set; }
private ILogger<HistoryPersister> Logger { get; set; }
public HistoryPersister(ApplicationDbContext db, DataJsonContext json, HistoryPersisterConfig config, ILogger<HistoryPersister> logger = null)
private readonly Dictionary<string, int> Durations;
public HistoryPersister(
ApplicationDbContext db,
DataJsonContext json,
HistoryPersisterConfig config,
DurationPuller durationPuller = null,
ILogger<HistoryPersister> logger = null)
{
Config = config;
Db = db;
Json = json;
DurationPuller = durationPuller;
Logger = logger;
if (config.Apply50PercentRule && DurationPuller is null)
{
throw new ArgumentNullException(nameof(DurationPuller));
}
Durations = new();
}
public void Process(string input)
@ -48,7 +67,7 @@ public class HistoryPersister
var parsed = await JsonSerializer.DeserializeAsync(singleInput, Json.EndSongArray);
songs = songs.Concat(parsed);
Logger?.LogDebug("Parsed {.2f} items for {}", parsed.Length, Config.Username);
Logger?.LogDebug("Parsed {:n0} items for {}", parsed.Length, Config.Username);
}
await Process(songs);
@ -56,47 +75,136 @@ public class HistoryPersister
public async Task Process(IEnumerable<EndSong> input)
{
var user = Db.Users.Single(u => u.UserName == Config.Username);
if (Config.InitialClear)
{
var latestTime = input.OrderBy(x => x.ts).Last().ts;
var time = DateTime.Parse(latestTime).ToUniversalTime();
Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.User.UserName == Config.Username && x.Timestamp <= time));
Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.UserId == user.Id && x.Timestamp <= time));
}
var user = Db.Users.Single(u => u.UserName == Config.Username);
var counter = 0;
var filtered = input.Where(x => x.ms_played > 30000)
var filtered = input.Where(x => x.ms_played > 30000
&& !string.IsNullOrWhiteSpace(x.master_metadata_track_name))
.DistinctBy(x => (x.offline_timestamp, x.ts, x.spotify_track_uri))
.ToArray();
Logger.LogInformation("{.2f} items after filtering", filtered.Length);
Logger.LogInformation("{:n0} items after filtering", filtered.Length);
foreach (var item in filtered)
var processedCounter = 0;
foreach (var item in filtered.Chunk(1000))
{
if(!string.IsNullOrWhiteSpace(item.master_metadata_track_name))
{
Db.SpotifyListen.Add(new()
{
TrackName = item.master_metadata_track_name,
AlbumName = item.master_metadata_album_album_name,
ArtistName = item.master_metadata_album_artist_name,
IEnumerable<EndSong> toPopulate = item;
Timestamp = DateTime.Parse(item.ts).ToUniversalTime(),
PlayedDuration = item.ms_played,
if (Config.Apply50PercentRule)
{
Logger.LogDebug("Validating tracks {:n0}/{:n0}", processedCounter + 1, filtered.Length);
TrackUri = item.spotify_track_uri,
toPopulate = Passes50PcRule(toPopulate);
}
Db.SpotifyListen.AddRange(toPopulate.Select(x => new SpotifyListen()
{
TrackName = x.master_metadata_track_name,
AlbumName = x.master_metadata_album_album_name,
ArtistName = x.master_metadata_album_artist_name,
Timestamp = DateTime.Parse(x.ts).ToUniversalTime(),
PlayedDuration = x.ms_played,
TrackUri = x.spotify_track_uri,
UserId = user.Id
});
}));
counter++;
}
processedCounter += item.Length;
}
Logger?.LogInformation("Added {} historical items for {}", counter, user.UserName);
Logger?.LogInformation("Saving {:n0} historical items for {}", processedCounter, user.UserName);
await Db.SaveChangesAsync();
Logger?.LogInformation("Added {:n0} historical items for {}", processedCounter, user.UserName);
}
private const int FOUR_MINUTES = 4 * 60 * 1000;
public async Task<bool> Passes50PcRule(EndSong song)
{
if (string.IsNullOrWhiteSpace(song.spotify_track_uri)) return true;
int duration;
if (Durations.TryGetValue(song.spotify_track_uri, out duration))
{
}
else
{
var pulledDuration = await DurationPuller.Get(song.spotify_track_uri);
if (pulledDuration is int d)
{
duration = d;
Durations.Add(song.spotify_track_uri, duration);
}
else
{
Logger.LogDebug("No duration returned for {}/{}", song.master_metadata_track_name, song.master_metadata_album_artist_name);
return true; // if can't get duration, just pass
}
}
return CheckDuration(song, duration);
}
public IEnumerable<EndSong> Passes50PcRule(IEnumerable<EndSong> inputTracks)
{
var toPullOverWire = new List<EndSong>();
// quick return items from local cache
foreach(var track in inputTracks)
{
if (string.IsNullOrWhiteSpace(track.spotify_track_uri)) yield return track;
if (Durations.TryGetValue(track.spotify_track_uri, out var duration))
{
if (CheckDuration(track, duration))
{
yield return track;
}
}
else
{
toPullOverWire.Add(track);
}
}
var pulledDuration = DurationPuller.Get(toPullOverWire.Select(x => x.spotify_track_uri)).Result;
// apply results to cache
foreach((var uri, var dur) in pulledDuration)
{
Durations[uri] = dur;
}
// check return acceptable tracks from pulled
foreach(var track in toPullOverWire)
{
if(pulledDuration.TryGetValue(track.spotify_track_uri, out var duration))
{
if(CheckDuration(track, duration))
{
yield return track;
}
}
else
{
yield return track;
}
}
}
public bool CheckDuration(EndSong song, int duration) => song.ms_played >= duration / 2 || song.ms_played >= FOUR_MINUTES;
}

5
Selector.Data/Selector.Data.csproj
View File

@ -1,16 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Selector.Model\Selector.Model.csproj" />
<ProjectReference Include="..\Selector.Cache\Selector.Cache.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.0" />
</ItemGroup>
</Project>

2
Selector.Event/Selector.Event.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>

4
Selector.Model/ApplicationDbContext.cs
View File

@ -70,6 +70,8 @@ namespace Selector.Model
modelBuilder.Entity<UserScrobble>()
.Property(s => s.ArtistName)
.UseCollation("case_insensitive");
//modelBuilder.Entity<UserScrobble>()
// .HasIndex(x => new { x.UserId, x.ArtistName, x.TrackName });
modelBuilder.Entity<TrackLastfmSpotifyMapping>().HasKey(s => s.SpotifyUri);
modelBuilder.Entity<TrackLastfmSpotifyMapping>()
@ -102,6 +104,8 @@ namespace Selector.Model
modelBuilder.Entity<SpotifyListen>()
.Property(s => s.ArtistName)
.UseCollation("case_insensitive");
//modelBuilder.Entity<SpotifyListen>()
// .HasIndex(x => new { x.UserId, x.ArtistName, x.TrackName });
SeedData.Seed(modelBuilder);
}

2
Selector.Model/Listen/IListenRepository.cs
View File

@ -5,7 +5,7 @@ namespace Selector.Model
{
public interface IListenRepository
{
IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false);
int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
}
}

20
Selector.Model/Listen/MetaListenRepository.cs
View File

@ -28,7 +28,8 @@ public class MetaListenRepository: IListenRepository
albumName: albumName,
artistName: artistName,
from: from,
to:to).Count();
to:to,
tracking: false).Count();
public IEnumerable<IListen> GetAll(
string includes = null,
@ -38,7 +39,9 @@ public class MetaListenRepository: IListenRepository
string albumName = null,
string artistName = null,
DateTime? from = null,
DateTime? to = null)
DateTime? to = null,
bool tracking = true,
bool orderTime = false)
{
var scrobbles = scrobbleRepository.GetAll(
include: includes,
@ -48,9 +51,9 @@ public class MetaListenRepository: IListenRepository
albumName: albumName,
artistName: artistName,
from: from,
to: to)
.OrderBy(x => x.Timestamp)
.ToArray();
to: to,
tracking: tracking,
orderTime: true).ToArray();
var spotListens = spotifyRepository.GetAll(
include: includes,
@ -60,10 +63,9 @@ public class MetaListenRepository: IListenRepository
albumName: albumName,
artistName: artistName,
from: from,
to: scrobbles.FirstOrDefault()?.Timestamp)
.OrderBy(x => x.Timestamp)
.ToArray();
to: scrobbles.FirstOrDefault()?.Timestamp,
tracking: tracking,
orderTime: orderTime);
return spotListens.Concat(scrobbles);
}

18
Selector.Model/Listen/SpotifyListenRepository.cs
View File

@ -39,10 +39,15 @@ namespace Selector.Model
return listens.FirstOrDefault();
}
private IQueryable<SpotifyListen> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
private IQueryable<SpotifyListen> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
{
var listens = db.SpotifyListen.AsQueryable();
if (!tracking)
{
listens = listens.AsNoTracking();
}
if (!string.IsNullOrWhiteSpace(include))
{
listens = listens.Include(include);
@ -92,11 +97,16 @@ namespace Selector.Model
listens = listens.Where(u => u.Timestamp < to.Value);
}
if (orderTime)
{
listens = listens.OrderBy(x => x.Timestamp);
}
return listens;
}
public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
=> GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
=> GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: tracking, orderTime: orderTime).AsEnumerable();
public void Remove(DateTime key)
{
@ -124,6 +134,6 @@ namespace Selector.Model
}
public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
=> GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
=> GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: false).Count();
}
}

2
Selector.Model/Scrobble/DBPlayCountPuller.cs
View File

@ -27,7 +27,7 @@ namespace Selector.Cache
var userScrobbleCount = ScrobbleRepository.Count(username: username);
var artistScrobbles = ScrobbleRepository.GetAll(username: username, artistName: artist).ToArray();
var artistScrobbles = ScrobbleRepository.GetAll(username: username, artistName: artist, tracking: false, orderTime: true).ToArray();
var albumScrobbles = artistScrobbles.Where(
s => s.AlbumName.Equals(album, StringComparison.CurrentCultureIgnoreCase)).ToArray();
var trackScrobbles = artistScrobbles.Where(

18
Selector.Model/Scrobble/ScrobbleRepository.cs
View File

@ -39,10 +39,15 @@ namespace Selector.Model
return scrobbles.FirstOrDefault();
}
private IQueryable<UserScrobble> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
private IQueryable<UserScrobble> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
{
var scrobbles = db.Scrobble.AsQueryable();
if (!tracking)
{
scrobbles = scrobbles.AsNoTracking();
}
if (!string.IsNullOrWhiteSpace(include))
{
scrobbles = scrobbles.Include(include);
@ -92,11 +97,16 @@ namespace Selector.Model
scrobbles = scrobbles.Where(u => u.Timestamp < to.Value);
}
if (orderTime)
{
scrobbles = scrobbles.OrderBy(x => x.Timestamp);
}
return scrobbles;
}
public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
=> GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null, bool tracking = true, bool orderTime = false)
=> GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: tracking, orderTime: orderTime).AsEnumerable();
public void Remove(int key)
{
@ -124,6 +134,6 @@ namespace Selector.Model
}
public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
=> GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
=> GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to, tracking: false).Count();
}
}

18
Selector.Model/Selector.Model.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<EnableDefaultCompileItems>true</EnableDefaultCompileItems>
<LangVersion>latest</LangVersion>
</PropertyGroup>
@ -12,20 +12,20 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Identity" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.9" />
<PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.9">
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="7.0.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.9">
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="7.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="6.0.9" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.0" />
</ItemGroup>
<ItemGroup>

2
Selector.Tests/Selector.Tests.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<IsPackable>false</IsPackable>
<LangVersion>latest</LangVersion>

16
Selector.Web/Selector.Web.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<LangVersion>latest</LangVersion>
<TypeScriptCompileBlocked>true</TypeScriptCompileBlocked>
</PropertyGroup>
@ -14,21 +14,21 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="6.0.9" />
<PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="6.0.9" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Identity.UI" Version="7.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="6.0.9" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="7.0.0" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.9">
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.9" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="7.0.0" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.0" />
<PackageReference Include="NLog" Version="5.0.4" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.0.4" />
<PackageReference Include="NLog.Web.AspNetCore" Version="5.1.4" />

2
Selector.Web/nlog.config
View File

@ -46,5 +46,7 @@
<logger name="System.Net.Http.*" maxlevel="Info" final="true" />
<!--<logger name="*" minlevel="Debug" writeTo="logfile" />-->
<logger name="Selector.*" minlevel="Info" writeTo="logfile" />
<logger name="Microsoft.*" minlevel="Warning" writeTo="logfile" />
</rules>
</nlog>

874
Selector.Web/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
Selector.Web/scripts/now.ts
View File

@ -10,6 +10,7 @@ import BaseInfoCard from "./Now/BaseInfoCard";
const connection = new signalR.HubConnectionBuilder()
.withUrl("/nowhub")
.withAutomaticReconnect()
.build();
connection.start()

4
Selector/Selector.csproj
View File

@ -1,13 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<EnableDefaultCompileItems>true</EnableDefaultCompileItems>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
<PackageReference Include="SpotifyAPI.Web" Version="6.3.0" />
<PackageReference Include="Inflatable.Lastfm" Version="1.2.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />

4
docker-compose.build.yml
View File

@ -27,11 +27,11 @@ services:
DOTNET_ENVIRONMENT: Production
redis:
image: redis:alpine
image: redis:7
ports:
- "6379:6379"
database:
image: postgres
image: postgres:14
ports:
- "5432:5432"
env_file: .env # set POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB

4
docker-compose.yml
View File

@ -25,12 +25,12 @@ services:
DOTNET_ENVIRONMENT: Production
redis:
image: redis:alpine
image: redis:7
restart: unless-stopped
ports:
- "6379:6379"
database:
image: postgres
image: postgres:14
restart: unless-stopped
ports:
- "5432:5432"