From 2b8e0a273559c4ae8fd769d5272a870eb8487522 Mon Sep 17 00:00:00 2001 From: Andy Pack Date: Fri, 7 Oct 2022 18:29:33 +0100 Subject: [PATCH 1/3] skeleton of spotify history model and insert --- Selector.CLI/Command/SpotHistoryCommand.cs | 91 ++++++++++++ Selector.CLI/Extensions/ServiceExtensions.cs | 1 + Selector.CLI/Extensions/SpotifyExtensions.cs | 39 ++++++ Selector.CLI/Program.cs | 1 + Selector.CLI/Selector.CLI.csproj | 1 + Selector.Data/DataJsonContext.cs | 14 ++ Selector.Data/EndSong.cs | 27 ++++ Selector.Data/HistoryPersister.cs | 96 +++++++++++++ Selector.Data/Selector.Data.csproj | 16 +++ Selector.Model/ApplicationDbContext.cs | 14 +- .../Listen/ISpotifyListenRepository.cs | 21 +++ Selector.Model/Listen/SpotifyListen.cs | 14 ++ .../Listen/SpotifyListenRepository.cs | 129 ++++++++++++++++++ Selector.Model/Selector.Model.csproj | 4 + Selector.sln | 6 + Selector/Listen/IListen.cs | 13 ++ Selector/Listen/Listen.cs | 13 ++ Selector/Scrobble/Resampler.cs | 4 +- Selector/Scrobble/Scrobble.cs | 2 +- spotify-data.ipynb | 109 +++++++++++++++ 20 files changed, 609 insertions(+), 6 deletions(-) create mode 100644 Selector.CLI/Command/SpotHistoryCommand.cs create mode 100644 Selector.CLI/Extensions/SpotifyExtensions.cs create mode 100644 Selector.Data/DataJsonContext.cs create mode 100644 Selector.Data/EndSong.cs create mode 100644 Selector.Data/HistoryPersister.cs create mode 100644 Selector.Data/Selector.Data.csproj create mode 100644 Selector.Model/Listen/ISpotifyListenRepository.cs create mode 100644 Selector.Model/Listen/SpotifyListen.cs create mode 100644 Selector.Model/Listen/SpotifyListenRepository.cs create mode 100644 Selector/Listen/IListen.cs create mode 100644 Selector/Listen/Listen.cs create mode 100644 spotify-data.ipynb diff --git a/Selector.CLI/Command/SpotHistoryCommand.cs b/Selector.CLI/Command/SpotHistoryCommand.cs new file mode 100644 index 0000000..e9d5ea7 --- /dev/null +++ b/Selector.CLI/Command/SpotHistoryCommand.cs @@ -0,0 +1,91 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Selector.CLI.Extensions; +using Selector.Data; +using Selector.Model; +using System; +using System.IO; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Linq; +using System.Collections.Generic; + +namespace Selector.CLI +{ + public class SpotifyHistoryCommand : Command + { + public SpotifyHistoryCommand(string name, string description = null) : base(name, description) + { + var connectionString = new Option("--connection", "database to migrate"); + connectionString.AddAlias("-c"); + AddOption(connectionString); + + var pathString = new Option("--path", "path to find data"); + pathString.AddAlias("-i"); + AddOption(pathString); + + var username = new Option("--username", "user to pulls scrobbles for"); + username.AddAlias("-u"); + AddOption(username); + + Handler = CommandHandler.Create((string connectionString, string path, string username) => Execute(connectionString, path, username)); + } + + public static int Execute(string connectionString, string path, string username) + { + var streams = new List(); + + try + { + var context = new CommandContext().WithLogger().WithDb(connectionString).WithLastfmApi(); + var logger = context.Logger.CreateLogger("Scrobble"); + + using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger()); + + var historyPersister = new HistoryPersister(db, new DataJsonContext(), new() + { + Username = username + }, context.Logger.CreateLogger()); + + logger.LogInformation("Preparing to parse from {} for {}", path, username); + + var directoryContents = Directory.EnumerateFiles(path); + var endSongs = directoryContents.Where(f => f.Contains("endsong_")).ToArray(); + + + foreach(var file in endSongs) + { + streams.Add(File.OpenRead(file)); + } + + Console.WriteLine("Parse {0} historical data files? (y/n) ", endSongs.Length); + var input = Console.ReadLine(); + + if (input.Trim().Equals("y", StringComparison.OrdinalIgnoreCase)) + { + logger.LogInformation("Parsing files"); + + historyPersister.Process(streams).Wait(); + } + else + { + logger.LogInformation("Exiting"); + } + } + catch (Exception ex) + { + Console.WriteLine(ex); + return 1; + } + finally + { + foreach(var stream in streams) + { + stream.Dispose(); + } + } + + return 0; + } + } +} diff --git a/Selector.CLI/Extensions/ServiceExtensions.cs b/Selector.CLI/Extensions/ServiceExtensions.cs index 65f678c..1ac1ea8 100644 --- a/Selector.CLI/Extensions/ServiceExtensions.cs +++ b/Selector.CLI/Extensions/ServiceExtensions.cs @@ -119,6 +119,7 @@ namespace Selector.CLI.Extensions ); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddHostedService(); diff --git a/Selector.CLI/Extensions/SpotifyExtensions.cs b/Selector.CLI/Extensions/SpotifyExtensions.cs new file mode 100644 index 0000000..c775fa0 --- /dev/null +++ b/Selector.CLI/Extensions/SpotifyExtensions.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using SpotifyAPI.Web; + +namespace Selector.CLI.Extensions +{ + public static class SpotifyExtensions + { + public static async Task<(FullPlaylist, IEnumerable>)> GetPopulated(this ISpotifyClient client, string playlistId, ILogger logger = null) + { + try + { + var playlist = await client.Playlists.Get(playlistId); + var items = await client.Paginate(playlist.Tracks).ToListAsync(); + + return (playlist, items); + } + catch (APIUnauthorizedException e) + { + logger?.LogDebug("Unauthorised error: [{message}] (should be refreshed and retried?)", e.Message); + throw e; + } + catch (APITooManyRequestsException e) + { + logger?.LogDebug("Too many requests error: [{message}]", e.Message); + throw e; + } + catch (APIException e) + { + logger?.LogDebug("API error: [{message}]", e.Message); + throw e; + } + } + } +} + diff --git a/Selector.CLI/Program.cs b/Selector.CLI/Program.cs index fb0cc5c..e8dba6b 100644 --- a/Selector.CLI/Program.cs +++ b/Selector.CLI/Program.cs @@ -9,6 +9,7 @@ namespace Selector.CLI var cmd = new HostRootCommand(); cmd.AddCommand(new ScrobbleCommand("scrobble", "Manipulate scrobbles")); cmd.AddCommand(new MigrateCommand("migrate", "Migrate database")); + cmd.AddCommand(new SpotifyHistoryCommand("history", "Insert Spotify history")); cmd.Invoke(args); } diff --git a/Selector.CLI/Selector.CLI.csproj b/Selector.CLI/Selector.CLI.csproj index 3f5963a..21977e6 100644 --- a/Selector.CLI/Selector.CLI.csproj +++ b/Selector.CLI/Selector.CLI.csproj @@ -31,6 +31,7 @@ + diff --git a/Selector.Data/DataJsonContext.cs b/Selector.Data/DataJsonContext.cs new file mode 100644 index 0000000..141fc79 --- /dev/null +++ b/Selector.Data/DataJsonContext.cs @@ -0,0 +1,14 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace Selector.Data; + +[JsonSerializable(typeof(EndSong))] +[JsonSerializable(typeof(EndSong[]))] +public partial class DataJsonContext : JsonSerializerContext +{ + +} + diff --git a/Selector.Data/EndSong.cs b/Selector.Data/EndSong.cs new file mode 100644 index 0000000..1278401 --- /dev/null +++ b/Selector.Data/EndSong.cs @@ -0,0 +1,27 @@ +namespace Selector.Data; + +public record struct EndSong +{ + public string conn_country { get; set; } + public string episode_name { get; set; } + public string episode_show_name { get; set; } + public bool? incognito_mode { get; set; } + public string ip_addr_decrypted { get; set; } + public string master_metadata_album_album_name { get; set; } + public string master_metadata_album_artist_name { get; set; } + public string master_metadata_track_name { get; set; } + public int ms_played { get; set; } + public bool? offline { get; set; } + public long? offline_timestamp { get; set; } + public string platform { get; set; } + public string reason_end { get; set; } + public string reason_start { get; set; } + public bool shuffle { get; set; } + public bool? skipped { get; set; } + public string spotify_episode_uri { get; set; } + public string spotify_track_uri { get; set; } + public string ts { get; set; } + public string user_agent_decrypted { get; set; } + public string username { get; set; } +} + diff --git a/Selector.Data/HistoryPersister.cs b/Selector.Data/HistoryPersister.cs new file mode 100644 index 0000000..c252140 --- /dev/null +++ b/Selector.Data/HistoryPersister.cs @@ -0,0 +1,96 @@ +using System; +using System.Text.Json; +using Selector.Model; +using Microsoft.Extensions.Logging; +using System.Diagnostics.Metrics; + +namespace Selector.Data; + +public class HistoryPersisterConfig +{ + public string Username { get; set; } + public bool InitialClear { get; set; } = true; + public bool Apply50PercentRule { get; set; } = false; +} + +public class HistoryPersister +{ + private HistoryPersisterConfig Config { get; set; } + private ApplicationDbContext Db { get; set; } + private DataJsonContext Json { get; set; } + + private ILogger Logger { get; set; } + + public HistoryPersister(ApplicationDbContext db, DataJsonContext json, HistoryPersisterConfig config, ILogger logger = null) + { + Config = config; + Db = db; + Json = json; + Logger = logger; + } + + public void Process(string input) + { + var parsed = JsonSerializer.Deserialize(input, Json.EndSongArray); + Process(parsed).Wait(); + } + + public async Task Process(Stream input) + { + var parsed = await JsonSerializer.DeserializeAsync(input, Json.EndSongArray); + await Process(parsed); + } + + public async Task Process(IEnumerable input) + { + var songs = Enumerable.Empty(); + + foreach(var singleInput in input) + { + var parsed = await JsonSerializer.DeserializeAsync(singleInput, Json.EndSongArray); + songs = songs.Concat(parsed); + + Logger?.LogDebug("Parsed {} items for {}", parsed.Length, Config.Username); + } + + await Process(songs); + } + + public async Task Process(IEnumerable input) + { + if (Config.InitialClear) + { + Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.User.UserName == Config.Username)); + } + + var user = Db.Users.Single(u => u.UserName == Config.Username); + + var counter = 0; + + foreach(var item in input) + { + if(!string.IsNullOrWhiteSpace(item.master_metadata_track_name)) + { + Db.SpotifyListen.Add(new() + { + TrackName = item.master_metadata_track_name, + AlbumName = item.master_metadata_album_album_name, + ArtistName = item.master_metadata_album_artist_name, + + Timestamp = DateTime.Parse(item.ts).ToUniversalTime(), + PlayedDuration = item.ms_played, + + TrackUri = item.spotify_track_uri, + UserId = user.Id + }); + + counter++; + } + } + + Logger?.LogInformation("Added {} historical items for {}", counter, user.UserName); + + await Db.SaveChangesAsync(); + } +} + diff --git a/Selector.Data/Selector.Data.csproj b/Selector.Data/Selector.Data.csproj new file mode 100644 index 0000000..661434f --- /dev/null +++ b/Selector.Data/Selector.Data.csproj @@ -0,0 +1,16 @@ + + + + net6.0 + enable + + + + + + + + + + + diff --git a/Selector.Model/ApplicationDbContext.cs b/Selector.Model/ApplicationDbContext.cs index 7eba707..c656451 100644 --- a/Selector.Model/ApplicationDbContext.cs +++ b/Selector.Model/ApplicationDbContext.cs @@ -24,6 +24,8 @@ namespace Selector.Model public DbSet AlbumMapping { get; set; } public DbSet ArtistMapping { get; set; } + public DbSet SpotifyListen { get; set; } + public ApplicationDbContext( DbContextOptions options, ILogger logger @@ -85,9 +87,15 @@ namespace Selector.Model .Property(s => s.LastfmArtistName) .UseCollation("case_insensitive"); - modelBuilder.Entity().HasKey(s => s.SpotifyUri); - modelBuilder.Entity() - .Property(s => s.LastfmArtistName) + modelBuilder.Entity().HasKey(s => s.Timestamp); + modelBuilder.Entity() + .Property(s => s.TrackName) + .UseCollation("case_insensitive"); + modelBuilder.Entity() + .Property(s => s.AlbumName) + .UseCollation("case_insensitive"); + modelBuilder.Entity() + .Property(s => s.ArtistName) .UseCollation("case_insensitive"); SeedData.Seed(modelBuilder); diff --git a/Selector.Model/Listen/ISpotifyListenRepository.cs b/Selector.Model/Listen/ISpotifyListenRepository.cs new file mode 100644 index 0000000..7e5e5c5 --- /dev/null +++ b/Selector.Model/Listen/ISpotifyListenRepository.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Selector.Model +{ + public interface ISpotifyListenRepository + { + void Add(SpotifyListen item); + void AddRange(IEnumerable item); + IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + SpotifyListen Find(DateTime key, string include = null); + void Remove(DateTime key); + public void Remove(SpotifyListen scrobble); + public void RemoveRange(IEnumerable scrobbles); + void Update(SpotifyListen item); + Task Save(); + int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + } +} + diff --git a/Selector.Model/Listen/SpotifyListen.cs b/Selector.Model/Listen/SpotifyListen.cs new file mode 100644 index 0000000..d3f8570 --- /dev/null +++ b/Selector.Model/Listen/SpotifyListen.cs @@ -0,0 +1,14 @@ +using System; + +namespace Selector.Model; + +public class SpotifyListen: Listen +{ + public int? PlayedDuration { get; set; } + + public string TrackUri { get; set; } + + public string UserId { get; set; } + public ApplicationUser User { get; set; } +} + diff --git a/Selector.Model/Listen/SpotifyListenRepository.cs b/Selector.Model/Listen/SpotifyListenRepository.cs new file mode 100644 index 0000000..eefabfd --- /dev/null +++ b/Selector.Model/Listen/SpotifyListenRepository.cs @@ -0,0 +1,129 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Selector.Model +{ + public class SpotifyListenRepository : ISpotifyListenRepository + { + private readonly ApplicationDbContext db; + + public SpotifyListenRepository(ApplicationDbContext context) + { + db = context; + } + + public void Add(SpotifyListen item) + { + db.SpotifyListen.Add(item); + } + + public void AddRange(IEnumerable item) + { + db.SpotifyListen.AddRange(item); + } + + public SpotifyListen Find(DateTime key, string include = null) + { + var listens = db.SpotifyListen.Where(s => s.Timestamp == key); + + if (!string.IsNullOrWhiteSpace(include)) + { + listens = listens.Include(include); + } + + return listens.FirstOrDefault(); + } + + private IQueryable GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + { + var listens = db.SpotifyListen.AsQueryable(); + + if (!string.IsNullOrWhiteSpace(include)) + { + listens = listens.Include(include); + } + + if (!string.IsNullOrWhiteSpace(userId)) + { + listens = listens.Where(s => s.UserId == userId); + } + + if (!string.IsNullOrWhiteSpace(username)) + { + var normalUsername = username.ToUpperInvariant(); + var user = db.Users.AsNoTracking().Where(u => u.NormalizedUserName == normalUsername).FirstOrDefault(); + if (user is not null) + { + listens = listens.Where(s => s.UserId == user.Id); + } + else + { + listens = Enumerable.Empty().AsQueryable(); + } + } + + if (!string.IsNullOrWhiteSpace(trackName)) + { + listens = listens.Where(s => s.TrackName == trackName); + } + + if (!string.IsNullOrWhiteSpace(albumName)) + { + listens = listens.Where(s => s.AlbumName == albumName); + } + + if (!string.IsNullOrWhiteSpace(artistName)) + { + listens = listens.Where(s => s.ArtistName == artistName); + } + + if (from is not null) + { + listens = listens.Where(u => u.Timestamp >= from.Value); + } + + if (to is not null) + { + listens = listens.Where(u => u.Timestamp < to.Value); + } + + return listens; + } + + public IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable(); + + public void Remove(DateTime key) + { + Remove(Find(key)); + } + + public void Remove(SpotifyListen scrobble) + { + db.SpotifyListen.Remove(scrobble); + } + + public void RemoveRange(IEnumerable scrobbles) + { + db.SpotifyListen.RemoveRange(scrobbles); + } + + public void Update(SpotifyListen item) + { + db.SpotifyListen.Update(item); + } + + public Task Save() + { + return db.SaveChangesAsync(); + } + + public int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count(); + } +} diff --git a/Selector.Model/Selector.Model.csproj b/Selector.Model/Selector.Model.csproj index 10b162f..c6de22c 100644 --- a/Selector.Model/Selector.Model.csproj +++ b/Selector.Model/Selector.Model.csproj @@ -30,6 +30,10 @@ + + + + diff --git a/Selector.sln b/Selector.sln index d79642b..f7e9a73 100644 --- a/Selector.sln +++ b/Selector.sln @@ -24,6 +24,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Selector.Cache", "Selector. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Selector.Event", "Selector.Event\Selector.Event.csproj", "{C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Selector.Data", "Selector.Data\Selector.Data.csproj", "{CB62ACCB-94F1-4B78-A195-8B108B9E800D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -58,6 +60,10 @@ Global {C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Release|Any CPU.Build.0 = Release|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Selector/Listen/IListen.cs b/Selector/Listen/IListen.cs new file mode 100644 index 0000000..a43b222 --- /dev/null +++ b/Selector/Listen/IListen.cs @@ -0,0 +1,13 @@ +using System; + +namespace Selector; + +public interface IListen +{ + DateTime Timestamp { get; set; } + + string TrackName { get; set; } + string AlbumName { get; set; } + string ArtistName { get; set; } +} + diff --git a/Selector/Listen/Listen.cs b/Selector/Listen/Listen.cs new file mode 100644 index 0000000..c7a0d2c --- /dev/null +++ b/Selector/Listen/Listen.cs @@ -0,0 +1,13 @@ +using System; + +namespace Selector; + +public class Listen: IListen +{ + public string TrackName { get; set; } + public string AlbumName { get; set; } + public string ArtistName { get; set; } + + public DateTime Timestamp { get; set; } +} + diff --git a/Selector/Scrobble/Resampler.cs b/Selector/Scrobble/Resampler.cs index 254639b..7289d15 100644 --- a/Selector/Scrobble/Resampler.cs +++ b/Selector/Scrobble/Resampler.cs @@ -11,7 +11,7 @@ namespace Selector public static class Resampler { - public static IEnumerable Resample(this IEnumerable scrobbles, TimeSpan window) + public static IEnumerable Resample(this IEnumerable scrobbles, TimeSpan window) { var sortedScrobbles = scrobbles.OrderBy(s => s.Timestamp).ToList(); @@ -68,7 +68,7 @@ namespace Selector } } - public static IEnumerable ResampleByMonth(this IEnumerable scrobbles) + public static IEnumerable ResampleByMonth(this IEnumerable scrobbles) { var sortedScrobbles = scrobbles.OrderBy(s => s.Timestamp).ToList(); diff --git a/Selector/Scrobble/Scrobble.cs b/Selector/Scrobble/Scrobble.cs index 918cf05..fa4542b 100644 --- a/Selector/Scrobble/Scrobble.cs +++ b/Selector/Scrobble/Scrobble.cs @@ -2,7 +2,7 @@ namespace Selector { - public class Scrobble + public class Scrobble: IListen { public DateTime Timestamp { get; set; } public string TrackName { get; set; } diff --git a/spotify-data.ipynb b/spotify-data.ipynb new file mode 100644 index 0000000..2309439 --- /dev/null +++ b/spotify-data.ipynb @@ -0,0 +1,109 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from datetime import datetime\n", + "import random\n", + "from pprint import pprint" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total: 445582\n", + "Total Populated: 442034 (99.20%)\n", + "Total New: 130411 (29.27%)\n", + "{'conn_country': 'GB',\n", + " 'episode_name': None,\n", + " 'episode_show_name': None,\n", + " 'incognito_mode': False,\n", + " 'ip_addr_decrypted': '82.132.244.90',\n", + " 'master_metadata_album_album_name': 'Tales Told By Dead Friends',\n", + " 'master_metadata_album_artist_name': 'Mayday Parade',\n", + " 'master_metadata_track_name': 'Three Cheers For Five Years',\n", + " 'ms_played': 33274,\n", + " 'offline': False,\n", + " 'offline_timestamp': 0,\n", + " 'platform': 'iOS 9.2.1 (iPhone7,1)',\n", + " 'reason_end': 'fwdbtn',\n", + " 'reason_start': 'fwdbtn',\n", + " 'shuffle': True,\n", + " 'skipped': True,\n", + " 'spotify_episode_uri': None,\n", + " 'spotify_track_uri': 'spotify:track:1aw8gphDUzqalHEi9Z8M38',\n", + " 'ts': '2016-02-11T08:13:55Z',\n", + " 'user_agent_decrypted': 'unknown',\n", + " 'username': 'sarsoo'}\n" + ] + } + ], + "source": [ + "data = []\n", + "data_with_names = []\n", + "new_data = []\n", + "\n", + "folder = '/Users/andy/lab/backups/spotify-2022-03-07'\n", + "\n", + "for i in os.listdir(folder):\n", + " if i.startswith('endsong_'):\n", + " with open(f'{folder}/{i}') as f:\n", + " data += json.loads(f.read())\n", + "\n", + "data.sort(key = lambda a: a['ts'])\n", + "data_with_names = [i for i in data if i['master_metadata_track_name'] is not None]\n", + "new_data = [i for i in data_with_names if datetime.fromisoformat(i['ts'].split('T')[0]) < datetime(2017, 11, 3)]\n", + "\n", + "print(f'Total: {len(data)}')\n", + "print(f'Total Populated: {len(data_with_names)} ({len(data_with_names)/len(data)*100:.2f}%)')\n", + "print(f'Total New: {len(new_data)} ({len(new_data)/len(data)*100:.2f}%)')\n", + "\n", + "pprint(random.choice(new_data))\n", + "# print(min(i[0] for i in data))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "a0a5145e6c304e2a9afaf5b930a2955b950bd4b81fe94f7c42930f43f42762eb" + }, + "kernelspec": { + "display_name": "Python 3.10.7 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} -- 2.45.2 From 75415b4db455fd011e37594abf49969eb80e7c72 Mon Sep 17 00:00:00 2001 From: Andy Pack Date: Fri, 7 Oct 2022 23:33:54 +0100 Subject: [PATCH 2/3] integrated and first working --- Selector.CLI/Command/SpotHistoryCommand.cs | 1 - Selector.CLI/Extensions/ServiceExtensions.cs | 8 +- Selector.CLI/ScrobbleSaver.cs | 2 +- Selector.Data/HistoryPersister.cs | 10 +- Selector.Model/ApplicationDbContext.cs | 7 +- Selector.Model/Listen/IListenRepository.cs | 12 + .../Listen/ISpotifyListenRepository.cs | 6 +- Selector.Model/Listen/IUserListen.cs | 10 + Selector.Model/Listen/MetaListenRepository.cs | 75 +++ Selector.Model/Listen/SpotifyListen.cs | 4 +- .../Listen/SpotifyListenRepository.cs | 6 +- .../20221007214100_SpotifyHistory.Designer.cs | 491 ++++++++++++++++++ .../20221007214100_SpotifyHistory.cs | 63 +++ .../ApplicationDbContextModelSnapshot.cs | 52 +- Selector.Model/Scrobble/DBPlayCountPuller.cs | 4 +- .../Scrobble/IScrobbleRepository.cs | 6 +- Selector.Model/Scrobble/ScrobbleRepository.cs | 6 +- Selector.Model/Scrobble/UserScrobble.cs | 2 +- Selector.Web/Startup.cs | 6 +- Selector/Scrobble/PlayDensity.cs | 6 +- Selector/Scrobble/ScrobbleMatcher.cs | 26 +- 21 files changed, 761 insertions(+), 42 deletions(-) create mode 100644 Selector.Model/Listen/IListenRepository.cs create mode 100644 Selector.Model/Listen/IUserListen.cs create mode 100644 Selector.Model/Listen/MetaListenRepository.cs create mode 100644 Selector.Model/Migrations/20221007214100_SpotifyHistory.Designer.cs create mode 100644 Selector.Model/Migrations/20221007214100_SpotifyHistory.cs diff --git a/Selector.CLI/Command/SpotHistoryCommand.cs b/Selector.CLI/Command/SpotHistoryCommand.cs index e9d5ea7..4b073f3 100644 --- a/Selector.CLI/Command/SpotHistoryCommand.cs +++ b/Selector.CLI/Command/SpotHistoryCommand.cs @@ -52,7 +52,6 @@ namespace Selector.CLI var directoryContents = Directory.EnumerateFiles(path); var endSongs = directoryContents.Where(f => f.Contains("endsong_")).ToArray(); - foreach(var file in endSongs) { streams.Add(File.OpenRead(file)); diff --git a/Selector.CLI/Extensions/ServiceExtensions.cs b/Selector.CLI/Extensions/ServiceExtensions.cs index 1ac1ea8..0f155c3 100644 --- a/Selector.CLI/Extensions/ServiceExtensions.cs +++ b/Selector.CLI/Extensions/ServiceExtensions.cs @@ -118,8 +118,12 @@ namespace Selector.CLI.Extensions options.UseNpgsql(config.DatabaseOptions.ConnectionString) ); - services.AddTransient(); - services.AddTransient(); + services.AddTransient() + .AddTransient(); + + //services.AddTransient(); + services.AddTransient(); + services.AddTransient(); services.AddHostedService(); diff --git a/Selector.CLI/ScrobbleSaver.cs b/Selector.CLI/ScrobbleSaver.cs index 2c0c652..3cfbe27 100644 --- a/Selector.CLI/ScrobbleSaver.cs +++ b/Selector.CLI/ScrobbleSaver.cs @@ -112,7 +112,7 @@ namespace Selector logger.LogDebug("Identifying difference sets"); var time = Stopwatch.StartNew(); - (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles); + (var toAdd, var toRemove) = ListenMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles); time.Stop(); logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds); diff --git a/Selector.Data/HistoryPersister.cs b/Selector.Data/HistoryPersister.cs index c252140..f9e3cc5 100644 --- a/Selector.Data/HistoryPersister.cs +++ b/Selector.Data/HistoryPersister.cs @@ -50,7 +50,7 @@ public class HistoryPersister var parsed = await JsonSerializer.DeserializeAsync(singleInput, Json.EndSongArray); songs = songs.Concat(parsed); - Logger?.LogDebug("Parsed {} items for {}", parsed.Length, Config.Username); + Logger?.LogDebug("Parsed {.2f} items for {}", parsed.Length, Config.Username); } await Process(songs); @@ -67,7 +67,13 @@ public class HistoryPersister var counter = 0; - foreach(var item in input) + var filtered = input.Where(x => x.ms_played > 20000) + .DistinctBy(x => (x.offline_timestamp, x.ts, x.spotify_track_uri)) + .ToArray(); + + Logger.LogInformation("{.2f} items after filtering", filtered.Length); + + foreach (var item in filtered) { if(!string.IsNullOrWhiteSpace(item.master_metadata_track_name)) { diff --git a/Selector.Model/ApplicationDbContext.cs b/Selector.Model/ApplicationDbContext.cs index c656451..492a17d 100644 --- a/Selector.Model/ApplicationDbContext.cs +++ b/Selector.Model/ApplicationDbContext.cs @@ -87,7 +87,12 @@ namespace Selector.Model .Property(s => s.LastfmArtistName) .UseCollation("case_insensitive"); - modelBuilder.Entity().HasKey(s => s.Timestamp); + modelBuilder.Entity().HasKey(s => s.SpotifyUri); + modelBuilder.Entity() + .Property(s => s.LastfmArtistName) + .UseCollation("case_insensitive"); + + modelBuilder.Entity().HasKey(s => s.Id); modelBuilder.Entity() .Property(s => s.TrackName) .UseCollation("case_insensitive"); diff --git a/Selector.Model/Listen/IListenRepository.cs b/Selector.Model/Listen/IListenRepository.cs new file mode 100644 index 0000000..22861ed --- /dev/null +++ b/Selector.Model/Listen/IListenRepository.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; + +namespace Selector.Model +{ + public interface IListenRepository + { + IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + } +} + diff --git a/Selector.Model/Listen/ISpotifyListenRepository.cs b/Selector.Model/Listen/ISpotifyListenRepository.cs index 7e5e5c5..bf7902a 100644 --- a/Selector.Model/Listen/ISpotifyListenRepository.cs +++ b/Selector.Model/Listen/ISpotifyListenRepository.cs @@ -4,18 +4,18 @@ using System.Threading.Tasks; namespace Selector.Model { - public interface ISpotifyListenRepository + public interface ISpotifyListenRepository: IListenRepository { void Add(SpotifyListen item); void AddRange(IEnumerable item); - IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + //IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); SpotifyListen Find(DateTime key, string include = null); void Remove(DateTime key); public void Remove(SpotifyListen scrobble); public void RemoveRange(IEnumerable scrobbles); void Update(SpotifyListen item); Task Save(); - int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + //int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); } } diff --git a/Selector.Model/Listen/IUserListen.cs b/Selector.Model/Listen/IUserListen.cs new file mode 100644 index 0000000..1369048 --- /dev/null +++ b/Selector.Model/Listen/IUserListen.cs @@ -0,0 +1,10 @@ +using System; + +namespace Selector.Model; + +public interface IUserListen: IListen +{ + string UserId { get; set; } + ApplicationUser User { get; set; } +} + diff --git a/Selector.Model/Listen/MetaListenRepository.cs b/Selector.Model/Listen/MetaListenRepository.cs new file mode 100644 index 0000000..fa997c7 --- /dev/null +++ b/Selector.Model/Listen/MetaListenRepository.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Selector.Model; + +public enum PreferenceMode +{ + Greedy, LastFm, Spotify +} + +public class MetaListenRepository: IListenRepository +{ + private readonly IScrobbleRepository scrobbleRepository; + private readonly ISpotifyListenRepository spotifyRepository; + + public MetaListenRepository(IScrobbleRepository scrobbleRepository, ISpotifyListenRepository listenRepository) + { + this.scrobbleRepository = scrobbleRepository; + spotifyRepository = listenRepository; + } + + public int Count( + string userId = null, + string username = null, + string trackName = null, + string albumName = null, + string artistName = null, + DateTime? from = null, + DateTime? to = null) + { + throw new NotImplementedException(); + } + + public IEnumerable GetAll( + string includes = null, + string userId = null, + string username = null, + string trackName = null, + string albumName = null, + string artistName = null, + DateTime? from = null, + DateTime? to = null) + { + var scrobbles = scrobbleRepository.GetAll( + include: includes, + userId: userId, + username: username, + trackName: trackName, + albumName: albumName, + artistName: artistName, + from: from, + to: to) + .OrderBy(x => x.Timestamp) + .ToArray(); + + var spotListens = spotifyRepository.GetAll( + include: includes, + userId: userId, + username: username, + trackName: trackName, + albumName: albumName, + artistName: artistName, + from: from, + to: to) + .OrderBy(x => x.Timestamp) + .ToArray(); + + var scrobbleIter = scrobbles.GetEnumerator(); + var spotIter = spotListens.GetEnumerator(); + + return scrobbles; + } +} + diff --git a/Selector.Model/Listen/SpotifyListen.cs b/Selector.Model/Listen/SpotifyListen.cs index d3f8570..8491ff3 100644 --- a/Selector.Model/Listen/SpotifyListen.cs +++ b/Selector.Model/Listen/SpotifyListen.cs @@ -2,8 +2,10 @@ namespace Selector.Model; -public class SpotifyListen: Listen +public class SpotifyListen: Listen, IUserListen { + public int Id { get; set; } + public int? PlayedDuration { get; set; } public string TrackUri { get; set; } diff --git a/Selector.Model/Listen/SpotifyListenRepository.cs b/Selector.Model/Listen/SpotifyListenRepository.cs index eefabfd..ce728b4 100644 --- a/Selector.Model/Listen/SpotifyListenRepository.cs +++ b/Selector.Model/Listen/SpotifyListenRepository.cs @@ -95,7 +95,7 @@ namespace Selector.Model return listens; } - public IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + public IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable(); public void Remove(DateTime key) @@ -123,7 +123,7 @@ namespace Selector.Model return db.SaveChangesAsync(); } - public int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) - => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count(); + public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count(); } } diff --git a/Selector.Model/Migrations/20221007214100_SpotifyHistory.Designer.cs b/Selector.Model/Migrations/20221007214100_SpotifyHistory.Designer.cs new file mode 100644 index 0000000..95dfb0f --- /dev/null +++ b/Selector.Model/Migrations/20221007214100_SpotifyHistory.Designer.cs @@ -0,0 +1,491 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Selector.Model; + +#nullable disable + +namespace Selector.Model.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20221007214100_SpotifyHistory")] + partial class SpotifyHistory + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False") + .HasAnnotation("ProductVersion", "6.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex"); + + b.ToTable("AspNetRoles", (string)null); + + b.HasData( + new + { + Id = "00c64c0a-3387-4933-9575-83443fa9092b", + ConcurrencyStamp = "4b4a37c7-cc65-485a-ac0e-d88ef6dede78", + Name = "Admin", + NormalizedName = "ADMIN" + }); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("RoleId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("ProviderKey") + .HasColumnType("text"); + + b.Property("ProviderDisplayName") + .HasColumnType("text"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("text"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("text"); + + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Value") + .HasColumnType("text"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens", (string)null); + }); + + modelBuilder.Entity("Selector.Model.AlbumLastfmSpotifyMapping", b => + { + b.Property("SpotifyUri") + .HasColumnType("text"); + + b.Property("LastfmAlbumName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("LastfmArtistName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.HasKey("SpotifyUri"); + + b.ToTable("AlbumMapping"); + }); + + modelBuilder.Entity("Selector.Model.ApplicationUser", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("AccessFailedCount") + .HasColumnType("integer"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("EmailConfirmed") + .HasColumnType("boolean"); + + b.Property("LastFmUsername") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("LockoutEnabled") + .HasColumnType("boolean"); + + b.Property("LockoutEnd") + .HasColumnType("timestamp with time zone"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("PasswordHash") + .HasColumnType("text"); + + b.Property("PhoneNumber") + .HasColumnType("text"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("boolean"); + + b.Property("SaveScrobbles") + .HasColumnType("boolean"); + + b.Property("SecurityStamp") + .HasColumnType("text"); + + b.Property("SpotifyAccessToken") + .HasColumnType("text"); + + b.Property("SpotifyIsLinked") + .HasColumnType("boolean"); + + b.Property("SpotifyLastRefresh") + .HasColumnType("timestamp with time zone"); + + b.Property("SpotifyRefreshToken") + .HasColumnType("text"); + + b.Property("SpotifyTokenExpiry") + .HasColumnType("integer"); + + b.Property("TwoFactorEnabled") + .HasColumnType("boolean"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex"); + + b.ToTable("AspNetUsers", (string)null); + }); + + modelBuilder.Entity("Selector.Model.ArtistLastfmSpotifyMapping", b => + { + b.Property("SpotifyUri") + .HasColumnType("text"); + + b.Property("LastfmArtistName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.HasKey("SpotifyUri"); + + b.ToTable("ArtistMapping"); + }); + + modelBuilder.Entity("Selector.Model.SpotifyListen", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AlbumName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("ArtistName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("PlayedDuration") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("TrackName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("TrackUri") + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("SpotifyListen"); + }); + + modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b => + { + b.Property("SpotifyUri") + .HasColumnType("text"); + + b.Property("LastfmArtistName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("LastfmTrackName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.HasKey("SpotifyUri"); + + b.ToTable("TrackMapping"); + }); + + modelBuilder.Entity("Selector.Model.UserScrobble", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AlbumArtistName") + .HasColumnType("text"); + + b.Property("AlbumName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("ArtistName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("TrackName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("Scrobble"); + }); + + modelBuilder.Entity("Selector.Model.Watcher", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("Watcher"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("Selector.Model.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("Selector.Model.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Selector.Model.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("Selector.Model.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Selector.Model.SpotifyListen", b => + { + b.HasOne("Selector.Model.ApplicationUser", "User") + .WithMany() + .HasForeignKey("UserId"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Selector.Model.UserScrobble", b => + { + b.HasOne("Selector.Model.ApplicationUser", "User") + .WithMany("Scrobbles") + .HasForeignKey("UserId"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Selector.Model.Watcher", b => + { + b.HasOne("Selector.Model.ApplicationUser", "User") + .WithMany("Watchers") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Selector.Model.ApplicationUser", b => + { + b.Navigation("Scrobbles"); + + b.Navigation("Watchers"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Selector.Model/Migrations/20221007214100_SpotifyHistory.cs b/Selector.Model/Migrations/20221007214100_SpotifyHistory.cs new file mode 100644 index 0000000..a127b88 --- /dev/null +++ b/Selector.Model/Migrations/20221007214100_SpotifyHistory.cs @@ -0,0 +1,63 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Selector.Model.Migrations +{ + public partial class SpotifyHistory : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "SpotifyListen", + columns: table => new + { + Id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + PlayedDuration = table.Column(type: "integer", nullable: true), + TrackUri = table.Column(type: "text", nullable: true), + UserId = table.Column(type: "text", nullable: true), + TrackName = table.Column(type: "text", nullable: true, collation: "case_insensitive"), + AlbumName = table.Column(type: "text", nullable: true, collation: "case_insensitive"), + ArtistName = table.Column(type: "text", nullable: true, collation: "case_insensitive"), + Timestamp = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_SpotifyListen", x => x.Id); + table.ForeignKey( + name: "FK_SpotifyListen_AspNetUsers_UserId", + column: x => x.UserId, + principalTable: "AspNetUsers", + principalColumn: "Id"); + }); + + migrationBuilder.UpdateData( + table: "AspNetRoles", + keyColumn: "Id", + keyValue: "00c64c0a-3387-4933-9575-83443fa9092b", + column: "ConcurrencyStamp", + value: "4b4a37c7-cc65-485a-ac0e-d88ef6dede78"); + + migrationBuilder.CreateIndex( + name: "IX_SpotifyListen_UserId", + table: "SpotifyListen", + column: "UserId"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "SpotifyListen"); + + migrationBuilder.UpdateData( + table: "AspNetRoles", + keyColumn: "Id", + keyValue: "00c64c0a-3387-4933-9575-83443fa9092b", + column: "ConcurrencyStamp", + value: "ec454f56-2b26-4bd8-be8e-a7fd34981ac2"); + } + } +} diff --git a/Selector.Model/Migrations/ApplicationDbContextModelSnapshot.cs b/Selector.Model/Migrations/ApplicationDbContextModelSnapshot.cs index d79f2ca..d296389 100644 --- a/Selector.Model/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/Selector.Model/Migrations/ApplicationDbContextModelSnapshot.cs @@ -18,7 +18,7 @@ namespace Selector.Model.Migrations #pragma warning disable 612, 618 modelBuilder .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False") - .HasAnnotation("ProductVersion", "6.0.2") + .HasAnnotation("ProductVersion", "6.0.9") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); @@ -52,7 +52,7 @@ namespace Selector.Model.Migrations new { Id = "00c64c0a-3387-4933-9575-83443fa9092b", - ConcurrencyStamp = "ec454f56-2b26-4bd8-be8e-a7fd34981ac2", + ConcurrencyStamp = "4b4a37c7-cc65-485a-ac0e-d88ef6dede78", Name = "Admin", NormalizedName = "ADMIN" }); @@ -282,6 +282,45 @@ namespace Selector.Model.Migrations b.ToTable("ArtistMapping"); }); + modelBuilder.Entity("Selector.Model.SpotifyListen", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AlbumName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("ArtistName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("PlayedDuration") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("TrackName") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("TrackUri") + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("SpotifyListen"); + }); + modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b => { b.Property("SpotifyUri") @@ -409,6 +448,15 @@ namespace Selector.Model.Migrations .IsRequired(); }); + modelBuilder.Entity("Selector.Model.SpotifyListen", b => + { + b.HasOne("Selector.Model.ApplicationUser", "User") + .WithMany() + .HasForeignKey("UserId"); + + b.Navigation("User"); + }); + modelBuilder.Entity("Selector.Model.UserScrobble", b => { b.HasOne("Selector.Model.ApplicationUser", "User") diff --git a/Selector.Model/Scrobble/DBPlayCountPuller.cs b/Selector.Model/Scrobble/DBPlayCountPuller.cs index 802326c..2a3f6e2 100644 --- a/Selector.Model/Scrobble/DBPlayCountPuller.cs +++ b/Selector.Model/Scrobble/DBPlayCountPuller.cs @@ -9,12 +9,12 @@ namespace Selector.Cache { public class DBPlayCountPuller { - protected readonly IScrobbleRepository ScrobbleRepository; + protected readonly IListenRepository ScrobbleRepository; private readonly IOptions nowOptions; public DBPlayCountPuller( IOptions options, - IScrobbleRepository scrobbleRepository + IListenRepository scrobbleRepository ) { ScrobbleRepository = scrobbleRepository; diff --git a/Selector.Model/Scrobble/IScrobbleRepository.cs b/Selector.Model/Scrobble/IScrobbleRepository.cs index a59994d..030034f 100644 --- a/Selector.Model/Scrobble/IScrobbleRepository.cs +++ b/Selector.Model/Scrobble/IScrobbleRepository.cs @@ -6,17 +6,17 @@ using System.Threading.Tasks; namespace Selector.Model { - public interface IScrobbleRepository + public interface IScrobbleRepository: IListenRepository { void Add(UserScrobble item); void AddRange(IEnumerable item); - IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + //IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); UserScrobble Find(int key, string include = null); void Remove(int key); public void Remove(UserScrobble scrobble); public void RemoveRange(IEnumerable scrobbles); void Update(UserScrobble item); Task Save(); - int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + //int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); } } diff --git a/Selector.Model/Scrobble/ScrobbleRepository.cs b/Selector.Model/Scrobble/ScrobbleRepository.cs index 77dcb9f..90b5386 100644 --- a/Selector.Model/Scrobble/ScrobbleRepository.cs +++ b/Selector.Model/Scrobble/ScrobbleRepository.cs @@ -95,7 +95,7 @@ namespace Selector.Model return scrobbles; } - public IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + public IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable(); public void Remove(int key) @@ -123,7 +123,7 @@ namespace Selector.Model return db.SaveChangesAsync(); } - public int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) - => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count(); + public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count(); } } diff --git a/Selector.Model/Scrobble/UserScrobble.cs b/Selector.Model/Scrobble/UserScrobble.cs index 5a379d8..fce0be8 100644 --- a/Selector.Model/Scrobble/UserScrobble.cs +++ b/Selector.Model/Scrobble/UserScrobble.cs @@ -2,7 +2,7 @@ namespace Selector.Model { - public class UserScrobble: Scrobble + public class UserScrobble: Scrobble, IUserListen { public int Id { get; set; } public string UserId { get; set; } diff --git a/Selector.Web/Startup.cs b/Selector.Web/Startup.cs index ea9f355..dd82986 100644 --- a/Selector.Web/Startup.cs +++ b/Selector.Web/Startup.cs @@ -64,7 +64,11 @@ namespace Selector.Web options.UseNpgsql(Configuration.GetConnectionString("Default")) ); services.AddDBPlayCountPuller(); - services.AddTransient(); + services.AddTransient() + .AddTransient(); + + //services.AddTransient(); + services.AddTransient(); services.AddIdentity() .AddEntityFrameworkStores() diff --git a/Selector/Scrobble/PlayDensity.cs b/Selector/Scrobble/PlayDensity.cs index 1622e11..445af28 100644 --- a/Selector/Scrobble/PlayDensity.cs +++ b/Selector/Scrobble/PlayDensity.cs @@ -6,9 +6,9 @@ namespace Selector { public static class PlayDensity { - public static decimal Density(this IEnumerable scrobbles, TimeSpan window) => scrobbles.Density(DateTime.UtcNow - window, DateTime.UtcNow); + public static decimal Density(this IEnumerable scrobbles, TimeSpan window) => scrobbles.Density(DateTime.UtcNow - window, DateTime.UtcNow); - public static decimal Density(this IEnumerable scrobbles, DateTime from, DateTime to) + public static decimal Density(this IEnumerable scrobbles, DateTime from, DateTime to) { var filteredScrobbles = scrobbles.Where(s => s.Timestamp > from && s.Timestamp < to); @@ -17,7 +17,7 @@ namespace Selector return filteredScrobbles.Count() / dayDelta; } - public static decimal Density(this IEnumerable scrobbles) + public static decimal Density(this IEnumerable scrobbles) { var minDate = scrobbles.Select(s => s.Timestamp).Min(); var maxDate = scrobbles.Select(s => s.Timestamp).Max(); diff --git a/Selector/Scrobble/ScrobbleMatcher.cs b/Selector/Scrobble/ScrobbleMatcher.cs index 6daa5c9..c17fcfa 100644 --- a/Selector/Scrobble/ScrobbleMatcher.cs +++ b/Selector/Scrobble/ScrobbleMatcher.cs @@ -6,22 +6,22 @@ using System.Linq; namespace Selector { - public static class ScrobbleMatcher + public static class ListenMatcher { - public static bool MatchTime(Scrobble nativeScrobble, LastTrack serviceScrobble) + public static bool MatchTime(IListen nativeScrobble, LastTrack serviceScrobble) => serviceScrobble.TimePlayed.Equals(nativeScrobble); - public static bool MatchTime(Scrobble nativeScrobble, Scrobble serviceScrobble) + public static bool MatchTime(IListen nativeScrobble, IListen serviceScrobble) => serviceScrobble.Timestamp.Equals(nativeScrobble.Timestamp); - public static (IEnumerable, IEnumerable) IdentifyDiffs(IEnumerable existing, IEnumerable toApply, bool matchContents = true) + public static (IEnumerable, IEnumerable) IdentifyDiffs(IEnumerable existing, IEnumerable toApply, bool matchContents = true) { existing = existing.OrderBy(s => s.Timestamp); toApply = toApply.OrderBy(s => s.Timestamp); var toApplyIter = toApply.GetEnumerator(); - var toAdd = new List(); - var toRemove = new List(); + var toAdd = new List(); + var toRemove = new List(); var toApplyOverrun = false; @@ -79,7 +79,7 @@ namespace Selector return (toAdd, toRemove); } - public static void MatchData(Scrobble currentExisting, Scrobble toApply) + public static void MatchData(IListen currentExisting, IListen toApply) { if (!currentExisting.TrackName.Equals(toApply.TrackName, StringComparison.InvariantCultureIgnoreCase)) { @@ -97,19 +97,19 @@ namespace Selector } } - public static (IEnumerable, IEnumerable) IdentifyDiffsContains(IEnumerable existing, IEnumerable toApply) + public static (IEnumerable, IEnumerable) IdentifyDiffsContains(IEnumerable existing, IEnumerable toApply) { - var toAdd = toApply.Where(s => !existing.Contains(s, new ScrobbleComp())); - var toRemove = existing.Where(s => !toApply.Contains(s, new ScrobbleComp())); + var toAdd = toApply.Where(s => !existing.Contains(s, new ListenComp())); + var toRemove = existing.Where(s => !toApply.Contains(s, new ListenComp())); return (toAdd, toRemove); } - public class ScrobbleComp : IEqualityComparer + public class ListenComp : IEqualityComparer { - public bool Equals(Scrobble x, Scrobble y) => x.Timestamp == y.Timestamp; + public bool Equals(IListen x, IListen y) => x.Timestamp == y.Timestamp; - public int GetHashCode([DisallowNull] Scrobble obj) => obj.Timestamp.GetHashCode(); + public int GetHashCode([DisallowNull] IListen obj) => obj.Timestamp.GetHashCode(); } } } -- 2.45.2 From 26b4c3fa6404e9966b85130ce292570a4e1c1e7c Mon Sep 17 00:00:00 2001 From: Andy Pack Date: Sat, 8 Oct 2022 17:07:50 +0100 Subject: [PATCH 3/3] using metarepo --- Selector.CLI/Extensions/ServiceExtensions.cs | 4 ++-- Selector.Data/HistoryPersister.cs | 12 +++++----- Selector.Model/Listen/MetaListenRepository.cs | 24 ++++++++----------- Selector.Web/Startup.cs | 4 ++-- 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/Selector.CLI/Extensions/ServiceExtensions.cs b/Selector.CLI/Extensions/ServiceExtensions.cs index 0f155c3..5273460 100644 --- a/Selector.CLI/Extensions/ServiceExtensions.cs +++ b/Selector.CLI/Extensions/ServiceExtensions.cs @@ -121,8 +121,8 @@ namespace Selector.CLI.Extensions services.AddTransient() .AddTransient(); - //services.AddTransient(); - services.AddTransient(); + services.AddTransient(); + //services.AddTransient(); services.AddTransient(); diff --git a/Selector.Data/HistoryPersister.cs b/Selector.Data/HistoryPersister.cs index f9e3cc5..e01d1cd 100644 --- a/Selector.Data/HistoryPersister.cs +++ b/Selector.Data/HistoryPersister.cs @@ -1,8 +1,6 @@ -using System; -using System.Text.Json; -using Selector.Model; +using System.Text.Json; using Microsoft.Extensions.Logging; -using System.Diagnostics.Metrics; +using Selector.Model; namespace Selector.Data; @@ -60,14 +58,16 @@ public class HistoryPersister { if (Config.InitialClear) { - Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.User.UserName == Config.Username)); + var latestTime = input.OrderBy(x => x.ts).Last().ts; + var time = DateTime.Parse(latestTime).ToUniversalTime(); + Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.User.UserName == Config.Username && x.Timestamp <= time)); } var user = Db.Users.Single(u => u.UserName == Config.Username); var counter = 0; - var filtered = input.Where(x => x.ms_played > 20000) + var filtered = input.Where(x => x.ms_played > 30000) .DistinctBy(x => (x.offline_timestamp, x.ts, x.spotify_track_uri)) .ToArray(); diff --git a/Selector.Model/Listen/MetaListenRepository.cs b/Selector.Model/Listen/MetaListenRepository.cs index fa997c7..97ed5c2 100644 --- a/Selector.Model/Listen/MetaListenRepository.cs +++ b/Selector.Model/Listen/MetaListenRepository.cs @@ -4,11 +4,6 @@ using System.Linq; namespace Selector.Model; -public enum PreferenceMode -{ - Greedy, LastFm, Spotify -} - public class MetaListenRepository: IListenRepository { private readonly IScrobbleRepository scrobbleRepository; @@ -27,10 +22,13 @@ public class MetaListenRepository: IListenRepository string albumName = null, string artistName = null, DateTime? from = null, - DateTime? to = null) - { - throw new NotImplementedException(); - } + DateTime? to = null) => GetAll(userId: userId, + username: username, + trackName: trackName, + albumName: albumName, + artistName: artistName, + from: from, + to:to).Count(); public IEnumerable GetAll( string includes = null, @@ -62,14 +60,12 @@ public class MetaListenRepository: IListenRepository albumName: albumName, artistName: artistName, from: from, - to: to) + to: scrobbles.FirstOrDefault()?.Timestamp) .OrderBy(x => x.Timestamp) .ToArray(); + - var scrobbleIter = scrobbles.GetEnumerator(); - var spotIter = spotListens.GetEnumerator(); - - return scrobbles; + return spotListens.Concat(scrobbles); } } diff --git a/Selector.Web/Startup.cs b/Selector.Web/Startup.cs index dd82986..aea55db 100644 --- a/Selector.Web/Startup.cs +++ b/Selector.Web/Startup.cs @@ -67,8 +67,8 @@ namespace Selector.Web services.AddTransient() .AddTransient(); - //services.AddTransient(); - services.AddTransient(); + services.AddTransient(); + //services.AddTransient(); services.AddIdentity() .AddEntityFrameworkStores() -- 2.45.2