From 2b8e0a273559c4ae8fd769d5272a870eb8487522 Mon Sep 17 00:00:00 2001 From: Andy Pack Date: Fri, 7 Oct 2022 18:29:33 +0100 Subject: [PATCH] skeleton of spotify history model and insert --- Selector.CLI/Command/SpotHistoryCommand.cs | 91 ++++++++++++ Selector.CLI/Extensions/ServiceExtensions.cs | 1 + Selector.CLI/Extensions/SpotifyExtensions.cs | 39 ++++++ Selector.CLI/Program.cs | 1 + Selector.CLI/Selector.CLI.csproj | 1 + Selector.Data/DataJsonContext.cs | 14 ++ Selector.Data/EndSong.cs | 27 ++++ Selector.Data/HistoryPersister.cs | 96 +++++++++++++ Selector.Data/Selector.Data.csproj | 16 +++ Selector.Model/ApplicationDbContext.cs | 14 +- .../Listen/ISpotifyListenRepository.cs | 21 +++ Selector.Model/Listen/SpotifyListen.cs | 14 ++ .../Listen/SpotifyListenRepository.cs | 129 ++++++++++++++++++ Selector.Model/Selector.Model.csproj | 4 + Selector.sln | 6 + Selector/Listen/IListen.cs | 13 ++ Selector/Listen/Listen.cs | 13 ++ Selector/Scrobble/Resampler.cs | 4 +- Selector/Scrobble/Scrobble.cs | 2 +- spotify-data.ipynb | 109 +++++++++++++++ 20 files changed, 609 insertions(+), 6 deletions(-) create mode 100644 Selector.CLI/Command/SpotHistoryCommand.cs create mode 100644 Selector.CLI/Extensions/SpotifyExtensions.cs create mode 100644 Selector.Data/DataJsonContext.cs create mode 100644 Selector.Data/EndSong.cs create mode 100644 Selector.Data/HistoryPersister.cs create mode 100644 Selector.Data/Selector.Data.csproj create mode 100644 Selector.Model/Listen/ISpotifyListenRepository.cs create mode 100644 Selector.Model/Listen/SpotifyListen.cs create mode 100644 Selector.Model/Listen/SpotifyListenRepository.cs create mode 100644 Selector/Listen/IListen.cs create mode 100644 Selector/Listen/Listen.cs create mode 100644 spotify-data.ipynb diff --git a/Selector.CLI/Command/SpotHistoryCommand.cs b/Selector.CLI/Command/SpotHistoryCommand.cs new file mode 100644 index 0000000..e9d5ea7 --- /dev/null +++ b/Selector.CLI/Command/SpotHistoryCommand.cs @@ -0,0 +1,91 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Selector.CLI.Extensions; +using Selector.Data; +using Selector.Model; +using System; +using System.IO; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Linq; +using System.Collections.Generic; + +namespace Selector.CLI +{ + public class SpotifyHistoryCommand : Command + { + public SpotifyHistoryCommand(string name, string description = null) : base(name, description) + { + var connectionString = new Option("--connection", "database to migrate"); + connectionString.AddAlias("-c"); + AddOption(connectionString); + + var pathString = new Option("--path", "path to find data"); + pathString.AddAlias("-i"); + AddOption(pathString); + + var username = new Option("--username", "user to pulls scrobbles for"); + username.AddAlias("-u"); + AddOption(username); + + Handler = CommandHandler.Create((string connectionString, string path, string username) => Execute(connectionString, path, username)); + } + + public static int Execute(string connectionString, string path, string username) + { + var streams = new List(); + + try + { + var context = new CommandContext().WithLogger().WithDb(connectionString).WithLastfmApi(); + var logger = context.Logger.CreateLogger("Scrobble"); + + using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger()); + + var historyPersister = new HistoryPersister(db, new DataJsonContext(), new() + { + Username = username + }, context.Logger.CreateLogger()); + + logger.LogInformation("Preparing to parse from {} for {}", path, username); + + var directoryContents = Directory.EnumerateFiles(path); + var endSongs = directoryContents.Where(f => f.Contains("endsong_")).ToArray(); + + + foreach(var file in endSongs) + { + streams.Add(File.OpenRead(file)); + } + + Console.WriteLine("Parse {0} historical data files? (y/n) ", endSongs.Length); + var input = Console.ReadLine(); + + if (input.Trim().Equals("y", StringComparison.OrdinalIgnoreCase)) + { + logger.LogInformation("Parsing files"); + + historyPersister.Process(streams).Wait(); + } + else + { + logger.LogInformation("Exiting"); + } + } + catch (Exception ex) + { + Console.WriteLine(ex); + return 1; + } + finally + { + foreach(var stream in streams) + { + stream.Dispose(); + } + } + + return 0; + } + } +} diff --git a/Selector.CLI/Extensions/ServiceExtensions.cs b/Selector.CLI/Extensions/ServiceExtensions.cs index 65f678c..1ac1ea8 100644 --- a/Selector.CLI/Extensions/ServiceExtensions.cs +++ b/Selector.CLI/Extensions/ServiceExtensions.cs @@ -119,6 +119,7 @@ namespace Selector.CLI.Extensions ); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddHostedService(); diff --git a/Selector.CLI/Extensions/SpotifyExtensions.cs b/Selector.CLI/Extensions/SpotifyExtensions.cs new file mode 100644 index 0000000..c775fa0 --- /dev/null +++ b/Selector.CLI/Extensions/SpotifyExtensions.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using SpotifyAPI.Web; + +namespace Selector.CLI.Extensions +{ + public static class SpotifyExtensions + { + public static async Task<(FullPlaylist, IEnumerable>)> GetPopulated(this ISpotifyClient client, string playlistId, ILogger logger = null) + { + try + { + var playlist = await client.Playlists.Get(playlistId); + var items = await client.Paginate(playlist.Tracks).ToListAsync(); + + return (playlist, items); + } + catch (APIUnauthorizedException e) + { + logger?.LogDebug("Unauthorised error: [{message}] (should be refreshed and retried?)", e.Message); + throw e; + } + catch (APITooManyRequestsException e) + { + logger?.LogDebug("Too many requests error: [{message}]", e.Message); + throw e; + } + catch (APIException e) + { + logger?.LogDebug("API error: [{message}]", e.Message); + throw e; + } + } + } +} + diff --git a/Selector.CLI/Program.cs b/Selector.CLI/Program.cs index fb0cc5c..e8dba6b 100644 --- a/Selector.CLI/Program.cs +++ b/Selector.CLI/Program.cs @@ -9,6 +9,7 @@ namespace Selector.CLI var cmd = new HostRootCommand(); cmd.AddCommand(new ScrobbleCommand("scrobble", "Manipulate scrobbles")); cmd.AddCommand(new MigrateCommand("migrate", "Migrate database")); + cmd.AddCommand(new SpotifyHistoryCommand("history", "Insert Spotify history")); cmd.Invoke(args); } diff --git a/Selector.CLI/Selector.CLI.csproj b/Selector.CLI/Selector.CLI.csproj index 3f5963a..21977e6 100644 --- a/Selector.CLI/Selector.CLI.csproj +++ b/Selector.CLI/Selector.CLI.csproj @@ -31,6 +31,7 @@ + diff --git a/Selector.Data/DataJsonContext.cs b/Selector.Data/DataJsonContext.cs new file mode 100644 index 0000000..141fc79 --- /dev/null +++ b/Selector.Data/DataJsonContext.cs @@ -0,0 +1,14 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace Selector.Data; + +[JsonSerializable(typeof(EndSong))] +[JsonSerializable(typeof(EndSong[]))] +public partial class DataJsonContext : JsonSerializerContext +{ + +} + diff --git a/Selector.Data/EndSong.cs b/Selector.Data/EndSong.cs new file mode 100644 index 0000000..1278401 --- /dev/null +++ b/Selector.Data/EndSong.cs @@ -0,0 +1,27 @@ +namespace Selector.Data; + +public record struct EndSong +{ + public string conn_country { get; set; } + public string episode_name { get; set; } + public string episode_show_name { get; set; } + public bool? incognito_mode { get; set; } + public string ip_addr_decrypted { get; set; } + public string master_metadata_album_album_name { get; set; } + public string master_metadata_album_artist_name { get; set; } + public string master_metadata_track_name { get; set; } + public int ms_played { get; set; } + public bool? offline { get; set; } + public long? offline_timestamp { get; set; } + public string platform { get; set; } + public string reason_end { get; set; } + public string reason_start { get; set; } + public bool shuffle { get; set; } + public bool? skipped { get; set; } + public string spotify_episode_uri { get; set; } + public string spotify_track_uri { get; set; } + public string ts { get; set; } + public string user_agent_decrypted { get; set; } + public string username { get; set; } +} + diff --git a/Selector.Data/HistoryPersister.cs b/Selector.Data/HistoryPersister.cs new file mode 100644 index 0000000..c252140 --- /dev/null +++ b/Selector.Data/HistoryPersister.cs @@ -0,0 +1,96 @@ +using System; +using System.Text.Json; +using Selector.Model; +using Microsoft.Extensions.Logging; +using System.Diagnostics.Metrics; + +namespace Selector.Data; + +public class HistoryPersisterConfig +{ + public string Username { get; set; } + public bool InitialClear { get; set; } = true; + public bool Apply50PercentRule { get; set; } = false; +} + +public class HistoryPersister +{ + private HistoryPersisterConfig Config { get; set; } + private ApplicationDbContext Db { get; set; } + private DataJsonContext Json { get; set; } + + private ILogger Logger { get; set; } + + public HistoryPersister(ApplicationDbContext db, DataJsonContext json, HistoryPersisterConfig config, ILogger logger = null) + { + Config = config; + Db = db; + Json = json; + Logger = logger; + } + + public void Process(string input) + { + var parsed = JsonSerializer.Deserialize(input, Json.EndSongArray); + Process(parsed).Wait(); + } + + public async Task Process(Stream input) + { + var parsed = await JsonSerializer.DeserializeAsync(input, Json.EndSongArray); + await Process(parsed); + } + + public async Task Process(IEnumerable input) + { + var songs = Enumerable.Empty(); + + foreach(var singleInput in input) + { + var parsed = await JsonSerializer.DeserializeAsync(singleInput, Json.EndSongArray); + songs = songs.Concat(parsed); + + Logger?.LogDebug("Parsed {} items for {}", parsed.Length, Config.Username); + } + + await Process(songs); + } + + public async Task Process(IEnumerable input) + { + if (Config.InitialClear) + { + Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.User.UserName == Config.Username)); + } + + var user = Db.Users.Single(u => u.UserName == Config.Username); + + var counter = 0; + + foreach(var item in input) + { + if(!string.IsNullOrWhiteSpace(item.master_metadata_track_name)) + { + Db.SpotifyListen.Add(new() + { + TrackName = item.master_metadata_track_name, + AlbumName = item.master_metadata_album_album_name, + ArtistName = item.master_metadata_album_artist_name, + + Timestamp = DateTime.Parse(item.ts).ToUniversalTime(), + PlayedDuration = item.ms_played, + + TrackUri = item.spotify_track_uri, + UserId = user.Id + }); + + counter++; + } + } + + Logger?.LogInformation("Added {} historical items for {}", counter, user.UserName); + + await Db.SaveChangesAsync(); + } +} + diff --git a/Selector.Data/Selector.Data.csproj b/Selector.Data/Selector.Data.csproj new file mode 100644 index 0000000..661434f --- /dev/null +++ b/Selector.Data/Selector.Data.csproj @@ -0,0 +1,16 @@ + + + + net6.0 + enable + + + + + + + + + + + diff --git a/Selector.Model/ApplicationDbContext.cs b/Selector.Model/ApplicationDbContext.cs index 7eba707..c656451 100644 --- a/Selector.Model/ApplicationDbContext.cs +++ b/Selector.Model/ApplicationDbContext.cs @@ -24,6 +24,8 @@ namespace Selector.Model public DbSet AlbumMapping { get; set; } public DbSet ArtistMapping { get; set; } + public DbSet SpotifyListen { get; set; } + public ApplicationDbContext( DbContextOptions options, ILogger logger @@ -85,9 +87,15 @@ namespace Selector.Model .Property(s => s.LastfmArtistName) .UseCollation("case_insensitive"); - modelBuilder.Entity().HasKey(s => s.SpotifyUri); - modelBuilder.Entity() - .Property(s => s.LastfmArtistName) + modelBuilder.Entity().HasKey(s => s.Timestamp); + modelBuilder.Entity() + .Property(s => s.TrackName) + .UseCollation("case_insensitive"); + modelBuilder.Entity() + .Property(s => s.AlbumName) + .UseCollation("case_insensitive"); + modelBuilder.Entity() + .Property(s => s.ArtistName) .UseCollation("case_insensitive"); SeedData.Seed(modelBuilder); diff --git a/Selector.Model/Listen/ISpotifyListenRepository.cs b/Selector.Model/Listen/ISpotifyListenRepository.cs new file mode 100644 index 0000000..7e5e5c5 --- /dev/null +++ b/Selector.Model/Listen/ISpotifyListenRepository.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Selector.Model +{ + public interface ISpotifyListenRepository + { + void Add(SpotifyListen item); + void AddRange(IEnumerable item); + IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + SpotifyListen Find(DateTime key, string include = null); + void Remove(DateTime key); + public void Remove(SpotifyListen scrobble); + public void RemoveRange(IEnumerable scrobbles); + void Update(SpotifyListen item); + Task Save(); + int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + } +} + diff --git a/Selector.Model/Listen/SpotifyListen.cs b/Selector.Model/Listen/SpotifyListen.cs new file mode 100644 index 0000000..d3f8570 --- /dev/null +++ b/Selector.Model/Listen/SpotifyListen.cs @@ -0,0 +1,14 @@ +using System; + +namespace Selector.Model; + +public class SpotifyListen: Listen +{ + public int? PlayedDuration { get; set; } + + public string TrackUri { get; set; } + + public string UserId { get; set; } + public ApplicationUser User { get; set; } +} + diff --git a/Selector.Model/Listen/SpotifyListenRepository.cs b/Selector.Model/Listen/SpotifyListenRepository.cs new file mode 100644 index 0000000..eefabfd --- /dev/null +++ b/Selector.Model/Listen/SpotifyListenRepository.cs @@ -0,0 +1,129 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Selector.Model +{ + public class SpotifyListenRepository : ISpotifyListenRepository + { + private readonly ApplicationDbContext db; + + public SpotifyListenRepository(ApplicationDbContext context) + { + db = context; + } + + public void Add(SpotifyListen item) + { + db.SpotifyListen.Add(item); + } + + public void AddRange(IEnumerable item) + { + db.SpotifyListen.AddRange(item); + } + + public SpotifyListen Find(DateTime key, string include = null) + { + var listens = db.SpotifyListen.Where(s => s.Timestamp == key); + + if (!string.IsNullOrWhiteSpace(include)) + { + listens = listens.Include(include); + } + + return listens.FirstOrDefault(); + } + + private IQueryable GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + { + var listens = db.SpotifyListen.AsQueryable(); + + if (!string.IsNullOrWhiteSpace(include)) + { + listens = listens.Include(include); + } + + if (!string.IsNullOrWhiteSpace(userId)) + { + listens = listens.Where(s => s.UserId == userId); + } + + if (!string.IsNullOrWhiteSpace(username)) + { + var normalUsername = username.ToUpperInvariant(); + var user = db.Users.AsNoTracking().Where(u => u.NormalizedUserName == normalUsername).FirstOrDefault(); + if (user is not null) + { + listens = listens.Where(s => s.UserId == user.Id); + } + else + { + listens = Enumerable.Empty().AsQueryable(); + } + } + + if (!string.IsNullOrWhiteSpace(trackName)) + { + listens = listens.Where(s => s.TrackName == trackName); + } + + if (!string.IsNullOrWhiteSpace(albumName)) + { + listens = listens.Where(s => s.AlbumName == albumName); + } + + if (!string.IsNullOrWhiteSpace(artistName)) + { + listens = listens.Where(s => s.ArtistName == artistName); + } + + if (from is not null) + { + listens = listens.Where(u => u.Timestamp >= from.Value); + } + + if (to is not null) + { + listens = listens.Where(u => u.Timestamp < to.Value); + } + + return listens; + } + + public IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable(); + + public void Remove(DateTime key) + { + Remove(Find(key)); + } + + public void Remove(SpotifyListen scrobble) + { + db.SpotifyListen.Remove(scrobble); + } + + public void RemoveRange(IEnumerable scrobbles) + { + db.SpotifyListen.RemoveRange(scrobbles); + } + + public void Update(SpotifyListen item) + { + db.SpotifyListen.Update(item); + } + + public Task Save() + { + return db.SaveChangesAsync(); + } + + public int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count(); + } +} diff --git a/Selector.Model/Selector.Model.csproj b/Selector.Model/Selector.Model.csproj index 10b162f..c6de22c 100644 --- a/Selector.Model/Selector.Model.csproj +++ b/Selector.Model/Selector.Model.csproj @@ -30,6 +30,10 @@ + + + + diff --git a/Selector.sln b/Selector.sln index d79642b..f7e9a73 100644 --- a/Selector.sln +++ b/Selector.sln @@ -24,6 +24,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Selector.Cache", "Selector. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Selector.Event", "Selector.Event\Selector.Event.csproj", "{C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Selector.Data", "Selector.Data\Selector.Data.csproj", "{CB62ACCB-94F1-4B78-A195-8B108B9E800D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -58,6 +60,10 @@ Global {C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Release|Any CPU.Build.0 = Release|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Selector/Listen/IListen.cs b/Selector/Listen/IListen.cs new file mode 100644 index 0000000..a43b222 --- /dev/null +++ b/Selector/Listen/IListen.cs @@ -0,0 +1,13 @@ +using System; + +namespace Selector; + +public interface IListen +{ + DateTime Timestamp { get; set; } + + string TrackName { get; set; } + string AlbumName { get; set; } + string ArtistName { get; set; } +} + diff --git a/Selector/Listen/Listen.cs b/Selector/Listen/Listen.cs new file mode 100644 index 0000000..c7a0d2c --- /dev/null +++ b/Selector/Listen/Listen.cs @@ -0,0 +1,13 @@ +using System; + +namespace Selector; + +public class Listen: IListen +{ + public string TrackName { get; set; } + public string AlbumName { get; set; } + public string ArtistName { get; set; } + + public DateTime Timestamp { get; set; } +} + diff --git a/Selector/Scrobble/Resampler.cs b/Selector/Scrobble/Resampler.cs index 254639b..7289d15 100644 --- a/Selector/Scrobble/Resampler.cs +++ b/Selector/Scrobble/Resampler.cs @@ -11,7 +11,7 @@ namespace Selector public static class Resampler { - public static IEnumerable Resample(this IEnumerable scrobbles, TimeSpan window) + public static IEnumerable Resample(this IEnumerable scrobbles, TimeSpan window) { var sortedScrobbles = scrobbles.OrderBy(s => s.Timestamp).ToList(); @@ -68,7 +68,7 @@ namespace Selector } } - public static IEnumerable ResampleByMonth(this IEnumerable scrobbles) + public static IEnumerable ResampleByMonth(this IEnumerable scrobbles) { var sortedScrobbles = scrobbles.OrderBy(s => s.Timestamp).ToList(); diff --git a/Selector/Scrobble/Scrobble.cs b/Selector/Scrobble/Scrobble.cs index 918cf05..fa4542b 100644 --- a/Selector/Scrobble/Scrobble.cs +++ b/Selector/Scrobble/Scrobble.cs @@ -2,7 +2,7 @@ namespace Selector { - public class Scrobble + public class Scrobble: IListen { public DateTime Timestamp { get; set; } public string TrackName { get; set; } diff --git a/spotify-data.ipynb b/spotify-data.ipynb new file mode 100644 index 0000000..2309439 --- /dev/null +++ b/spotify-data.ipynb @@ -0,0 +1,109 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from datetime import datetime\n", + "import random\n", + "from pprint import pprint" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total: 445582\n", + "Total Populated: 442034 (99.20%)\n", + "Total New: 130411 (29.27%)\n", + "{'conn_country': 'GB',\n", + " 'episode_name': None,\n", + " 'episode_show_name': None,\n", + " 'incognito_mode': False,\n", + " 'ip_addr_decrypted': '82.132.244.90',\n", + " 'master_metadata_album_album_name': 'Tales Told By Dead Friends',\n", + " 'master_metadata_album_artist_name': 'Mayday Parade',\n", + " 'master_metadata_track_name': 'Three Cheers For Five Years',\n", + " 'ms_played': 33274,\n", + " 'offline': False,\n", + " 'offline_timestamp': 0,\n", + " 'platform': 'iOS 9.2.1 (iPhone7,1)',\n", + " 'reason_end': 'fwdbtn',\n", + " 'reason_start': 'fwdbtn',\n", + " 'shuffle': True,\n", + " 'skipped': True,\n", + " 'spotify_episode_uri': None,\n", + " 'spotify_track_uri': 'spotify:track:1aw8gphDUzqalHEi9Z8M38',\n", + " 'ts': '2016-02-11T08:13:55Z',\n", + " 'user_agent_decrypted': 'unknown',\n", + " 'username': 'sarsoo'}\n" + ] + } + ], + "source": [ + "data = []\n", + "data_with_names = []\n", + "new_data = []\n", + "\n", + "folder = '/Users/andy/lab/backups/spotify-2022-03-07'\n", + "\n", + "for i in os.listdir(folder):\n", + " if i.startswith('endsong_'):\n", + " with open(f'{folder}/{i}') as f:\n", + " data += json.loads(f.read())\n", + "\n", + "data.sort(key = lambda a: a['ts'])\n", + "data_with_names = [i for i in data if i['master_metadata_track_name'] is not None]\n", + "new_data = [i for i in data_with_names if datetime.fromisoformat(i['ts'].split('T')[0]) < datetime(2017, 11, 3)]\n", + "\n", + "print(f'Total: {len(data)}')\n", + "print(f'Total Populated: {len(data_with_names)} ({len(data_with_names)/len(data)*100:.2f}%)')\n", + "print(f'Total New: {len(new_data)} ({len(new_data)/len(data)*100:.2f}%)')\n", + "\n", + "pprint(random.choice(new_data))\n", + "# print(min(i[0] for i in data))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "a0a5145e6c304e2a9afaf5b930a2955b950bd4b81fe94f7c42930f43f42762eb" + }, + "kernelspec": { + "display_name": "Python 3.10.7 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}