sarsoo/Mixonomer
1
0
Fork 0
Code Issues 15 Pull Requests Actions Packages Projects Releases Wiki Activity

begin testing tasks, pass objects to playlist and tag refreshing

Browse Source
This commit is contained in:
andy 2021-02-08 16:18:16 +00:00
parent 14c28ca21b
commit d8a52437fd
11 changed files with 257 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/ci.yml vendored
View File

@ -1,5 +1,13 @@
name: test and deploy
on: [push, pull_request]
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:

2
main.run_playlist.py
View File

@ -7,7 +7,7 @@ def run_user_playlist(event, context):
if 'username' in event['attributes'] and 'name' in event['attributes']:
from music.tasks.run_user_playlist import run_user_playlist as do_run_user_playlist
do_run_user_playlist(username=event['attributes']['username'], playlist_name=event['attributes']["name"])
do_run_user_playlist(user=event['attributes']['username'], playlist=event['attributes']["name"])
else:
logger.error('no parameters in event attributes')

2
main.update_tag.py
View File

@ -7,7 +7,7 @@ def update_tag(event, context):
if 'username' in event['attributes'] and 'tag_id' in event['attributes']:
from music.tasks.update_tag import update_tag as do_update_tag
do_update_tag(username=event['attributes']['username'], tag_id=event['attributes']["tag_id"])
do_update_tag(user=event['attributes']['username'], tag=event['attributes']["tag_id"])
else:
logger.error('no parameters in event attributes')

58
music/__init__.py
View File

@ -8,31 +8,35 @@ spotframework_logger = logging.getLogger('spotframework')
fmframework_logger = logging.getLogger('fmframework')
spotfm_logger = logging.getLogger('spotfm')
def init_log(cloud=False, console=False):
if cloud:
import google.cloud.logging
from google.cloud.logging.handlers import CloudLoggingHandler
log_format = '%(funcName)s - %(message)s'
formatter = logging.Formatter(log_format)
client = google.cloud.logging.Client()
handler = CloudLoggingHandler(client, name="music-tools")
handler.setFormatter(formatter)
logger.addHandler(handler)
spotframework_logger.addHandler(handler)
fmframework_logger.addHandler(handler)
spotfm_logger.addHandler(handler)
if console:
log_format = '%(levelname)s %(name)s:%(funcName)s - %(message)s'
formatter = logging.Formatter(log_format)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
spotframework_logger.addHandler(stream_handler)
fmframework_logger.addHandler(stream_handler)
spotfm_logger.addHandler(stream_handler)
if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
import google.cloud.logging
from google.cloud.logging.handlers import CloudLoggingHandler
log_format = '%(funcName)s - %(message)s'
formatter = logging.Formatter(log_format)
client = google.cloud.logging.Client()
handler = CloudLoggingHandler(client, name="music-tools")
handler.setFormatter(formatter)
logger.addHandler(handler)
spotframework_logger.addHandler(handler)
fmframework_logger.addHandler(handler)
spotfm_logger.addHandler(handler)
else:
log_format = '%(levelname)s %(name)s:%(funcName)s - %(message)s'
formatter = logging.Formatter(log_format)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
spotframework_logger.addHandler(stream_handler)
fmframework_logger.addHandler(stream_handler)
spotfm_logger.addHandler(stream_handler)
init_log(cloud=True)

2
music/api/api.py
View File

@ -234,7 +234,7 @@ def run_playlist(user=None):
if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
queue_run_user_playlist(user.username, request.args['name']) # pass to either cloud tasks or functions
else:
run_user_playlist(user.username, request.args['name']) # update synchronously
run_user_playlist(user, request.args['name']) # update synchronously
return jsonify({'message': 'execution requested', 'status': 'success'}), 200

2
music/api/tag.py
View File

@ -133,7 +133,7 @@ def tag_refresh(tag_id, user=None):
if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
serverless_update_tag(username=user.username, tag_id=tag_id)
else:
update_tag(username=user.username, tag_id=tag_id)
update_tag(user=user, tag=tag_id)
return jsonify({"message": 'tag updated', "status": "success"}), 200

2
music/cloud/__init__.py
View File

@ -37,7 +37,7 @@ def offload_or_run_user_playlist(username: str, playlist_name: str):
run_user_playlist_function(username=username, playlist_name=playlist_name)
if config.playlist_cloud_operating_mode == 'task':
run_now(username=username, playlist_name=playlist_name)
run_now(user=username, playlist=playlist_name)
elif config.playlist_cloud_operating_mode == 'function':
logger.debug(f'offloading {username} / {playlist_name} to cloud function')

2
music/cloud/tasks.py
View File

@ -68,7 +68,7 @@ def update_playlists(username):
if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
run_user_playlist_task(username, iterate_playlist.name, seconds_delay)
else:
run_user_playlist(username, iterate_playlist.name)
run_user_playlist(user, iterate_playlist)
seconds_delay += 6

59
music/tasks/run_user_playlist.py
View File

@ -21,40 +21,51 @@ from music.model.playlist import Playlist
logger = logging.getLogger(__name__)
def run_user_playlist(username, playlist_name):
def run_user_playlist(user, playlist, spotnet=None, fmnet=None):
"""Generate and upadate a user's playlist"""
user = User.collection.filter('username', '==', username.strip().lower()).get()
# PRE-RUN CHECKS
if isinstance(user, str):
username = user
user = User.collection.filter('username', '==', username.strip().lower()).get()
else:
username = user.username
if user is None:
logger.error(f'user not found {username} / {playlist_name}')
return
logger.info(f'running {username} / {playlist_name}')
playlist = Playlist.collection.parent(user.key).filter('name', '==', playlist_name).get()
logger.error(f'user {username} not found')
raise NameError(f'User {username} not found')
if isinstance(playlist, str):
playlist_name = playlist
playlist = Playlist.collection.parent(user.key).filter('name', '==', playlist_name).get()
else:
playlist_name = playlist.name
if playlist is None:
logger.critical(f'playlist not found {username} / {playlist_name}')
return
raise NameError(f'Playlist {playlist_name} not found for {username}')
if playlist.uri is None:
logger.critical(f'no playlist id to populate {username} / {playlist_name}')
return
raise AttributeError(f'No URI for {playlist_name} ({username})')
# END CHECKS
net = database.get_authed_spotify_network(user)
logger.info(f'running {username} / {playlist_name}')
if net is None:
if spotnet is None:
spotnet = database.get_authed_spotify_network(user)
if spotnet is None:
logger.error(f'no spotify network returned for {username} / {playlist_name}')
return
raise NameError(f'No Spotify network returned ({username} / {playlist_name})')
try:
user_playlists = net.playlists()
except SpotifyNetworkException:
user_playlists = spotnet.playlists()
except SpotifyNetworkException as e:
logger.exception(f'error occured while retrieving playlists {username} / {playlist_name}')
return
raise e
part_generator = PartGenerator(user=user)
part_names = part_generator.get_recursive_parts(playlist.name)
@ -82,7 +93,7 @@ def run_user_playlist(username, playlist_name):
log_name = part_playlist.name
try:
_tracks = net.playlist_tracks(uri=uri)
_tracks = spotnet.playlist_tracks(uri=uri)
if _tracks and len(_tracks) > 0:
playlist_tracks += _tracks
else:
@ -95,7 +106,7 @@ def run_user_playlist(username, playlist_name):
# LIBRARY
if playlist.include_library_tracks:
try:
library_tracks = net.saved_tracks()
library_tracks = spotnet.saved_tracks()
if library_tracks and len(library_tracks) > 0:
playlist_tracks += library_tracks
else:
@ -118,10 +129,12 @@ def run_user_playlist(username, playlist_name):
except KeyError:
logger.error(f'invalid last.fm chart range found {playlist.chart_range}, '
f'defaulting to 1 month {username} / {playlist_name}')
if fmnet is None:
fmnet = database.get_authed_lastfm_network(user)
fmnet = database.get_authed_lastfm_network(user)
if fmnet is not None:
chart_tracks = map_lastfm_track_chart_to_spotify(spotnet=net,
chart_tracks = map_lastfm_track_chart_to_spotify(spotnet=spotnet,
fmnet=fmnet,
period=chart_range,
limit=playlist.chart_limit)
@ -142,7 +155,7 @@ def run_user_playlist(username, playlist_name):
# RECOMMENDATIONS
if playlist.include_recommendations:
try:
recommendations = net.recommendations(tracks=[i.uri.object_id for i, j
recommendations = spotnet.recommendations(tracks=[i.uri.object_id for i, j
in get_track_objects(
random.sample(playlist_tracks,
k=min(5, len(playlist_tracks))
@ -162,7 +175,7 @@ def run_user_playlist(username, playlist_name):
# EXECUTE
try:
net.replace_playlist_tracks(uri=playlist.uri, uris=[i.uri for i, j in get_track_objects(playlist_tracks)])
spotnet.replace_playlist_tracks(uri=playlist.uri, uris=[i.uri for i, j in get_track_objects(playlist_tracks)])
if playlist.description_overwrite:
string = playlist.description_overwrite
@ -177,7 +190,7 @@ def run_user_playlist(username, playlist_name):
return None
try:
net.change_playlist_details(uri=playlist.uri, description=string)
spotnet.change_playlist_details(uri=playlist.uri, description=string)
except SpotifyNetworkException:
logger.exception(f'error changing description for {username} / {playlist_name}')

55
music/tasks/update_tag.py
View File

@ -12,32 +12,49 @@ from spotfm.timer import time, seconds_to_time_str
logger = logging.getLogger(__name__)
def update_tag(username, tag_id):
logger.info(f'updating {username} / {tag_id}')
def update_tag(user, tag, spotnet=None, fmnet=None):
# PRE-RUN CHECKS
if isinstance(user, str):
username = user
user = User.collection.filter('username', '==', username.strip().lower()).get()
else:
username = user.username
user = User.collection.filter('username', '==', username.strip().lower()).get()
if user is None:
logger.error(f'user {username} not found')
return
tag = Tag.collection.parent(user.key).filter('tag_id', '==', tag_id).get()
raise NameError(f'User {username} not found')
if isinstance(tag, str):
tag_id = tag
tag = Tag.collection.parent(user.key).filter('tag_id', '==', tag_id).get()
else:
tag_id = tag.tag_id
if tag is None:
logger.error(f'{tag_id} for {username} not found')
return
raise NameError(f'Tag {tag_id} not found for {username}')
if user.lastfm_username is None or len(user.lastfm_username) == 0:
logger.error(f'{username} has no last.fm username')
return
raise AttributeError(f'{username} has no Last.fm username ({tag_id})')
net = database.get_authed_lastfm_network(user)
if net is None:
# END CHECKS
logger.info(f'updating {username} / {tag_id}')
if fmnet is None:
fmnet = database.get_authed_lastfm_network(user)
if fmnet is None:
logger.error(f'no last.fm network returned for {username} / {tag_id}')
return
raise NameError(f'No Last.fm network returned ({username} / {tag_id})')
if tag.time_objects:
if user.spotify_linked:
spotnet = database.get_authed_spotify_network(user)
if spotnet is None:
spotnet = database.get_authed_spotify_network(user)
else:
logger.warning(f'timing objects requested but no spotify linked {username} / {tag_id}')
@ -45,7 +62,7 @@ def update_tag(username, tag_id):
tag.total_time_ms = 0
try:
user_scrobbles = net.user_scrobble_count()
user_scrobbles = fmnet.user_scrobble_count()
except LastFMNetworkException:
logger.exception(f'error retrieving scrobble count {username} / {tag_id}')
user_scrobbles = 1
@ -54,7 +71,7 @@ def update_tag(username, tag_id):
for artist in tag.artists:
try:
if tag.time_objects and user.spotify_linked:
total_ms, timed_tracks = time(spotnet=spotnet, fmnet=net,
total_ms, timed_tracks = time(spotnet=spotnet, fmnet=fmnet,
artist=artist['name'], username=user.lastfm_username,
return_tracks=True)
scrobbles = sum(i[0].user_scrobbles for i in timed_tracks)
@ -64,7 +81,7 @@ def update_tag(username, tag_id):
tag.total_time_ms += total_ms
else:
net_artist = net.artist(name=artist['name'])
net_artist = fmnet.artist(name=artist['name'])
if net_artist is not None:
scrobbles = net_artist.user_scrobbles
@ -82,7 +99,7 @@ def update_tag(username, tag_id):
for album in tag.albums:
try:
if tag.time_objects and user.spotify_linked:
total_ms, timed_tracks = time(spotnet=spotnet, fmnet=net,
total_ms, timed_tracks = time(spotnet=spotnet, fmnet=fmnet,
album=album['name'], artist=album['artist'],
username=user.lastfm_username, return_tracks=True)
scrobbles = sum(i[0].user_scrobbles for i in timed_tracks)
@ -92,7 +109,7 @@ def update_tag(username, tag_id):
tag.total_time_ms += total_ms
else:
net_album = net.album(name=album['name'], artist=album['artist'])
net_album = fmnet.album(name=album['name'], artist=album['artist'])
if net_album is not None:
scrobbles = net_album.user_scrobbles
@ -112,7 +129,7 @@ def update_tag(username, tag_id):
for track in tag.tracks:
try:
if tag.time_objects and user.spotify_linked:
total_ms, timed_tracks = time(spotnet=spotnet, fmnet=net,
total_ms, timed_tracks = time(spotnet=spotnet, fmnet=fmnet,
track=track['name'], artist=track['artist'],
username=user.lastfm_username, return_tracks=True)
scrobbles = sum(i[0].user_scrobbles for i in timed_tracks)
@ -122,7 +139,7 @@ def update_tag(username, tag_id):
tag.total_time_ms += total_ms
else:
net_track = net.track(name=track['name'], artist=track['artist'])
net_track = fmnet.track(name=track['name'], artist=track['artist'])
if net_track is not None:
scrobbles = net_track.user_scrobbles

139
tests/test_tasks.py Normal file
View File

@ -0,0 +1,139 @@
import unittest
from unittest.mock import Mock
from music.tasks.run_user_playlist import run_user_playlist
from music.tasks.update_tag import update_tag
class TestRunPlaylist(unittest.TestCase):
def test_run_unknown_name(self):
with self.assertRaises(NameError):
run_user_playlist(user='unknown_name', playlist='test_playlist')
def test_run_unknown_playlist(self):
with self.assertRaises(NameError):
run_user_playlist(user='test', playlist='test')
def test_run_no_uri(self):
with self.assertRaises(AttributeError):
run_user_playlist(user='test', playlist='test_playlist')
def test_run_no_network(self):
with self.assertRaises(NameError):
run_user_playlist(user='test', playlist='test_uri')
class TestRunTag(unittest.TestCase):
def test_run_unknown_name(self):
with self.assertRaises(NameError):
update_tag(user='unknown_name', tag='test_tag')
def test_run_unknown_tag(self):
with self.assertRaises(NameError):
update_tag(user='test', tag='unknown_tag')
def test_run_no_service_username(self):
with self.assertRaises(AttributeError):
update_tag(user='test', tag='test_tag')
def test_mocked_without_components(self):
spotnet = Mock()
fmnet = Mock()
fmnet.user_scrobble_count.return_value = 10
user_mock = Mock()
user_mock.lastfm_username = 'test_username'
tag_mock = Mock()
tag_mock.time_objects = True
tag_mock.artists = []
tag_mock.albums = []
tag_mock.tracks = []
update_tag(user=user_mock, tag=tag_mock, spotnet=spotnet, fmnet=fmnet)
tag_mock.update.assert_called_once()
def test_mocked_artists(self):
spotnet = Mock()
fmnet = Mock()
fmnet.user_scrobble_count.return_value = 10
artist_mock = Mock()
artist_mock.user_scrobbles = 10
fmnet.artist.return_value = artist_mock
user_mock = Mock()
user_mock.lastfm_username = 'test_username'
dict_mock = {'name': 'test_name'}
tag_mock = Mock()
tag_mock.time_objects = False
tag_mock.artists = [dict_mock, dict_mock, dict_mock]
tag_mock.albums = []
tag_mock.tracks = []
update_tag(user=user_mock, tag=tag_mock, spotnet=spotnet, fmnet=fmnet)
tag_mock.update.assert_called_once()
self.assertEqual(tag_mock.count, 30)
self.assertEqual(tag_mock.proportion, 300)
self.assertEqual(len(tag_mock.artists), 3)
self.assertEqual(dict_mock['count'], 10)
def test_mocked_albums(self):
spotnet = Mock()
fmnet = Mock()
fmnet.user_scrobble_count.return_value = 10
album_mock = Mock()
album_mock.user_scrobbles = 10
fmnet.album.return_value = album_mock
user_mock = Mock()
user_mock.lastfm_username = 'test_username'
dict_mock = {'name': 'test_name', 'artist': 'test_artist'}
tag_mock = Mock()
tag_mock.time_objects = False
tag_mock.artists = []
tag_mock.albums = [dict_mock, dict_mock, dict_mock]
tag_mock.tracks = []
update_tag(user=user_mock, tag=tag_mock, spotnet=spotnet, fmnet=fmnet)
tag_mock.update.assert_called_once()
self.assertEqual(tag_mock.count, 30)
self.assertEqual(tag_mock.proportion, 300)
self.assertEqual(len(tag_mock.albums), 3)
self.assertEqual(dict_mock['count'], 10)
def test_mocked_tracks(self):
spotnet = Mock()
fmnet = Mock()
fmnet.user_scrobble_count.return_value = 10
track_mock = Mock()
track_mock.user_scrobbles = 10
fmnet.track.return_value = track_mock
user_mock = Mock()
user_mock.lastfm_username = 'test_username'
dict_mock = {'name': 'test_name', 'artist': 'test_artist'}
tag_mock = Mock()
tag_mock.time_objects = False
tag_mock.artists = []
tag_mock.albums = []
tag_mock.tracks = [dict_mock, dict_mock, dict_mock]
update_tag(user=user_mock, tag=tag_mock, spotnet=spotnet, fmnet=fmnet)
tag_mock.update.assert_called_once()
self.assertEqual(tag_mock.count, 30)
self.assertEqual(tag_mock.proportion, 300)
self.assertEqual(len(tag_mock.tracks), 3)
self.assertEqual(dict_mock['count'], 10)