diff --git a/admin b/admin new file mode 100755 index 0000000..d3b3c71 --- /dev/null +++ b/admin @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +import shutil +import os +import sys +from cmd import Cmd + +stage_dir = '_playlist-manager' +scss_rel_path = os.path.join('src', 'scss', 'style.scss') +css_rel_path = os.path.join('build', 'style.css') + + +class Admin(Cmd): + intro = 'Music Tools Admin... ? for help' + prompt = '> ' + + def prepare_stage(self): + print('>> backing up a directory') + os.chdir('..') + + print('>> deleting old deployment stage') + shutil.rmtree(stage_dir, ignore_errors=True) + + print('>> copying main source') + shutil.copytree('playlist-manager', stage_dir) + + for dependency in ['spotframework', 'fmframework', 'spotfm']: + print(f'>> injecting {dependency}') + shutil.copytree( + os.path.join(dependency, dependency), + os.path.join(stage_dir, dependency) + ) + + os.chdir(stage_dir) + os.system('gcloud config set project sarsooxyz') + + def prepare_frontend(self): + print('>> building css') + os.system(f'sass --style=compressed {scss_rel_path} {css_rel_path}') + + print('>> building javascript') + os.system('npm run build') + + def prepare_main(self, path): + print('>> preparing main.py') + shutil.copy(f'main.{path}.py', 'main.py') + + def deploy_function(self, name, timeout: int = 60): + os.system(f'gcloud functions deploy {name} ' + f'--runtime=python38 ' + f'--set-env-vars DEPLOY_DESTINATION=PROD ' + f'--timeout={timeout}s') + + def do_api(self, args): + self.prepare_stage() + self.prepare_frontend() + self.prepare_main('api') + + print('>> deploying') + os.system('gcloud app deploy') + + def do_tag(self, args): + self.prepare_stage() + self.prepare_main('update_tag') + + print('>> deploying') + self.deploy_function('update_tag') + + def do_playlist(self, args): + self.prepare_stage() + self.prepare_main('run_playlist') + + print('>> deploying') + self.deploy_function('run_user_playlist') + + def do_exit(self, args): + exit(0) + + def do_sass(self, args): + os.system(f'sass --style=compressed {scss_rel_path} {css_rel_path}') + + def do_watchsass(self, args): + os.system(f'sass --style=compressed --watch {scss_rel_path} {css_rel_path}') + + def do_rename(self, args): + from music.model.user import User + from music.model.playlist import Playlist + + username = input('enter username: ') + user = User.collection.filter('username', '==', username).get() + + if user is None: + print('>> user not found') + + name = input('enter playlist name: ') + playlist = Playlist.collection.parent(user.key).filter('name', '==', name).get() + + if playlist is None: + print('>> playlist not found') + + new_name = input('enter new name: ') + playlist.name = new_name + playlist.update() + + +if __name__ == '__main__': + console = Admin() + if len(sys.argv) > 1: + console.onecmd(' '.join(sys.argv[1:])) + else: + console.cmdloop() diff --git a/app.yaml b/app.yaml index ef617a5..c7a41bd 100644 --- a/app.yaml +++ b/app.yaml @@ -1,4 +1,4 @@ -runtime: python37 +runtime: python38 service: spotify instance_class: F2 diff --git a/deploy b/deploy deleted file mode 100755 index 22b0468..0000000 --- a/deploy +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -stage_dir=_playlist-manager - -echo '>> backing up a directory' -cd .. - -echo '>> deleting old deployment stage' -rm -rf $stage_dir - -echo '>> copying main source' -cp -r playlist-manager $stage_dir - -echo '>> injecting spotframework' -cp -r spotframework/spotframework $stage_dir/ - -echo '>> injecting fmframework' -cp -r fmframework/fmframework $stage_dir/ - -echo '>> injecting spotfm' -cp -r spotfm/spotfm $stage_dir/ - -cd $stage_dir - -gcloud config set project sarsooxyz - -echo '>>> Target?' -echo '' -echo '(0) > api' -echo '(1) > update_tag' -read deploy_target - - -case "$deploy_target" in - 0) - echo '>> building css' - sass --style=compressed src/scss/style.scss build/style.css - - echo '>> building javascript' - npm run build - - echo '>> deploying' - gcloud app deploy - ;; - - 1) - echo '>> deploying update_tag' - gcloud functions deploy update_tag - ;; - -esac - - - - diff --git a/main.api.py b/main.api.py new file mode 100644 index 0000000..e0c10a8 --- /dev/null +++ b/main.api.py @@ -0,0 +1,6 @@ +from music.music import create_app + +app = create_app() + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/main.run_playlist.py b/main.run_playlist.py new file mode 100644 index 0000000..c567782 --- /dev/null +++ b/main.run_playlist.py @@ -0,0 +1,15 @@ +def run_user_playlist(event, context): + import logging + + logger = logging.getLogger('music') + + if event.get('attributes'): + if 'username' in event['attributes'] and 'name' in event['attributes']: + + from music.tasks.run_user_playlist import run_user_playlist as do_run_user_playlist + do_run_user_playlist(username=event['attributes']['username'], playlist_name=event['attributes']["name"]) + + else: + logger.error('no parameters in event attributes') + else: + logger.error('no attributes in event') diff --git a/main.py b/main.update_tag.py similarity index 71% rename from main.py rename to main.update_tag.py index 083c8dd..29e08d2 100644 --- a/main.py +++ b/main.update_tag.py @@ -1,9 +1,3 @@ -from music import app -from music.tasks.update_tag import update_tag as do_update_tag - -app = app - - def update_tag(event, context): import logging @@ -11,12 +5,11 @@ def update_tag(event, context): if event.get('attributes'): if 'username' in event['attributes'] and 'tag_id' in event['attributes']: + + from music.tasks.update_tag import update_tag as do_update_tag do_update_tag(username=event['attributes']['username'], tag_id=event['attributes']["tag_id"]) + else: logger.error('no parameters in event attributes') else: logger.error('no attributes in event') - - -if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/music/__init__.py b/music/__init__.py index 37e6942..188d82c 100644 --- a/music/__init__.py +++ b/music/__init__.py @@ -1,5 +1,3 @@ -from .music import app - import logging import os diff --git a/music/api/api.py b/music/api/api.py index e091b58..4be0779 100644 --- a/music/api/api.py +++ b/music/api/api.py @@ -8,7 +8,8 @@ import logging from datetime import datetime from music.api.decorators import login_required, login_or_basic_auth, admin_required, gae_cron, cloud_task -from music.cloud.tasks import update_all_user_playlists, update_playlists, run_user_playlist_task +from music.cloud import queue_run_user_playlist, offload_or_run_user_playlist +from music.cloud.tasks import update_all_user_playlists, update_playlists from music.tasks.run_user_playlist import run_user_playlist from music.model.user import User @@ -284,9 +285,9 @@ def run_playlist(user=None): if playlist_name: if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD': - run_user_playlist_task(user.username, playlist_name) + queue_run_user_playlist(user.username, playlist_name) # pass to either cloud tasks or functions else: - run_user_playlist(user.username, playlist_name) + run_user_playlist(user.username, playlist_name) # update synchronously return jsonify({'message': 'execution requested', 'status': 'success'}), 200 @@ -297,7 +298,7 @@ def run_playlist(user=None): @blueprint.route('/playlist/run/task', methods=['POST']) @cloud_task -def run_playlist_task(): +def run_playlist_task(): # receives cloud tasks request for update payload = request.get_data(as_text=True) if payload: @@ -305,7 +306,7 @@ def run_playlist_task(): logger.info(f'running {payload["username"]} / {payload["name"]}') - run_user_playlist(payload['username'], payload['name']) + offload_or_run_user_playlist(payload['username'], payload['name']) # check whether offloading to cloud function return jsonify({'message': 'executed playlist', 'status': 'success'}), 200 diff --git a/music/auth/auth.py b/music/auth/auth.py index 2cdbce4..d578e2a 100644 --- a/music/auth/auth.py +++ b/music/auth/auth.py @@ -1,7 +1,7 @@ from flask import Blueprint, session, flash, request, redirect, url_for, render_template -from google.cloud import firestore from werkzeug.security import generate_password_hash from music.model.user import User +from music.model.config import Config import urllib import datetime @@ -11,8 +11,6 @@ import requests blueprint = Blueprint('authapi', __name__) -db = firestore.Client() - logger = logging.getLogger(__name__) @@ -98,7 +96,7 @@ def register(): user = User() user.username = username user.password = generate_password_hash(password) - user.last_login = datetime.utcnow() + user.last_login = datetime.datetime.utcnow() user.save() @@ -112,10 +110,10 @@ def auth(): if 'username' in session: - client_id = db.document('key/spotify').get().to_dict()['clientid'] + config = Config.collection.get("config/music-tools") params = urllib.parse.urlencode( { - 'client_id': client_id, + 'client_id': config.spotify_client_id, 'response_type': 'code', 'scope': 'playlist-modify-public playlist-modify-private playlist-read-private user-read-playback-state user-modify-playback-state user-library-read', 'redirect_uri': 'https://music.sarsoo.xyz/auth/spotify/token' @@ -137,9 +135,11 @@ def token(): flash('authorization failed') return redirect('app_route') else: - app_credentials = db.document('key/spotify').get().to_dict() + config = Config.collection.get("config/music-tools") - idsecret = b64encode(bytes(app_credentials['clientid'] + ':' + app_credentials['clientsecret'], "utf-8")).decode("ascii") + idsecret = b64encode( + bytes(config.spotify_client_id + ':' + config.spotify_client_secret, "utf-8") + ).decode("ascii") headers = {'Authorization': 'Basic %s' % idsecret} data = { diff --git a/music/cloud/__init__.py b/music/cloud/__init__.py index e69de29..d25785f 100644 --- a/music/cloud/__init__.py +++ b/music/cloud/__init__.py @@ -0,0 +1,49 @@ +import logging + +from music.model.config import Config +from music.tasks.run_user_playlist import run_user_playlist as run_now +from .function import run_user_playlist_function +from .tasks import run_user_playlist_task + +logger = logging.getLogger(__name__) + + +def queue_run_user_playlist(username: str, playlist_name: str): + config = Config.collection.get("config/music-tools") + + if config is None: + logger.error(f'no config object returned, passing to cloud function {username} / {playlist_name}') + run_user_playlist_function(username=username, playlist_name=playlist_name) + + if config.playlist_cloud_operating_mode == 'task': + logger.debug(f'passing {username} / {playlist_name} to cloud tasks') + run_user_playlist_task(username=username, playlist_name=playlist_name) + + elif config.playlist_cloud_operating_mode == 'function': + logger.debug(f'passing {username} / {playlist_name} to cloud function') + run_user_playlist_function(username=username, playlist_name=playlist_name) + + else: + logger.critical(f'invalid operating mode {username} / {playlist_name}, ' + f'{config.playlist_cloud_operating_mode}, passing to cloud function') + run_user_playlist_function(username=username, playlist_name=playlist_name) + + +def offload_or_run_user_playlist(username: str, playlist_name: str): + config = Config.collection.get("config/music-tools") + + if config is None: + logger.error(f'no config object returned, passing to cloud function {username} / {playlist_name}') + run_user_playlist_function(username=username, playlist_name=playlist_name) + + if config.playlist_cloud_operating_mode == 'task': + run_now(username=username, playlist_name=playlist_name) + + elif config.playlist_cloud_operating_mode == 'function': + logger.debug(f'offloading {username} / {playlist_name} to cloud function') + run_user_playlist_function(username=username, playlist_name=playlist_name) + + else: + logger.critical(f'invalid operating mode {username} / {playlist_name}, ' + f'{config.playlist_cloud_operating_mode}, passing to cloud function') + run_user_playlist_function(username=username, playlist_name=playlist_name) diff --git a/music/cloud/function.py b/music/cloud/function.py index 670c790..a04551b 100644 --- a/music/cloud/function.py +++ b/music/cloud/function.py @@ -9,12 +9,27 @@ def update_tag(username, tag_id): """Queue serverless tag update for user""" logger.info(f'queuing {tag_id} update for {username}') - if username is None: - logger.error(f'no username provided') + if username is None or tag_id is None: + logger.error(f'less than two args provided, {username} / {tag_id}') return - if tag_id is None: - logger.error(f'no tag_id provided for {username}') + if not isinstance(username, str) or not isinstance(tag_id, str): + logger.error(f'less than two strings provided, {type(username)} / {type(tag_id)}') return publisher.publish('projects/sarsooxyz/topics/update_tag', b'', tag_id=tag_id, username=username) + + +def run_user_playlist_function(username, playlist_name): + """Queue serverless playlist update for user""" + logger.info(f'queuing {playlist_name} update for {username}') + + if username is None or playlist_name is None: + logger.error(f'less than two args provided, {username} / {playlist_name}') + return + + if not isinstance(username, str) or not isinstance(playlist_name, str): + logger.error(f'less than two strings provided, {type(username)} / {type(playlist_name)}') + return + + publisher.publish('projects/sarsooxyz/topics/run_user_playlist', b'', name=playlist_name, username=username) diff --git a/music/db/database.py b/music/db/database.py index 58eb7c7..6bbe90d 100644 --- a/music/db/database.py +++ b/music/db/database.py @@ -1,13 +1,12 @@ -from google.cloud import firestore +from dataclasses import dataclass import logging from datetime import timedelta, datetime, timezone from spotframework.net.network import Network as SpotifyNetwork, SpotifyNetworkException +from spotframework.net.user import NetworkUser from fmframework.net.network import Network as FmNetwork -from music.db.user import DatabaseUser from music.model.user import User - -db = firestore.Client() +from music.model.config import Config logger = logging.getLogger(__name__) @@ -33,10 +32,10 @@ def refresh_token_database_callback(user): def get_authed_spotify_network(user): if user is not None: if user.spotify_linked: - spotify_keys = db.document('key/spotify').get().to_dict() + config = Config.collection.get("config/music-tools") - user_obj = DatabaseUser(client_id=spotify_keys['clientid'], - client_secret=spotify_keys['clientsecret'], + user_obj = DatabaseUser(client_id=config.spotify_client_id, + client_secret=config.spotify_client_secret, refresh_token=user.refresh_token, user_id=user.username, access_token=user.access_token) @@ -66,9 +65,15 @@ def get_authed_spotify_network(user): def get_authed_lastfm_network(user): if user is not None: if user.lastfm_username: - fm_keys = db.document('key/fm').get().to_dict() - return FmNetwork(username=user.lastfm_username, api_key=fm_keys['clientid']) + config = Config.collection.get("config/music-tools") + return FmNetwork(username=user.lastfm_username, api_key=config.last_fm_client_id) else: logger.error(f'{user.username} has no last.fm username') else: logger.error(f'no user provided') + + +@dataclass +class DatabaseUser(NetworkUser): + """adding music tools username to spotframework network user""" + user_id: str = None diff --git a/music/db/user.py b/music/db/user.py deleted file mode 100644 index 6957abc..0000000 --- a/music/db/user.py +++ /dev/null @@ -1,14 +0,0 @@ -from spotframework.util.console import Color -from spotframework.net.user import NetworkUser - - -class DatabaseUser(NetworkUser): - - def __init__(self, client_id, client_secret, refresh_token, user_id, access_token=None): - super().__init__(client_id=client_id, client_secret=client_secret, - refresh_token=refresh_token, access_token=access_token) - self.user_id = user_id - - def __repr__(self): - return Color.RED + Color.BOLD + 'DatabaseUser' + Color.END + \ - f': {self.user_id}, {self.username}, {self.display_name}, {self.uri}' diff --git a/music/model/config.py b/music/model/config.py new file mode 100644 index 0000000..449d04b --- /dev/null +++ b/music/model/config.py @@ -0,0 +1,14 @@ +from fireo.models import Model +from fireo.fields import TextField, BooleanField, DateTime, NumberField, ListField + + +class Config(Model): + class Meta: + collection_name = 'config' + + spotify_client_id = TextField() + spotify_client_secret = TextField() + last_fm_client_id = TextField() + + playlist_cloud_operating_mode = TextField() # task, function + secret_key = TextField() diff --git a/music/music.py b/music/music.py index e625544..a5c16c0 100644 --- a/music/music.py +++ b/music/music.py @@ -1,46 +1,55 @@ from flask import Flask, render_template, redirect, session, flash, url_for -from google.cloud import firestore +import logging import os from music.auth import auth_blueprint from music.api import api_blueprint, player_blueprint, fm_blueprint, \ spotfm_blueprint, spotify_blueprint, admin_blueprint, tag_blueprint +from music.model.config import Config -db = firestore.Client() - -app = Flask(__name__, static_folder=os.path.join(os.path.dirname(__file__), '..', 'build'), template_folder="templates") -app.secret_key = db.collection(u'spotify').document(u'config').get().to_dict()['secret_key'] -app.register_blueprint(auth_blueprint, url_prefix='/auth') -app.register_blueprint(api_blueprint, url_prefix='/api') -app.register_blueprint(player_blueprint, url_prefix='/api/player') -app.register_blueprint(fm_blueprint, url_prefix='/api/fm') -app.register_blueprint(spotfm_blueprint, url_prefix='/api/spotfm') -app.register_blueprint(spotify_blueprint, url_prefix='/api/spotify') -app.register_blueprint(admin_blueprint, url_prefix='/api/admin') -app.register_blueprint(tag_blueprint, url_prefix='/api') +logger = logging.getLogger(__name__) -@app.route('/') -def index(): +def create_app(): + app = Flask(__name__, static_folder=os.path.join(os.path.dirname(__file__), '..', 'build'), template_folder="templates") - if 'username' in session: - logged_in = True - return redirect(url_for('app_route')) + config = Config.collection.get("config/music-tools") + if config is not None: + app.secret_key = config.secret_key else: - logged_in = False + logger.error('no config returned, skipping secret key') - return render_template('login.html', logged_in=logged_in) + app.register_blueprint(auth_blueprint, url_prefix='/auth') + app.register_blueprint(api_blueprint, url_prefix='/api') + app.register_blueprint(player_blueprint, url_prefix='/api/player') + app.register_blueprint(fm_blueprint, url_prefix='/api/fm') + app.register_blueprint(spotfm_blueprint, url_prefix='/api/spotfm') + app.register_blueprint(spotify_blueprint, url_prefix='/api/spotify') + app.register_blueprint(admin_blueprint, url_prefix='/api/admin') + app.register_blueprint(tag_blueprint, url_prefix='/api') + @app.route('/') + def index(): -@app.route('/app', defaults={'path': ''}) -@app.route('/app/') -def app_route(path): + if 'username' in session: + logged_in = True + return redirect(url_for('app_route')) + else: + logged_in = False - if 'username' not in session: - flash('please log in') - return redirect(url_for('index')) + return render_template('login.html', logged_in=logged_in) - return render_template('app.html') + @app.route('/app', defaults={'path': ''}) + @app.route('/app/') + def app_route(path): + + if 'username' not in session: + flash('please log in') + return redirect(url_for('index')) + + return render_template('app.html') + + return app # [END gae_python37_app] diff --git a/music/tasks/create_playlist.py b/music/tasks/create_playlist.py index 001d5bc..f031a4e 100644 --- a/music/tasks/create_playlist.py +++ b/music/tasks/create_playlist.py @@ -1,12 +1,8 @@ -from google.cloud import firestore - import logging import music.db.database as database from spotframework.net.network import SpotifyNetworkException -db = firestore.Client() - logger = logging.getLogger(__name__) diff --git a/music/tasks/refresh_lastfm_stats.py b/music/tasks/refresh_lastfm_stats.py index 99eb2bc..3e0ef40 100644 --- a/music/tasks/refresh_lastfm_stats.py +++ b/music/tasks/refresh_lastfm_stats.py @@ -1,5 +1,3 @@ -from google.cloud import firestore - import logging from datetime import datetime @@ -13,8 +11,6 @@ from spotframework.net.network import SpotifyNetworkException from fmframework.net.network import LastFMNetworkException -db = firestore.Client() - logger = logging.getLogger(__name__) diff --git a/rename_playlist.py b/rename_playlist.py deleted file mode 100644 index 5d45daf..0000000 --- a/rename_playlist.py +++ /dev/null @@ -1,14 +0,0 @@ -from music.model.user import User -from music.model.playlist import Playlist - -user = User.collection.filter('username', '==', 'andy').get() - -name = input('enter playlist name: ') -playlist = Playlist.collection.parent(user.key).filter('name', '==', name).get() - -if playlist is not None: - new_name = input('enter new name: ') - playlist.name = new_name - playlist.update() -else: - print('playlist not found') diff --git a/requirements.txt b/requirements.txt index 845e4ea..344401b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,15 +15,17 @@ google-cloud-tasks==1.5.0 googleapis-common-protos==1.52.0 grpc-google-iam-v1==0.12.3 grpcio==1.30.0 +gunicorn==20.0.4 idna==2.9 isort==4.3.21 itsdangerous==1.1.0 Jinja2==2.11.2 -lazy-object-proxy==1.5.0 +lazy-object-proxy==1.4.3 MarkupSafe==1.1.1 mccabe==0.6.1 numpy==1.19.0 opencv-python==4.2.0.34 +pathtools==0.1.2 protobuf==3.12.2 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -35,5 +37,6 @@ six==1.15.0 tabulate==0.8.7 toml==0.10.1 urllib3==1.25.9 +watchdog==0.10.3 Werkzeug==1.0.1 wrapt==1.12.1 diff --git a/sass b/sass deleted file mode 100755 index 9ed3ed9..0000000 --- a/sass +++ /dev/null @@ -1 +0,0 @@ -sass --style=compressed src/scss/style.scss build/style.css diff --git a/watchsass b/watchsass deleted file mode 100755 index c080fd3..0000000 --- a/watchsass +++ /dev/null @@ -1 +0,0 @@ -sass --style=compressed --watch src/scss/style.scss build/style.css