From 4a0c045b27da977669eae6a3e5c6736d60257613 Mon Sep 17 00:00:00 2001 From: Andy Pack Date: Mon, 4 Sep 2023 23:06:59 +0100 Subject: [PATCH] tweaking function entry, updating admin for gen2 adding func frameworks --- admin.py | 9 +++ main.cron.py | 12 +++- main.run_playlist.py | 19 +++--- main.update_tag.py | 20 ++++--- poetry.lock | 138 +++++++++++++++++++++++++++++++++++++++---- pyproject.toml | 1 + 6 files changed, 170 insertions(+), 29 deletions(-) diff --git a/admin.py b/admin.py index f0fe4cd..9ec1270 100755 --- a/admin.py +++ b/admin.py @@ -76,6 +76,13 @@ class Admin(Cmd): print('>> setting project') subprocess.check_call('gcloud config set project sarsooxyz', shell=True) + @property + def gcloud_project(self): + return subprocess.run(["gcloud", "config", "get-value", "project"], stdout=subprocess.PIPE).stdout.decode("utf-8").strip() + + def do_project(self, args): + print(f"\"{self.gcloud_project}\"") + def deploy_function(self, name, timeout: int = 60, region='europe-west2'): """ Deploy function with required environment variables @@ -83,9 +90,11 @@ class Admin(Cmd): subprocess.check_call( f'gcloud functions deploy {name} ' f'--region {region} ' + f'--gen2 ' '--runtime=python311 ' f'--trigger-topic {name} ' '--set-env-vars DEPLOY_DESTINATION=PROD ' + f'--service-account {name.replace("_", "-")}-func@{self.gcloud_project}.iam.gserviceaccount.com ' f'--timeout={timeout}s', shell=True ) diff --git a/main.cron.py b/main.cron.py index dd7f8e3..f1e4725 100644 --- a/main.cron.py +++ b/main.cron.py @@ -1,13 +1,19 @@ +from cloudevents.http import CloudEvent +import functions_framework + from music.cloud.tasks import update_all_user_playlists, refresh_all_user_playlist_stats, update_all_user_tags -def run_all_playlists(event, context): +@functions_framework.cloud_event +def run_all_playlists(event: CloudEvent): update_all_user_playlists() -def run_all_playlist_stats(event, context): +@functions_framework.cloud_event +def run_all_playlist_stats(event: CloudEvent): refresh_all_user_playlist_stats() -def run_all_tags(event, context): +@functions_framework.cloud_event +def run_all_tags(event: CloudEvent): update_all_user_tags() diff --git a/main.run_playlist.py b/main.run_playlist.py index b1812e7..ab2715f 100644 --- a/main.run_playlist.py +++ b/main.run_playlist.py @@ -1,15 +1,18 @@ -def run_user_playlist(event, context): +from cloudevents.http import CloudEvent +import functions_framework + +# Register a CloudEvent function with the Functions Framework +@functions_framework.cloud_event +def run_user_playlist_new(event: CloudEvent): import logging logger = logging.getLogger('music') - if event.get('attributes'): - if 'username' in event['attributes'] and 'name' in event['attributes']: + attr = event.get_data()['message']['attributes'] + if 'username' in attr and 'name' in attr: - from music.tasks.run_user_playlist import run_user_playlist as do_run_user_playlist - do_run_user_playlist(user=event['attributes']['username'], playlist=event['attributes']["name"]) + from music.tasks.run_user_playlist import run_user_playlist as do_run_user_playlist + do_run_user_playlist(user=attr['username'], playlist=attr["name"]) - else: - logger.error('no parameters in event attributes') else: - logger.error('no attributes in event') + logger.error('no parameters in event attributes') diff --git a/main.update_tag.py b/main.update_tag.py index 31530fd..d684bba 100644 --- a/main.update_tag.py +++ b/main.update_tag.py @@ -1,15 +1,19 @@ -def update_tag(event, context): +from cloudevents.http import CloudEvent +import functions_framework + +# Register a CloudEvent function with the Functions Framework +@functions_framework.cloud_event +def update_tag(event: CloudEvent): import logging logger = logging.getLogger('music') - if event.get('attributes'): - if 'username' in event['attributes'] and 'tag_id' in event['attributes']: + attr = event.get_data()['message']['attributes'] - from music.tasks.update_tag import update_tag as do_update_tag - do_update_tag(user=event['attributes']['username'], tag=event['attributes']["tag_id"]) + if 'username' in attr and 'tag_id' in attr: + + from music.tasks.update_tag import update_tag as do_update_tag + do_update_tag(user=attr['username'], tag=attr["tag_id"]) - else: - logger.error('no parameters in event attributes') else: - logger.error('no attributes in event') + logger.error('no parameters in event attributes') diff --git a/poetry.lock b/poetry.lock index 602f644..4d5a1cf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -206,6 +206,23 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cloudevents" +version = "1.9.0" +description = "CloudEvents Python SDK" +optional = false +python-versions = "*" +files = [ + {file = "cloudevents-1.9.0-py3-none-any.whl", hash = "sha256:1011459d56d8f0184a46456f5d72632a2565f18171e51b33e06f643e723d30c9"}, + {file = "cloudevents-1.9.0.tar.gz", hash = "sha256:8beb27503f97e215f886f73c17671012e96bb6268137fb3b2f9ef552727ab5b1"}, +] + +[package.dependencies] +deprecation = ">=2.0,<3.0" + +[package.extras] +pydantic = ["pydantic (>=1.0.0,<2.0)"] + [[package]] name = "colorama" version = "0.4.6" @@ -217,6 +234,20 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "deprecation" +version = "2.1.0" +description = "A library to handle automated deprecations" +optional = false +python-versions = "*" +files = [ + {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, + {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, +] + +[package.dependencies] +packaging = "*" + [[package]] name = "dill" version = "0.3.7" @@ -300,6 +331,24 @@ url = "https://github.com/Sarsoo/pyfmframework.git" reference = "master" resolved_reference = "0561b0036e9f45bc35f51ff83fb33d79f61329e7" +[[package]] +name = "functions-framework" +version = "3.4.0" +description = "An open source FaaS (Function as a service) framework for writing portable Python functions -- brought to you by the Google Cloud Functions team." +optional = false +python-versions = ">=3.5, <4" +files = [ + {file = "functions-framework-3.4.0.tar.gz", hash = "sha256:c05639bcdd19f11a2eef1b6dabde5998f7d67cbddbffdf2f06ca56dc13c54e95"}, + {file = "functions_framework-3.4.0-py3-none-any.whl", hash = "sha256:e22f30a66aab7180373f9a81f88114da088915b3622c62dcd6ec972ac908e0c4"}, +] + +[package.dependencies] +click = ">=7.0,<9.0" +cloudevents = ">=1.2.0,<2.0.0" +flask = ">=1.0,<3.0" +gunicorn = {version = ">=19.2.0,<21.0", markers = "platform_system != \"Windows\""} +watchdog = ">=1.0.0" + [[package]] name = "google-api-core" version = "2.11.1" @@ -314,14 +363,8 @@ files = [ [package.dependencies] google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" -grpcio = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, -] -grpcio-status = [ - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}, - {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, -] +grpcio = {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} +grpcio-status = {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -604,6 +647,26 @@ googleapis-common-protos = ">=1.5.5" grpcio = ">=1.56.2" protobuf = ">=4.21.6" +[[package]] +name = "gunicorn" +version = "20.1.0" +description = "WSGI HTTP Server for UNIX" +optional = false +python-versions = ">=3.5" +files = [ + {file = "gunicorn-20.1.0-py3-none-any.whl", hash = "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e"}, + {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, +] + +[package.dependencies] +setuptools = ">=3.0" + +[package.extras] +eventlet = ["eventlet (>=0.24.1)"] +gevent = ["gevent (>=1.4.0)"] +setproctitle = ["setproctitle"] +tornado = ["tornado (>=0.2)"] + [[package]] name = "h11" version = "0.14.0" @@ -1073,6 +1136,22 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "setuptools" +version = "68.1.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, + {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -1320,6 +1399,45 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "watchdog" +version = "3.0.0" +description = "Filesystem events monitoring" +optional = false +python-versions = ">=3.7" +files = [ + {file = "watchdog-3.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41"}, + {file = "watchdog-3.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397"}, + {file = "watchdog-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96"}, + {file = "watchdog-3.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae"}, + {file = "watchdog-3.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9"}, + {file = "watchdog-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7"}, + {file = "watchdog-3.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674"}, + {file = "watchdog-3.0.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f"}, + {file = "watchdog-3.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc"}, + {file = "watchdog-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3"}, + {file = "watchdog-3.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3"}, + {file = "watchdog-3.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0"}, + {file = "watchdog-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8"}, + {file = "watchdog-3.0.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100"}, + {file = "watchdog-3.0.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346"}, + {file = "watchdog-3.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33"}, + {file = "watchdog-3.0.0-py3-none-win32.whl", hash = "sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f"}, + {file = "watchdog-3.0.0-py3-none-win_amd64.whl", hash = "sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c"}, + {file = "watchdog-3.0.0-py3-none-win_ia64.whl", hash = "sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759"}, + {file = "watchdog-3.0.0.tar.gz", hash = "sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9"}, +] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + [[package]] name = "werkzeug" version = "2.3.6" @@ -1424,4 +1542,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "6078bb265686e365bd85f887faf7ccad94131fd398d83ce0a9d9474b2515ed13" +content-hash = "a62b8ff1d207a4c91dd32c0dbbaa6cfd54ad8517ca2af1496ab8f7bb5bbc02c8" diff --git a/pyproject.toml b/pyproject.toml index 2ddfeec..6b85c51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ google-cloud-tasks = "^2.10.0" requests = "^2.28.1" httpx = {extras = ["http2"], version = "^0.24.1"} PyJWT = "^2.4.0" +functions-framework = "^3.4.0" spotframework = { git = "https://github.com/Sarsoo/spotframework.git", branch = "master" } fmframework = { git = "https://github.com/Sarsoo/pyfmframework.git", branch = "master" }