From a97f7d3e161cfd198bf06970a4a4923e470f5a96 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Sun, 19 May 2019 12:53:58 -0700 Subject: [PATCH] Migrate updater from lambda to container --- cmd/updater/.dockerignore | 6 ++ cmd/updater/.gitignore | 3 + cmd/updater/Dockerfile | 14 +++++ cmd/updater/Makefile | 5 ++ cmd/updater/dynamo.py | 110 ----------------------------------- cmd/updater/dynamo_test.py | 10 ---- cmd/updater/function.py | 6 +- cmd/updater/requirements.txt | 1 + cmd/updater/server.py | 38 ++++++++++++ cmd/updater/updater.py | 4 +- cmd/updater/updater_test.py | 16 ++--- 11 files changed, 80 insertions(+), 133 deletions(-) create mode 100644 cmd/updater/.dockerignore create mode 100644 cmd/updater/Dockerfile delete mode 100644 cmd/updater/dynamo.py delete mode 100644 cmd/updater/dynamo_test.py create mode 100644 cmd/updater/server.py diff --git a/cmd/updater/.dockerignore b/cmd/updater/.dockerignore new file mode 100644 index 0000000..8650e53 --- /dev/null +++ b/cmd/updater/.dockerignore @@ -0,0 +1,6 @@ +.idea/ +venv/ +package/ +__pycache__/ + +function.zip \ No newline at end of file diff --git a/cmd/updater/.gitignore b/cmd/updater/.gitignore index b413a81..8650e53 100644 --- a/cmd/updater/.gitignore +++ b/cmd/updater/.gitignore @@ -1,3 +1,6 @@ +.idea/ +venv/ package/ +__pycache__/ function.zip \ No newline at end of file diff --git a/cmd/updater/Dockerfile b/cmd/updater/Dockerfile new file mode 100644 index 0000000..bee9059 --- /dev/null +++ b/cmd/updater/Dockerfile @@ -0,0 +1,14 @@ +FROM python:alpine3.7 + +WORKDIR /app +COPY . . + +RUN apk add --virtual deps --no-cache build-base && \ + pip3 install --no-cache-dir --requirement requirements.txt --target /app && \ + apk del deps && \ + addgroup -g 1000 -S app && adduser -u 1000 -G app -S app + +USER app + +ENTRYPOINT ["python3", "-m", "sanic", "server.app", "--host", "0.0.0.0", "--port", "8080"] +CMD ["--workers", "1"] diff --git a/cmd/updater/Makefile b/cmd/updater/Makefile index 1403859..7b12cb3 100644 --- a/cmd/updater/Makefile +++ b/cmd/updater/Makefile @@ -22,6 +22,11 @@ update: build --function-name $(NAME) \ --zip-file fileb://function.zip +.PHONY: push +push: + docker build -t mxpv/updater . + docker push mxpv/updater + clean: rm -rf package function.zip diff --git a/cmd/updater/dynamo.py b/cmd/updater/dynamo.py deleted file mode 100644 index 34b4ba2..0000000 --- a/cmd/updater/dynamo.py +++ /dev/null @@ -1,110 +0,0 @@ -import boto3 -import os -import time -from updater import DEFAULT_PAGE_SIZE, _get_updates, _get_format -from datetime import datetime - -dynamodb = boto3.resource('dynamodb') - -feeds_table = dynamodb.Table(os.getenv('UPDATER_DYNAMO_FEEDS_TABLE', 'Feeds')) - - -def _update_feed(hash_id): - print('Updating feed {}'.format(hash_id)) - feed = _query_feed(hash_id) - - page_size = int(feed.get('PageSize', DEFAULT_PAGE_SIZE)) - last_id = feed.get('LastID', None) - episodes = feed.get('Episodes', []) - item_url = feed['ItemURL'] - - # Format parameters - fmt = feed.get('Format', 'video') - quality = feed.get('Quality', 'high') - - # Rebuild episode list from scratch - if not last_id: - episodes = [] - - start = time.time() - _, items, new_last_id = _get_updates(1, page_size, item_url, _get_format(fmt, quality), last_id) - end = time.time() - - print('Got feed update: new {}, current {}. Update took: {}'.format(len(items), len(episodes), end-start)) - - # Update feed and submit back to Dynamo - - unix_time = int(datetime.utcnow().timestamp()) - feed['UpdatedAt'] = unix_time - - if len(items) > 0: - episodes = items + episodes # Prepand new episodes - del episodes[page_size:] # Truncate list - feed['Episodes'] = episodes - - # Update last seen video ID - feed['LastID'] = new_last_id - - _update_feed_episodes(hash_id, feed) - else: - # Update last access field only - _update_feed_updated_at(hash_id, unix_time) - - -def _query_feed(hash_id): - response = feeds_table.get_item( - Key={'HashID': hash_id}, - ProjectionExpression='#prov,#type,#size,#fmt,#quality,#level,#id,#last_id,#episodes,#updated_at,#item_url', - ExpressionAttributeNames={ - '#prov': 'Provider', - '#type': 'LinkType', - '#size': 'PageSize', - '#fmt': 'Format', - '#quality': 'Quality', - '#level': 'FeatureLevel', - '#id': 'ItemID', - '#last_id': 'LastID', - '#episodes': 'Episodes', - '#updated_at': 'UpdatedAt', - '#item_url': 'ItemURL', - }, - ) - - item = response['Item'] - return item - - -def _update_feed_episodes(hash_id, feed): - feeds_table.update_item( - Key={ - 'HashID': hash_id, - }, - UpdateExpression='SET #updated_at = :updated_at, #episodes = :episodes, #last_id = :last_id', - ExpressionAttributeNames={ - '#updated_at': 'UpdatedAt', - '#episodes': 'Episodes', - '#last_id': 'LastID', - }, - ExpressionAttributeValues={ - ':updated_at': feed['UpdatedAt'], - ':episodes': feed['Episodes'], - ':last_id': feed['LastID'], - }, - ReturnValues='NONE', - ) - - -def _update_feed_updated_at(hash_id, updated_at): - feeds_table.update_item( - Key={ - 'HashID': hash_id, - }, - UpdateExpression='SET #updated_at = :updated_at', - ExpressionAttributeNames={ - '#updated_at': 'UpdatedAt', - }, - ExpressionAttributeValues={ - ':updated_at': updated_at, - }, - ReturnValues='NONE', - ) diff --git a/cmd/updater/dynamo_test.py b/cmd/updater/dynamo_test.py deleted file mode 100644 index a308918..0000000 --- a/cmd/updater/dynamo_test.py +++ /dev/null @@ -1,10 +0,0 @@ -import dynamo -import unittest - -TEST_FEED = '86qZ' - - -class TestUpdater(unittest.TestCase): - @unittest.skip - def test_update_feed(self): - dynamo._update_feed(TEST_FEED) diff --git a/cmd/updater/function.py b/cmd/updater/function.py index 21e9076..d9d2f99 100644 --- a/cmd/updater/function.py +++ b/cmd/updater/function.py @@ -1,4 +1,4 @@ -from updater import DEFAULT_PAGE_SIZE, _get_updates, _get_format +from updater import DEFAULT_PAGE_SIZE, get_updates, get_format # AWS Lambda entry point @@ -13,10 +13,10 @@ def handler(event, context): # Detect item format fmt = event.get('format', 'video') quality = event.get('quality', 'high') - ytdl_fmt = _get_format(fmt, quality) + ytdl_fmt = get_format(fmt, quality) print('Getting updates for %s (start=%d, count=%d, fmt: %s, last id: %s)' % (url, start, count, ytdl_fmt, last_id)) - _, episodes, new_last_id = _get_updates(start, count, url, ytdl_fmt, last_id) + _, episodes, new_last_id = get_updates(start, count, url, ytdl_fmt, last_id) return { 'last_id': new_last_id, diff --git a/cmd/updater/requirements.txt b/cmd/updater/requirements.txt index 5a8ab32..6c0a492 100644 --- a/cmd/updater/requirements.txt +++ b/cmd/updater/requirements.txt @@ -1,2 +1,3 @@ boto3==1.9.129 youtube_dl==2019.04.24 +sanic==18.12 diff --git a/cmd/updater/server.py b/cmd/updater/server.py new file mode 100644 index 0000000..23c8f92 --- /dev/null +++ b/cmd/updater/server.py @@ -0,0 +1,38 @@ +import updater +from sanic import Sanic, response +from sanic.exceptions import InvalidUsage + +app = Sanic() + + +@app.get('/update') +async def update(req): + url = req.args.get('url', None) + start = req.args.get('start', 1) + count = req.args.get('count', updater.DEFAULT_PAGE_SIZE) + + # Last seen video ID + last_id = req.args.get('last_id', None) + + # Detect item format + fmt = req.args.get('format', 'video') + quality = req.args.get('quality', 'high') + ytdl_fmt = updater.get_format(fmt, quality) + + try: + _, episodes, new_last_id = updater.get_updates(start, count, url, ytdl_fmt, last_id) + return response.json({ + 'last_id': new_last_id, + 'episodes': episodes, + }) + except ValueError: + raise InvalidUsage() + + +@app.get('/ping') +async def ping(req): + return response.text('pong') + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8080) diff --git a/cmd/updater/updater.py b/cmd/updater/updater.py index 3f0f293..3d61e05 100644 --- a/cmd/updater/updater.py +++ b/cmd/updater/updater.py @@ -5,7 +5,7 @@ BEST_FORMAT = "bestvideo+bestaudio/best" DEFAULT_PAGE_SIZE = 50 -def _get_format(fmt, quality): +def get_format(fmt, quality): if fmt == 'video': # Video if quality == 'high': @@ -20,7 +20,7 @@ def _get_format(fmt, quality): return 'worstaudio' -def _get_updates(start, count, url, fmt, last_id=None): +def get_updates(start, count, url, fmt, last_id=None): if start < 1: raise ValueError('Invalid start value') diff --git a/cmd/updater/updater_test.py b/cmd/updater/updater_test.py index 2eda679..77d8778 100644 --- a/cmd/updater/updater_test.py +++ b/cmd/updater/updater_test.py @@ -7,30 +7,30 @@ TEST_URL = 'https://www.youtube.com/user/CNN/videos' class TestUpdater(unittest.TestCase): def test_get_updates(self): kinds = [ - updater._get_format('video', 'high'), - updater._get_format('video', 'low'), - updater._get_format('audio', 'high'), - updater._get_format('audio', 'low'), + updater.get_format('video', 'high'), + updater.get_format('video', 'low'), + updater.get_format('audio', 'high'), + updater.get_format('audio', 'low'), ] for kind in kinds: with self.subTest(kind): - feed, items, _ = updater._get_updates(1, 1, TEST_URL, kind) + feed, items, _ = updater.get_updates(1, 1, TEST_URL, kind) self.assertIsNotNone(feed) self.assertIsNotNone(items) def test_get_change_list(self): - feed, items, last_id = updater._get_updates(1, 5, TEST_URL, 'worst[ext=mp4]') + feed, items, last_id = updater.get_updates(1, 5, TEST_URL, 'worst[ext=mp4]') self.assertEqual(len(items), 5) self.assertEqual(items[0]['ID'], last_id) test_last_id = items[2]['ID'] self.assertIsNotNone(test_last_id) - feed, items, last_id = updater._get_updates(1, 5, TEST_URL, 'worst[ext=mp4]', test_last_id) + feed, items, last_id = updater.get_updates(1, 5, TEST_URL, 'worst[ext=mp4]', test_last_id) self.assertEqual(len(items), 2) self.assertEqual(items[0]['ID'], last_id) def test_last_id(self): - feed, items, last_id = updater._get_updates(1, 1, TEST_URL, 'worstaudio') + feed, items, last_id = updater.get_updates(1, 1, TEST_URL, 'worstaudio') self.assertEqual(len(items), 1) self.assertEqual(items[0]['ID'], last_id)