mirror/mxpv-podsync
1
0
Fork 0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00
Code Issues Releases Activity

Migrate updater from lambda to container

Browse Source
This commit is contained in:
Maksym Pavlenko
2019-05-19 12:53:58 -07:00
parent d57f04efc2
commit a97f7d3e16
11 changed files with 80 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/updater/.dockerignore
+6
View File
@@ -0,0 +1,6 @@
.idea/
venv/
package/
__pycache__/
function.zip
cmd/updater/.gitignore
+3
View File
@@ -1,3 +1,6 @@
.idea/
venv/
package/
__pycache__/
function.zip
cmd/updater/Dockerfile
+14
View File
@@ -0,0 +1,14 @@
FROM python:alpine3.7
WORKDIR /app
COPY . .
RUN apk add --virtual deps --no-cache build-base && \
pip3 install --no-cache-dir --requirement requirements.txt --target /app && \
apk del deps && \
addgroup -g 1000 -S app && adduser -u 1000 -G app -S app
USER app
ENTRYPOINT ["python3", "-m", "sanic", "server.app", "--host", "0.0.0.0", "--port", "8080"]
CMD ["--workers", "1"]
cmd/updater/Makefile
+5
View File
@@ -22,6 +22,11 @@ update: build
--function-name $(NAME) \
--zip-file fileb://function.zip
.PHONY: push
push:
docker build -t mxpv/updater .
docker push mxpv/updater
clean:
rm -rf package function.zip
cmd/updater/dynamo.py
-110
View File
@@ -1,110 +0,0 @@
import boto3
import os
import time
from updater import DEFAULT_PAGE_SIZE, _get_updates, _get_format
from datetime import datetime
dynamodb = boto3.resource('dynamodb')
feeds_table = dynamodb.Table(os.getenv('UPDATER_DYNAMO_FEEDS_TABLE', 'Feeds'))
def _update_feed(hash_id):
print('Updating feed {}'.format(hash_id))
feed = _query_feed(hash_id)
page_size = int(feed.get('PageSize', DEFAULT_PAGE_SIZE))
last_id = feed.get('LastID', None)
episodes = feed.get('Episodes', [])
item_url = feed['ItemURL']
# Format parameters
fmt = feed.get('Format', 'video')
quality = feed.get('Quality', 'high')
# Rebuild episode list from scratch
if not last_id:
episodes = []
start = time.time()
_, items, new_last_id = _get_updates(1, page_size, item_url, _get_format(fmt, quality), last_id)
end = time.time()
print('Got feed update: new {}, current {}. Update took: {}'.format(len(items), len(episodes), end-start))
# Update feed and submit back to Dynamo
unix_time = int(datetime.utcnow().timestamp())
feed['UpdatedAt'] = unix_time
if len(items) > 0:
episodes = items + episodes # Prepand new episodes
del episodes[page_size:] # Truncate list
feed['Episodes'] = episodes
# Update last seen video ID
feed['LastID'] = new_last_id
_update_feed_episodes(hash_id, feed)
else:
# Update last access field only
_update_feed_updated_at(hash_id, unix_time)
def _query_feed(hash_id):
response = feeds_table.get_item(
Key={'HashID': hash_id},
ProjectionExpression='#prov,#type,#size,#fmt,#quality,#level,#id,#last_id,#episodes,#updated_at,#item_url',
ExpressionAttributeNames={
'#prov': 'Provider',
'#type': 'LinkType',
'#size': 'PageSize',
'#fmt': 'Format',
'#quality': 'Quality',
'#level': 'FeatureLevel',
'#id': 'ItemID',
'#last_id': 'LastID',
'#episodes': 'Episodes',
'#updated_at': 'UpdatedAt',
'#item_url': 'ItemURL',
},
)
item = response['Item']
return item
def _update_feed_episodes(hash_id, feed):
feeds_table.update_item(
Key={
'HashID': hash_id,
},
UpdateExpression='SET #updated_at = :updated_at, #episodes = :episodes, #last_id = :last_id',
ExpressionAttributeNames={
'#updated_at': 'UpdatedAt',
'#episodes': 'Episodes',
'#last_id': 'LastID',
},
ExpressionAttributeValues={
':updated_at': feed['UpdatedAt'],
':episodes': feed['Episodes'],
':last_id': feed['LastID'],
},
ReturnValues='NONE',
)
def _update_feed_updated_at(hash_id, updated_at):
feeds_table.update_item(
Key={
'HashID': hash_id,
},
UpdateExpression='SET #updated_at = :updated_at',
ExpressionAttributeNames={
'#updated_at': 'UpdatedAt',
},
ExpressionAttributeValues={
':updated_at': updated_at,
},
ReturnValues='NONE',
)
cmd/updater/dynamo_test.py
-10
View File
@@ -1,10 +0,0 @@
import dynamo
import unittest
TEST_FEED = '86qZ'
class TestUpdater(unittest.TestCase):
@unittest.skip
def test_update_feed(self):
dynamo._update_feed(TEST_FEED)
cmd/updater/function.py
+3 -3
View File
@@ -1,4 +1,4 @@
from updater import DEFAULT_PAGE_SIZE, _get_updates, _get_format
from updater import DEFAULT_PAGE_SIZE, get_updates, get_format
# AWS Lambda entry point
@@ -13,10 +13,10 @@ def handler(event, context):
# Detect item format
fmt = event.get('format', 'video')
quality = event.get('quality', 'high')
ytdl_fmt = _get_format(fmt, quality)
ytdl_fmt = get_format(fmt, quality)
print('Getting updates for %s (start=%d, count=%d, fmt: %s, last id: %s)' % (url, start, count, ytdl_fmt, last_id))
_, episodes, new_last_id = _get_updates(start, count, url, ytdl_fmt, last_id)
_, episodes, new_last_id = get_updates(start, count, url, ytdl_fmt, last_id)
return {
'last_id': new_last_id,
cmd/updater/requirements.txt
+1
View File
@@ -1,2 +1,3 @@
boto3==1.9.129
youtube_dl==2019.04.24
sanic==18.12
cmd/updater/server.py
+38
View File
@@ -0,0 +1,38 @@
import updater
from sanic import Sanic, response
from sanic.exceptions import InvalidUsage
app = Sanic()
@app.get('/update')
async def update(req):
url = req.args.get('url', None)
start = req.args.get('start', 1)
count = req.args.get('count', updater.DEFAULT_PAGE_SIZE)
# Last seen video ID
last_id = req.args.get('last_id', None)
# Detect item format
fmt = req.args.get('format', 'video')
quality = req.args.get('quality', 'high')
ytdl_fmt = updater.get_format(fmt, quality)
try:
_, episodes, new_last_id = updater.get_updates(start, count, url, ytdl_fmt, last_id)
return response.json({
'last_id': new_last_id,
'episodes': episodes,
})
except ValueError:
raise InvalidUsage()
@app.get('/ping')
async def ping(req):
return response.text('pong')
if __name__ == '__main__':
app.run(host='0.0.0.0', port=8080)
cmd/updater/updater.py
+2 -2
View File
@@ -5,7 +5,7 @@ BEST_FORMAT = "bestvideo+bestaudio/best"
DEFAULT_PAGE_SIZE = 50
def _get_format(fmt, quality):
def get_format(fmt, quality):
if fmt == 'video':
# Video
if quality == 'high':
@@ -20,7 +20,7 @@ def _get_format(fmt, quality):
return 'worstaudio'
def _get_updates(start, count, url, fmt, last_id=None):
def get_updates(start, count, url, fmt, last_id=None):
if start < 1:
raise ValueError('Invalid start value')
cmd/updater/updater_test.py
+8 -8
View File
@@ -7,30 +7,30 @@ TEST_URL = 'https://www.youtube.com/user/CNN/videos'
class TestUpdater(unittest.TestCase):
def test_get_updates(self):
kinds = [
updater._get_format('video', 'high'),
updater._get_format('video', 'low'),
updater._get_format('audio', 'high'),
updater._get_format('audio', 'low'),
updater.get_format('video', 'high'),
updater.get_format('video', 'low'),
updater.get_format('audio', 'high'),
updater.get_format('audio', 'low'),
]
for kind in kinds:
with self.subTest(kind):
feed, items, _ = updater._get_updates(1, 1, TEST_URL, kind)
feed, items, _ = updater.get_updates(1, 1, TEST_URL, kind)
self.assertIsNotNone(feed)
self.assertIsNotNone(items)
def test_get_change_list(self):
feed, items, last_id = updater._get_updates(1, 5, TEST_URL, 'worst[ext=mp4]')
feed, items, last_id = updater.get_updates(1, 5, TEST_URL, 'worst[ext=mp4]')
self.assertEqual(len(items), 5)
self.assertEqual(items[0]['ID'], last_id)
test_last_id = items[2]['ID']
self.assertIsNotNone(test_last_id)
feed, items, last_id = updater._get_updates(1, 5, TEST_URL, 'worst[ext=mp4]', test_last_id)
feed, items, last_id = updater.get_updates(1, 5, TEST_URL, 'worst[ext=mp4]', test_last_id)
self.assertEqual(len(items), 2)
self.assertEqual(items[0]['ID'], last_id)
def test_last_id(self):
feed, items, last_id = updater._get_updates(1, 1, TEST_URL, 'worstaudio')
feed, items, last_id = updater.get_updates(1, 1, TEST_URL, 'worstaudio')
self.assertEqual(len(items), 1)
self.assertEqual(items[0]['ID'], last_id)