From 0b13bd6be89cc75e3f9ea2b3b3791d2c40e41d1a Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Sun, 17 Mar 2019 00:16:23 -0700 Subject: [PATCH] Implement resolver lambda --- cmd/resolver/.gitignore | 5 ++ cmd/resolver/Makefile | 25 ++++++++ cmd/resolver/function.py | 117 ++++++++++++++++++++++++++++++++++ cmd/resolver/function_test.py | 27 ++++++++ cmd/resolver/requirements.txt | 2 + cmd/resolver/setup.cfg | 2 + 6 files changed, 178 insertions(+) create mode 100644 cmd/resolver/.gitignore create mode 100644 cmd/resolver/Makefile create mode 100644 cmd/resolver/function.py create mode 100644 cmd/resolver/function_test.py create mode 100644 cmd/resolver/requirements.txt create mode 100644 cmd/resolver/setup.cfg diff --git a/cmd/resolver/.gitignore b/cmd/resolver/.gitignore new file mode 100644 index 0000000..444ccb1 --- /dev/null +++ b/cmd/resolver/.gitignore @@ -0,0 +1,5 @@ +.idea/ +venv/ +package/ + +function.zip \ No newline at end of file diff --git a/cmd/resolver/Makefile b/cmd/resolver/Makefile new file mode 100644 index 0000000..34beab7 --- /dev/null +++ b/cmd/resolver/Makefile @@ -0,0 +1,25 @@ + +build: + pip3 install --requirement requirements.txt --target package + cd package && zip -r9 ../function.zip . + zip -g function.zip function.py + +deploy: build + aws --profile Podsync lambda create-function \ + --function-name Resolver \ + --role $(shell aws --profile Podsync iam get-role --role-name AWSLambdaBasicExecutionRole --query 'Role.Arn' --output text) \ + --runtime python3.7 \ + --handler function.handler \ + --zip-file fileb://function.zip \ + --timeout 10 \ + --memory-size 128 + +update: build + aws --profile Podsync lambda update-function-code \ + --function-name Resolver \ + --zip-file fileb://function.zip + +clean: + rm -rf package function.zip + +.PHONY: deploy update clean \ No newline at end of file diff --git a/cmd/resolver/function.py b/cmd/resolver/function.py new file mode 100644 index 0000000..29dc7b2 --- /dev/null +++ b/cmd/resolver/function.py @@ -0,0 +1,117 @@ +import os +import requests +import youtube_dl + +METADATA_URL = os.getenv('METADATA_URL', 'http://podsync.net/api/metadata/{feed_id}') +print('Using metadata URL template: ' + METADATA_URL) + + +class InvalidUsage(Exception): + pass + + +opts = { + 'quiet': True, + 'no_warnings': True, + 'forceurl': True, + 'simulate': True, + 'skip_download': True, + 'call_home': False, + 'nocheckcertificate': True +} + +url_formats = { + 'youtube': 'https://youtube.com/watch?v={}', + 'vimeo': 'https://vimeo.com/{}', +} + + +def handler(event, context): + feed_id = event['feed_id'] + video_id = event['video_id'] + + redirect_url = download(feed_id, video_id) + + return { + 'redirect_url': redirect_url, + } + + +def download(feed_id, video_id): + if not feed_id: + raise InvalidUsage('Invalid feed id') + + # Remove extension and check if video id is ok + video_id = os.path.splitext(video_id)[0] + if not video_id: + raise InvalidUsage('Invalid video id') + + # Pull metadata from API server + metadata_url = METADATA_URL.format(feed_id=feed_id, video_id=video_id) + r = requests.get(url=metadata_url) + json = r.json() + + # Build URL + provider = json['provider'] + tpl = url_formats[provider] + if not tpl: + raise InvalidUsage('Invalid feed') + url = tpl.format(video_id) + + redirect_url = _resolve(url, json) + return redirect_url + + +def _resolve(url, metadata): + if not url: + raise InvalidUsage('Invalid URL') + + print('Resolving %s' % url) + + try: + provider = metadata['provider'] + + with youtube_dl.YoutubeDL(opts) as ytdl: + info = ytdl.extract_info(url, download=False) + if provider == 'youtube': + return _yt_choose_url(info, metadata) + elif provider == 'vimeo': + return _vimeo_choose_url(info, metadata) + else: + raise ValueError('undefined provider') + except Exception as e: + print(e) + raise + + +def _yt_choose_url(info, metadata): + is_video = metadata['format'] == 'video' + + # Filter formats by file extension + ext = 'mp4' if is_video else 'm4a' + fmt_list = [x for x in info['formats'] if x['ext'] == ext and 'acodec' in x and x['acodec'] != 'none'] + if not len(fmt_list): + return info['url'] + + # Sort list by field (width for videos, file size for audio) + sort_field = 'width' if is_video else 'filesize' + # Sometime 'filesize' field can be None + if not all(x[sort_field] is not None for x in fmt_list): + sort_field = 'format_id' + ordered = sorted(fmt_list, key=lambda x: x[sort_field], reverse=True) + + # Choose an item depending on quality, better at the beginning + is_high_quality = metadata['quality'] == 'high' + item = ordered[0] if is_high_quality else ordered[-1] + return item['url'] + + +def _vimeo_choose_url(info, metadata): + # Query formats with 'extension' = mp4 and 'format_id' = http-1080p/http-720p/../http-360p + fmt_list = [x for x in info['formats'] if x['ext'] == 'mp4' and x['format_id'].startswith('http-')] + + ordered = sorted(fmt_list, key=lambda x: x['width'], reverse=True) + is_high_quality = metadata['quality'] == 'high' + item = ordered[0] if is_high_quality else ordered[-1] + + return item['url'] diff --git a/cmd/resolver/function_test.py b/cmd/resolver/function_test.py new file mode 100644 index 0000000..ee3e916 --- /dev/null +++ b/cmd/resolver/function_test.py @@ -0,0 +1,27 @@ +import function as ytdl +import unittest + + +class TestYtdl(unittest.TestCase): + def test_resolve(self): + self.assertIsNotNone( + ytdl._resolve('https://youtube.com/watch?v=ygIUF678y40', + {'format': 'video', 'quality': 'low', 'provider': 'youtube'})) + self.assertIsNotNone( + ytdl._resolve('https://youtube.com/watch?v=WyaEiO4hyik', + {'format': 'audio', 'quality': 'high', 'provider': 'youtube'})) + self.assertIsNotNone( + ytdl._resolve('https://youtube.com/watch?v=5mjUF2j9dgA', + {'format': 'video', 'quality': 'low', 'provider': 'youtube'}) + ) + self.assertIsNotNone( + ytdl._resolve('https://www.youtube.com/watch?v=2nH7xAMqD2g', + {'format': 'video', 'quality': 'high', 'provider': 'youtube'}) + ) + + def test_vimeo(self): + self.assertIsNotNone( + ytdl._resolve('https://vimeo.com/237715420', {'format': 'video', 'quality': 'low', 'provider': 'vimeo'})) + self.assertIsNotNone( + ytdl._resolve('https://vimeo.com/275211960', {'format': 'video', 'quality': 'high', 'provider': 'vimeo'}) + ) diff --git a/cmd/resolver/requirements.txt b/cmd/resolver/requirements.txt new file mode 100644 index 0000000..4eb4bad --- /dev/null +++ b/cmd/resolver/requirements.txt @@ -0,0 +1,2 @@ +requests==2.19.1 +youtube_dl==2019.03.09 \ No newline at end of file diff --git a/cmd/resolver/setup.cfg b/cmd/resolver/setup.cfg new file mode 100644 index 0000000..6302b3a --- /dev/null +++ b/cmd/resolver/setup.cfg @@ -0,0 +1,2 @@ +[install] +prefix= \ No newline at end of file