Handle downloads in ytdl

docker-compose.yml

@@ -2,11 +2,8 @@
 
 services:
   app:
-    image: podsync/app
+    image: gcr.io/pod-sync/app:latest
     container_name: app
-    build:
-      context: ./src/Podsync
-      dockerfile: Dockerfile
     restart: always
     ports:
       - 5001
@@ -14,30 +11,31 @@ services:
       - ASPNETCORE_URLS=http://*:5001
       - ASPNETCORE_ENVIRONMENT=Production
       - Podsync:RedisConnectionString=redis
-      - Podsync:RemoteResolverUrl=http://ytdl:8080
+      - Podsync:RemoteResolverUrl=http://ytdl:5002
-    env_file:
+      - Podsync:YouTubeApiKey={YOUTUBE_API_KEY}
-      - ~/podsync.env
+      - Podsync:VimeoApiKey={VIMEO_API_KEY}
+      - Podsync:PatreonClientId={PATREON_CLIENT_ID}
+      - Podsync:PatreonSecret={PATREON_SECRET}
+      - Logging:LogLevel:Default=Information
+      - Logging:LogLevel:Microsoft=Warning
   ytdl:
-    image: podsync/ytdl
+    image: gcr.io/pod-sync/ytdl:latest
     container_name: ytdl
-    build:
-      context: ./src/ytdl
-      dockerfile: Dockerfile
     restart: always
     ports:
-      - 8080
+      - 5002
+    environment:
+      - REDIS_CONNECTION_STRING=redis://redis:6379/0
   redis:
     image: redis
     container_name: redis
-    command: redis-server --appendonly yes
+    command: redis-server --appendonly no --save 900 1 --save 300 10 --save 60 10000
     restart: always
     volumes:
       - /data/redis:/data
   nginx:
+    image: gcr.io/pod-sync/nginx:latest
     container_name: nginx
-    build:
-      context: ./src/nginx
-      dockerfile: Dockerfile
     restart: always
     ports:
-      - 80:80
+      - 80:80

src/nginx/nginx.conf

@@ -4,11 +4,15 @@ server {
     gzip                    on;
     server_tokens           off;
 
+    location /download {
+        proxy_pass          http://ytdl:5002;
+    }
+
     location / {
         proxy_pass          http://app:5001;
         proxy_http_version  1.1;
         proxy_set_header    Upgrade         $http_upgrade;
-        proxy_set_header    Connection      "";
+        proxy_set_header    Connection      keep-alive;
         proxy_set_header    Host            $host;
         proxy_set_header    X-Real-IP       $remote_addr;
         proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;

src/ytdl/Dockerfile

@@ -3,8 +3,10 @@ FROM python:3.6.0-alpine
 WORKDIR /app
 COPY . .
 
+ENV REDIS_CONNECTION_STRING localhost
+
 RUN apk add --update --no-cache gcc g++ make && \
     pip install --no-cache-dir --requirement /app/requirements.txt
 
-EXPOSE 8080
+EXPOSE 5002
 ENTRYPOINT ["python", "/app/ytdl.py"]

src/ytdl/requirements.txt

@@ -1,2 +1,3 @@
 youtube_dl
 sanic
+redis

src/ytdl/ytdl.py

@@ -1,11 +1,17 @@
 import youtube_dl
+import redis
+import os
 from youtube_dl.utils import DownloadError
 from sanic import Sanic
-from sanic.exceptions import InvalidUsage
+from sanic.exceptions import InvalidUsage, NotFound
-from sanic.response import text
+from sanic.response import text, redirect
+from datetime import timedelta
 
 app = Sanic()
 
+db = redis.from_url(os.getenv('REDIS_CONNECTION_STRING', 'localhost'))
+db.ping()
+
 default_opts = {
     'quiet': True,
     'no_warnings': True,
@@ -28,16 +34,55 @@ vimeo_quality = {
     'VideoLow': 'http-270p/http-360p/http-540p/http-720p/http-1080p'
 }
 
+url_formats = {
+    b'YouTube': 'https://youtube.com/watch?v={}',
+    b'Vimeo': 'https://vimeo.com/{}',
+}
 
-@app.route('/resolve')
+
-async def youtube(request):
+@app.route('/download/<feed_id>/<video_id>', methods=['GET'])
-    url = request.args.get('url')
+async def download(request, feed_id, video_id):
+    if not feed_id:
+        raise InvalidUsage('Invalid feed id')
+
+    # Remote extension and check if video id is ok
+    video_id = os.path.splitext(video_id)[0]
+    if not video_id:
+        raise InvalidUsage('Invalid video id')
+
+    # Query redis
+    entries = db.hgetall(feed_id)
+    if not entries:
+        raise NotFound('Feed not found')
+
+    # Delete this feed if no requests within 90 days
+    db.expire(feed_id, timedelta(days=90))
+
+    # Build URL
+    provider = entries.get(b'Provider') or entries[b'provider']  # HACK: we have to keep backward compatibility :(
+    tpl = url_formats[provider]
+    if not tpl:
+        raise InvalidUsage('Invalid feed')
+
+    url = tpl.format(video_id)
+    quality = entries.get('quality', '')
+
+    try:
+        redirect_url = _resolve(url, quality)
+        return redirect(redirect_url)
+    except DownloadError as e:
+        msg = str(e)
+        return text(msg, status=511)
+
+
+def _resolve(url, quality):
     if not url:
         raise InvalidUsage('Invalid URL')
 
-    opts = default_opts.copy()
+    if not quality:
+        quality = 'VideoHigh'
 
-    quality = request.args.get('quality', 'VideoHigh')
+    opts = default_opts.copy()
     fmt = _choose_format(quality, url)
 
     if fmt:
@@ -46,10 +91,12 @@ async def youtube(request):
     try:
         with youtube_dl.YoutubeDL(opts) as ytdl:
             info = ytdl.extract_info(url, download=False)
-            return text(info['url'])
+            return info['url']
-    except DownloadError as e:
+    except DownloadError:
-        msg = str(e)
+        raise
-        return text(msg, status=511)
+    except Exception as e:
+        print(e)
+        raise
 
 
 def _choose_format(quality, url):
@@ -63,4 +110,4 @@ def _choose_format(quality, url):
 
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=8080)
+    app.run(host='0.0.0.0', port=5002)