mirror/mxpv-podsync
1
0
Fork 0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00
Code Issues Releases Activity

Move ytdl to separate Docker container #7

Browse Source
This commit is contained in:
Maksym Pavlenko
2017-02-11 00:30:16 -08:00
parent 1865b2524b
commit f23b45a3d8
13 changed files with 120 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Podsync/Dockerfile
View File

@@ -3,12 +3,6 @@
ARG PUBLISH_DIR="bin/Publish"
WORKDIR /app
COPY $PUBLISH_DIR .
RUN apt-get update && \
apt-get install -y python-setuptools && \
easy_install pip && \
pip install youtube-dl
ENTRYPOINT ["dotnet", "Podsync.dll"]

2
src/Podsync/Services/PodsyncConfiguration.cs
View File

@@ -15,5 +15,7 @@ namespace Podsync.Services
public string PatreonSecret { get; set; }
public Tokens CreatorTokens { get; set; }
public string RemoteResolverUrl { get; set; }
}
}

31
src/Podsync/Services/Resolver/RemoteResolver.cs Normal file
View File

@@ -0,0 +1,31 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Podsync.Services.Storage;
namespace Podsync.Services.Resolver
{
public class RemoteResolver : CachedResolver
{
private readonly ILogger _logger;
private readonly HttpClient _client = new HttpClient();
public RemoteResolver(IStorageService storageService, IOptions<PodsyncConfiguration> options, ILogger<RemoteResolver> logger) : base(storageService)
{
_logger = logger;
_client.BaseAddress = new Uri(options.Value.RemoteResolverUrl);
_logger.LogInformation($"Remote resolver URL: {_client.BaseAddress}");
}
public override string Version { get; } = "Remote";
protected override async Task<Uri> ResolveInternal(Uri videoUrl, ResolveFormat format)
{
var response = await _client.GetStringAsync($"/resolve?url={videoUrl}&quality={format}");
return new Uri(response);
}
}
}

153
src/Podsync/Services/Resolver/YtdlWrapper.cs
View File

@@ -1,153 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Medallion.Shell;
using Microsoft.Extensions.Logging;
using Podsync.Services.Storage;
namespace Podsync.Services.Resolver
{
public class YtdlWrapper : CachedResolver
{
private static readonly TimeSpan ProcessWaitTimeout = TimeSpan.FromMinutes(1);
private static readonly TimeSpan WaitTimeoutBetweenFailedCalls = TimeSpan.FromSeconds(30);
private const string YtdlName = "youtube-dl";
private readonly ILogger _logger;
public YtdlWrapper(IStorageService storageService, ILogger<YtdlWrapper> logger) : base(storageService)
{
_logger = logger;
try
{
var cmd = Command.Run(YtdlName, "--version");
var version = cmd.Result.StandardOutput;
Version = version;
_logger.LogInformation("Uring youtube-dl {VERSION}", version);
}
catch (Exception ex)
{
throw new FileNotFoundException("Failed to execute youtube-dl executable", "youtube-dl", ex);
}
}
public override string Version { get; }
protected override async Task<Uri> ResolveInternal(Uri videoUrl, ResolveFormat format)
{
try
{
return await Ytdl(videoUrl, format);
}
catch (InvalidOperationException)
{
// Give a try one more time, often it helps
await Task.Delay(WaitTimeoutBetweenFailedCalls);
return await Ytdl(videoUrl, format);
}
}
private static IEnumerable<string> GetArguments(Uri videoUrl, ResolveFormat format)
{
var host = videoUrl.Host.ToLowerInvariant();
// Video format code, see the "FORMAT SELECTION"
yield return "-f";
if (host.Contains("youtube.com"))
{
if (format == ResolveFormat.VideoHigh)
{
yield return "best[ext=mp4]";
}
else if (format == ResolveFormat.VideoLow)
{
yield return "worst[ext=mp4]";
}
else if (format == ResolveFormat.AudioHigh)
{
yield return "bestaudio[ext=m4a]/worstaudio[ext=m4a]";
}
else if (format == ResolveFormat.AudioLow)
{
yield return "worstaudio[ext=m4a]/bestaudio[ext=m4a]";
}
else
{
throw new ArgumentException("Unsupported resolve format");
}
}
else if (host.Contains("vimeo.com"))
{
if (format == ResolveFormat.VideoHigh)
{
yield return "Original/http-1080p/http-720p/http-360p/http-270p";
}
else if (format == ResolveFormat.VideoLow)
{
yield return "http-270p/http-360p/http-540p/http-720p/http-1080p";
}
else
{
throw new ArgumentException("Unsupported resolve format");
}
}
else
{
throw new ArgumentException("Unsupported video provider");
}
// Simulate, quiet but print URL
yield return "-g";
// Do not download the video and do not write anything to disk
yield return "-s";
// Suppress HTTPS certificate validation
yield return "--no-check-certificate";
// Do NOT contact the youtube-dl server for debugging
yield return "--no-call-home";
yield return videoUrl.ToString();
}
private async Task<Uri> Ytdl(Uri videoUrl, ResolveFormat format)
{
var cmd = Command.Run(YtdlName, GetArguments(videoUrl, format), opts => opts.ThrowOnError().Timeout(ProcessWaitTimeout));
try
{
await cmd.Task;
}
catch (ErrorExitCodeException ex)
{
var errout = await cmd.StandardError.ReadToEndAsync();
var msg = !string.IsNullOrWhiteSpace(errout) ? errout : ex.Message;
_logger.LogError(Constants.Events.YtdlError, ex, "Failed to resolve {URL} in format {FORMAT}", videoUrl, format);
if (string.Equals(errout, "ERROR: requested format not available"))
{
throw new NotSupportedException("Requested format not available", ex);
}
throw new InvalidOperationException(msg, ex);
}
var stdout = await cmd.StandardOutput.ReadToEndAsync();
if (Uri.IsWellFormedUriString(stdout, UriKind.Absolute))
{
return new Uri(stdout);
}
throw new InvalidOperationException(stdout);
}
}
}

2
src/Podsync/Startup.cs
View File

@@ -53,7 +53,7 @@ namespace Podsync
services.AddSingleton<ILinkService, LinkService>();
services.AddSingleton<IYouTubeClient, YouTubeClient>();
services.AddSingleton<IVimeoClient, VimeoClient>();
services.AddSingleton<IResolverService, YtdlWrapper>();
services.AddSingleton<IResolverService, RemoteResolver>();
services.AddSingleton<IStorageService, RedisStorage>();
services.AddSingleton<IRssBuilder, CompositeRssBuilder>();
services.AddSingleton<IPatreonApi, PatreonApi>();

3
src/Podsync/appsettings.json
View File

@@ -13,6 +13,7 @@
"RedisConnectionString": "localhost",
"BaseUrl": "",
"PatreonClientId": "",
"PatreonSecret": ""
"PatreonSecret": "",
"RemoteResolverUrl": "http://localhost:8080"
}
}

13
src/Podsync/docker-compose.yml → src/docker-compose.yml
View File

@@ -2,9 +2,9 @@
services:
app:
image: podsync
image: podsync/app
build:
context: .
context: ./Podsync
dockerfile: Dockerfile
restart: always
ports:
@@ -12,8 +12,17 @@ services:
environment:
- ASPNETCORE_URLS=http://*:5001
- Podsync:RedisConnectionString=redis
- Podsync:RemoteResolverUrl=http://ytdl:8080
env_file:
- ~/podsync.env
ytdl:
image: podsync/ytdl
build:
context: ./ytdl
dockerfile: Dockerfile
restart: always
ports:
- 8080
redis:
image: redis
command: redis-server --appendonly yes

0
src/Podsync/nginx/Dockerfile → src/nginx/Dockerfile
View File

0
src/Podsync/nginx/nginx.conf → src/nginx/nginx.conf
View File

10
src/ytdl/Dockerfile Normal file
View File

@@ -0,0 +1,10 @@
FROM python:3.6.0-alpine
WORKDIR /app
COPY . .
RUN apk add --update --no-cache gcc g++ make && \
pip install --no-cache-dir --requirement /app/requirements.txt
EXPOSE 8080
ENTRYPOINT ["python", "/app/ytdl.py"]

2
src/ytdl/requirements.txt Normal file
View File

@@ -0,0 +1,2 @@
youtube_dl
sanic

61
src/ytdl/ytdl.py Normal file
View File

@@ -0,0 +1,61 @@
import youtube_dl
from sanic import Sanic
from sanic.exceptions import InvalidUsage
from sanic.response import text
app = Sanic()
default_opts = {
'quiet': True,
'no_warnings': True,
'forceurl': True,
'simulate': True,
'skip_download': True,
'call_home': False,
'nocheckcertificate': True
}
youtube_quality = {
'VideoHigh': 'best[ext=mp4]',
'VideoLow': 'worst[ext=mp4]',
'AudioHigh': 'bestaudio[ext=m4a]/worstaudio[ext=m4a]',
'AudioLow': 'worstaudio[ext=m4a]/bestaudio[ext=m4a]'
}
vimeo_quality = {
'VideoHigh': 'Original/http-1080p/http-720p/http-360p/http-270p',
'VideoLow': 'http-270p/http-360p/http-540p/http-720p/http-1080p'
}
@app.route('/resolve')
async def youtube(request):
url = request.args.get('url')
if not url:
raise InvalidUsage('Invalid URL')
opts = default_opts.copy()
quality = request.args.get('quality', 'VideoHigh')
fmt = _choose_format(quality, url)
if fmt:
opts.update(format=fmt)
with youtube_dl.YoutubeDL(opts) as ytdl:
info = ytdl.extract_info(url, download=False)
return text(info['url'])
def _choose_format(quality, url):
fmt = None
if 'youtube.com' in url:
fmt = youtube_quality.get(quality)
elif 'vimeo.com' in url:
fmt = vimeo_quality.get(quality)
return fmt
if __name__ == '__main__':
app.run(host='0.0.0.0', port=8080)

80
test/Podsync.Tests/Services/Resolver/YtdlWrapperTests.cs
View File

@@ -1,80 +0,0 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Moq;
using Podsync.Services;
using Podsync.Services.Resolver;
using Podsync.Services.Storage;
using Xunit;
namespace Podsync.Tests.Services.Resolver
{
public class YtdlWrapperTests : TestBase
{
private readonly Mock<ILogger<YtdlWrapper>> _logger = new Mock<ILogger<YtdlWrapper>>();
private readonly Mock<IStorageService> _storage = new Mock<IStorageService>();
private readonly IResolverService _resolver;
public YtdlWrapperTests()
{
_storage.Setup(x => x.GetCached(It.IsAny<string>(), It.IsAny<string>())).ReturnsAsync("");
_resolver = new YtdlWrapper(_storage.Object, _logger.Object);
}
[Theory]
[InlineData("https://www.youtube.com/watch?v=BaW_jenozKc")]
public async Task ResolveTest(string url)
{
_storage.ResetCalls();
var videoUrl = new Uri(url);
var downloadUrl = await _resolver.Resolve(videoUrl);
_storage.Verify(x => x.GetCached(Constants.Cache.VideosPrefix, videoUrl.GetHashCode().ToString()), Times.Once);
_storage.Verify(x => x.Cache(Constants.Cache.VideosPrefix, videoUrl.GetHashCode().ToString(), It.IsAny<string>(), It.IsAny<TimeSpan>()), Times.Once);
Assert.NotEqual(downloadUrl, videoUrl);
Assert.True(downloadUrl.IsAbsoluteUri);
}
[Theory]
[InlineData("https://www.youtube.com/watch?v=fiWMUkOgY9I")]
public async Task FailTest(string url)
{
var ex = await Assert.ThrowsAsync<InvalidOperationException>(async () => await _resolver.Resolve(new Uri(url)));
Assert.NotEmpty(ex.Message);
}
[Fact]
public void VersionTest()
{
Assert.NotNull(_resolver.Version);
}
[Fact]
public async Task ResolveOutputTest()
{
var downloadUrl = await _resolver.Resolve(new Uri("https://www.youtube.com/watch?v=-csRxRj_zcw&t=45s"), ResolveFormat.AudioHigh);
Assert.True(downloadUrl.IsAbsoluteUri);
}
[Theory]
[InlineData("https://vimeo.com/94747106", ResolveFormat.VideoHigh)]
[InlineData("https://vimeo.com/199203302", ResolveFormat.VideoHigh)]
[InlineData("https://vimeo.com/93003441", ResolveFormat.VideoHigh)]
[InlineData("https://vimeo.com/93003441", ResolveFormat.VideoLow)]
public async Task ResolveVimeoLinks(string link, ResolveFormat format)
{
var downloadUrl = await _resolver.Resolve(new Uri(link), format);
Assert.True(downloadUrl.IsWellFormedOriginalString());
}
[Fact]
public async Task VimeoAudioExceptionTest()
{
await Assert.ThrowsAsync<ArgumentException>(async () => await _resolver.Resolve(new Uri("https://vimeo.com/94747106"), ResolveFormat.AudioHigh));
await Assert.ThrowsAsync<ArgumentException>(async () => await _resolver.Resolve(new Uri("https://vimeo.com/94747106"), ResolveFormat.AudioLow));
}
}
}