Implement RSS feed builder

This commit is contained in:
Maksym Pavlenko
2016-12-11 21:16:03 -08:00
parent 7b746307f7
commit df451d9b5e
23 changed files with 442 additions and 49 deletions
+1
View File
@@ -21,6 +21,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Shared", "Shared", "{35F181
src\Shared\EnumerableExtensions.cs = src\Shared\EnumerableExtensions.cs
src\Shared\ListExtensions.cs = src\Shared\ListExtensions.cs
src\Shared\StringExtensions.cs = src\Shared\StringExtensions.cs
src\Shared\ServiceProviderExtensions.cs = src\Shared\ServiceProviderExtensions.cs
EndProjectSection
EndProject
Global
+13 -7
View File
@@ -2,6 +2,7 @@
using Microsoft.AspNetCore.Mvc;
using Podsync.Services.Links;
using Podsync.Services.Resolver;
using Podsync.Services.Storage;
namespace Podsync.Controllers
{
@@ -10,24 +11,29 @@ namespace Podsync.Controllers
{
private readonly IResolverService _resolverService;
private readonly ILinkService _linkService;
private readonly IStorageService _storageService;
public DownloadController(IResolverService resolverService, ILinkService linkService)
public DownloadController(IResolverService resolverService, ILinkService linkService, IStorageService storageService)
{
_resolverService = resolverService;
_linkService = linkService;
_storageService = storageService;
}
[Route("{provider}/{videoId}.mp4")]
public async Task<IActionResult> Download(Provider provider, string videoId)
// Main video download endpoint, don't forget to reflect any changes in LinkService.Download
[Route("{feedId}/{videoId}/")]
public async Task<IActionResult> Download(string feedId, string videoId)
{
var metadata = await _storageService.Load(feedId);
var url = _linkService.Make(new LinkInfo
{
Provider = provider,
LinkType = LinkType.Video,
Provider = metadata.Provider,
LinkType = metadata.LinkType,
Id = videoId
});
var redirectUrl = await _resolverService.Resolve(url);
var redirectUrl = await _resolverService.Resolve(url, metadata.Quality);
return Redirect(redirectUrl.ToString());
}
+38
View File
@@ -0,0 +1,38 @@
using System.IO;
using System.Threading.Tasks;
using System.Xml.Serialization;
using Microsoft.AspNetCore.Mvc;
using Podsync.Services.Builder;
using Podsync.Services.Feed;
namespace Podsync.Controllers
{
[Route("feed")]
public class FeedController : Controller
{
private readonly XmlSerializer _serializer = new XmlSerializer(typeof(Rss));
private readonly IRssBuilder _rssBuilder;
public FeedController(IRssBuilder rssBuilder)
{
_rssBuilder = rssBuilder;
}
[Route("{feedId}")]
public async Task<IActionResult> Index(string feedId)
{
var rss = await _rssBuilder.Query(feedId);
// Serialize feed to string
string body;
using (var writer = new StringWriter())
{
_serializer.Serialize(writer, rss);
body = writer.ToString();
}
return Content(body, "text/xml");
}
}
}
@@ -0,0 +1,34 @@
using System;
using System.Threading.Tasks;
using Podsync.Services.Feed;
using Podsync.Services.Links;
using Podsync.Services.Storage;
using Shared;
namespace Podsync.Services.Builder
{
public class CompositeRssBuilder : RssBuilderBase
{
private readonly YouTubeRssBuilder _youTubeBuilder;
public CompositeRssBuilder(IServiceProvider serviceProvider, IStorageService storageService) : base(storageService)
{
_youTubeBuilder = serviceProvider.CreateInstance<YouTubeRssBuilder>();
}
public override Provider Provider
{
get { throw new NotSupportedException(); }
}
public override Task<Rss> Query(FeedMetadata feed)
{
if (feed.Provider == Provider.YouTube)
{
return _youTubeBuilder.Query(feed);
}
throw new NotSupportedException("Not supported provider");
}
}
}
@@ -0,0 +1,13 @@
using System.Threading.Tasks;
using Podsync.Services.Feed;
using Podsync.Services.Storage;
namespace Podsync.Services.Builder
{
public interface IRssBuilder
{
Task<Rss> Query(string feedId);
Task<Rss> Query(FeedMetadata feed);
}
}
@@ -0,0 +1,28 @@
using System.Threading.Tasks;
using Podsync.Services.Feed;
using Podsync.Services.Links;
using Podsync.Services.Storage;
namespace Podsync.Services.Builder
{
public abstract class RssBuilderBase : IRssBuilder
{
private readonly IStorageService _storageService;
protected RssBuilderBase(IStorageService storageService)
{
_storageService = storageService;
}
public abstract Provider Provider { get; }
public async Task<Rss> Query(string feedId)
{
var metadata = await _storageService.Load(feedId);
return await Query(metadata);
}
public abstract Task<Rss> Query(FeedMetadata metadata);
}
}
@@ -0,0 +1,131 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Podsync.Services.Feed;
using Podsync.Services.Links;
using Podsync.Services.Resolver;
using Podsync.Services.Storage;
using Podsync.Services.Videos.YouTube;
using Channel = Podsync.Services.Feed.Channel;
using Video = Podsync.Services.Videos.YouTube.Video;
namespace Podsync.Services.Builder
{
public class YouTubeRssBuilder : RssBuilderBase
{
private readonly ILinkService _linkService;
private readonly IYouTubeClient _youTube;
public YouTubeRssBuilder(ILinkService linkService, IYouTubeClient youTube, IStorageService storageService) : base(storageService)
{
_linkService = linkService;
_youTube = youTube;
}
public override Provider Provider { get; } = Provider.YouTube;
public override async Task<Rss> Query(FeedMetadata metadata)
{
if (metadata.Provider != Provider.YouTube)
{
throw new ArgumentException("Invalid provider");
}
var linkType = metadata.LinkType;
Channel channel;
if (linkType == LinkType.Channel)
{
channel = await GetChannel(new ChannelQuery { ChannelId = metadata.Id });
}
else if (linkType == LinkType.User)
{
channel = await GetChannel(new ChannelQuery { Username = metadata.Id });
}
else if (linkType == LinkType.Playlist)
{
channel = await GetPlaylist(metadata.Id);
}
else
{
throw new NotSupportedException("URL type is not supported");
}
// Get video ids from this playlist
var ids = await _youTube.GetPlaylistItemIds(new PlaylistItemsQuery { PlaylistId = channel.Guid });
// Get video descriptions
var videos = await _youTube.GetVideos(new VideoQuery { Id = string.Join(",", ids) });
channel.Items = videos.Select(x => MakeItem(x, metadata));
var rss = new Rss
{
Channels = new[] { channel }
};
return rss;
}
private async Task<Channel> GetChannel(ChannelQuery query)
{
var list = await _youTube.GetChannels(query);
var item = list.Single();
var channel = MakeChannel(item);
channel.Guid = item.PlaylistId;
return channel;
}
private async Task<Channel> GetPlaylist(string playlistId)
{
var list = await _youTube.GetPlaylists(new PlaylistQuery { PlaylistId = playlistId });
var item = list.Single();
var channel = MakeChannel(item);
channel.Guid = item.PlaylistId;
return channel;
}
private static Channel MakeChannel(YouTubeItem item)
{
return new Channel
{
Title = item.Title,
Description = item.Description,
Link = item.Link,
LastBuildDate = DateTime.Now,
PubDate = item.PublishedAt,
Image = item.Thumbnail,
Thumbnail = item.Thumbnail
};
}
private Item MakeItem(Video video, FeedMetadata feed)
{
return new Item
{
Title = video.Title,
Description = video.Description,
PubDate = video.PublishedAt,
Link = video.Link,
Duration = video.Duration,
Content = new MediaContent
{
Length = video.Size,
MediaType = SelectMediaType(feed.Quality),
Url = _linkService.Download(feed.Id, video.VideoId)
}
};
}
private static string SelectMediaType(ResolveType resolveType)
{
return resolveType == ResolveType.VideoHigh || resolveType == ResolveType.VideoLow
? "video/mp4"
: "audio/webm";
}
}
}
+8 -6
View File
@@ -17,11 +17,13 @@ namespace Podsync.Services.Feed
Items = Enumerable.Empty<Item>();
}
public string Guid { get; set; }
public string Title { get; set; }
public string Description { get; set; }
public string Link { get; set; }
public Uri Link { get; set; }
public DateTime LastBuildDate { get; set; }
@@ -33,9 +35,9 @@ namespace Podsync.Services.Feed
public string Category { get; set; }
public string Image { get; set; }
public Uri Image { get; set; }
public string Thumbnail { get; set; }
public Uri Thumbnail { get; set; }
public IEnumerable<Item> Items { get; set; }
@@ -53,7 +55,7 @@ namespace Podsync.Services.Feed
{
writer.WriteElementString("title", Title);
writer.WriteElementString("description", Description);
writer.WriteElementString("link", Link);
writer.WriteElementString("link", Link.ToString());
writer.WriteElementString("generator", PodsyncGeneratorName);
@@ -81,7 +83,7 @@ namespace Podsync.Services.Feed
*/
writer.WriteStartElement("image", Namespaces.Itunes);
writer.WriteAttributeString("href", Image);
writer.WriteAttributeString("href", Image.ToString());
writer.WriteEndElement();
/*
@@ -89,7 +91,7 @@ namespace Podsync.Services.Feed
*/
writer.WriteStartElement("thumbnail", Namespaces.Media);
writer.WriteAttributeString("url", Thumbnail);
writer.WriteAttributeString("url", Thumbnail.ToString());
writer.WriteEndElement();
// Items
+6 -7
View File
@@ -1,5 +1,4 @@
using System;
using System;
using System.Xml;
using System.Xml.Schema;
using System.Xml.Serialization;
@@ -14,7 +13,7 @@ namespace Podsync.Services.Feed
public string Author { get; set; }
public string Link { get; set; }
public Uri Link { get; set; }
public DateTime PubDate { get; set; }
@@ -38,7 +37,7 @@ namespace Podsync.Services.Feed
{
writer.WriteElementString("title", Title);
writer.WriteElementString("description", Description);
writer.WriteElementString("link", Link);
writer.WriteElementString("link", Link.ToString());
writer.WriteElementString("pubDate", PubDate.ToString("R"));
writer.WriteElementString("author", Author);
@@ -48,7 +47,7 @@ namespace Podsync.Services.Feed
writer.WriteStartElement("guid");
writer.WriteAttributeString("isPermaLink", "true");
writer.WriteString(Link);
writer.WriteString(Link.ToString());
writer.WriteEndElement();
/*
@@ -56,7 +55,7 @@ namespace Podsync.Services.Feed
*/
writer.WriteStartElement("enclosure");
writer.WriteAttributeString("url", Content.Url);
writer.WriteAttributeString("url", Content.Url.ToString());
writer.WriteAttributeString("length", Content.Length.ToString());
writer.WriteAttributeString("type", Content.MediaType);
writer.WriteEndElement();
@@ -66,7 +65,7 @@ namespace Podsync.Services.Feed
*/
writer.WriteStartElement("content", Namespaces.Media);
writer.WriteAttributeString("url", Content.Url);
writer.WriteAttributeString("url", Content.Url.ToString());
writer.WriteAttributeString("fileSize", Content.Length.ToString());
writer.WriteAttributeString("type", Content.MediaType);
writer.WriteEndElement();
+4 -2
View File
@@ -1,8 +1,10 @@
namespace Podsync.Services.Feed
using System;
namespace Podsync.Services.Feed
{
public struct MediaContent
{
public string Url { get; set; }
public Uri Url { get; set; }
public long Length { get; set; }
@@ -7,5 +7,7 @@ namespace Podsync.Services.Links
LinkInfo Parse(Uri link);
Uri Make(LinkInfo info);
Uri Download(string feedId, string videoId);
}
}
+13
View File
@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.WebUtilities;
using Microsoft.Extensions.Options;
using Shared;
namespace Podsync.Services.Links
@@ -28,6 +29,13 @@ namespace Podsync.Services.Links
}
};
private readonly Uri _baseUrl;
public LinkService(IOptions<PodsyncConfiguration> configuration)
{
_baseUrl = new Uri(configuration.Value.BaseUrl ?? "http://localhost");
}
public LinkInfo Parse(Uri link)
{
if (link == null)
@@ -193,5 +201,10 @@ namespace Podsync.Services.Links
throw new ArgumentException("Unsupported provider or link type", nameof(info), ex);
}
}
public Uri Download(string feedId, string videoId)
{
return new Uri(_baseUrl, $"download/{feedId}/{videoId}/");
}
}
}
@@ -5,5 +5,7 @@
public string YouTubeApiKey { get; set; }
public string RedisConnectionString { get; set; }
public string BaseUrl { get; set; }
}
}
+4 -4
View File
@@ -78,13 +78,13 @@ namespace Podsync.Services.Resolver
switch (resolveType)
{
case ResolveType.VideoHigh:
return "best";
return "best[ext=mp4]/low[ext=mp4]";
case ResolveType.VideoLow:
return "worst";
return "low[ext=mp4]/best[ext=mp4]";
case ResolveType.AudioHigh:
return "bestaudio";
return "bestaudio[ext=webm]/worstaudio[ext=webm]";
case ResolveType.AudioLow:
return "worstaudio";
return "worstaudio[ext=webm]/bestaudio[ext=webm]";
default:
throw new ArgumentOutOfRangeException(nameof(resolveType), "Unsupported format", null);
}
@@ -14,5 +14,7 @@ namespace Podsync.Services.Storage
public ResolveType Quality { get; set; }
public int PageSize { get; set; }
public override string ToString() => $"{Provider} ({LinkType}) {Id}";
}
}
@@ -68,14 +68,25 @@ namespace Podsync.Services.Videos.YouTube
public async Task<IEnumerable<Video>> GetPlaylistItems(PlaylistItemsQuery query)
{
var response = await QueryPlaylistItems(query);
var request = _youtube.PlaylistItems.List("id,snippet");
request.MaxResults = query.Count ?? MaxResults;
request.Id = query.Id;
request.PlaylistId = query.PlaylistId;
request.VideoId = query.VideoId;
var response = await request.ExecuteAsync();
return response.Items.Select(ConvertPlaylistItem);
}
public async Task<IEnumerable<string>> GetPlaylistItemIds(PlaylistItemsQuery query)
{
var response = await QueryPlaylistItems(query);
var request = _youtube.PlaylistItems.List("id,snippet");
request.MaxResults = query.Count ?? MaxResults;
request.PlaylistId = query.PlaylistId;
var response = await request.ExecuteAsync();
return response.Items.Select(x => x.Snippet.ResourceId.VideoId);
}
@@ -98,18 +109,6 @@ namespace Podsync.Services.Videos.YouTube
return totalSeconds * (definition == "hd" ? hdBytesPerSecond : ldBytesPerSecond);
}
private Task<PlaylistItemListResponse> QueryPlaylistItems(PlaylistItemsQuery query)
{
var request = _youtube.PlaylistItems.List("id,snippet");
request.MaxResults = query.Count ?? MaxResults;
request.Id = query.Id;
request.PlaylistId = query.PlaylistId;
request.VideoId = query.VideoId;
return request.ExecuteAsync();
}
private Video ConvertVideo(Google.Apis.YouTube.v3.Data.Video item)
{
var snippet = item.Snippet;
+2
View File
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Podsync.Services;
using Podsync.Services.Builder;
using Podsync.Services.Links;
using Podsync.Services.Resolver;
using Podsync.Services.Storage;
@@ -41,6 +42,7 @@ namespace Podsync
services.AddSingleton<IYouTubeClient, YouTubeClient>();
services.AddSingleton<IResolverService, YtdlWrapper>();
services.AddSingleton<IStorageService, RedisStorage>();
services.AddSingleton<IRssBuilder, CompositeRssBuilder>();
// Add framework services.
services.AddMvc();
+23
View File
@@ -0,0 +1,23 @@
using System;
using Microsoft.Extensions.DependencyInjection;
namespace Shared
{
internal static class ServiceProviderExtensions
{
public static T CreateInstance<T>(this IServiceProvider serviceProvider)
{
return serviceProvider.CreateInstance<T>(typeof(T));
}
public static T CreateInstance<T>(this IServiceProvider serviceProvider, Type implementation)
{
return (T)serviceProvider.CreateInstance(implementation);
}
public static object CreateInstance(this IServiceProvider serviceProvider, Type type)
{
return ActivatorUtilities.CreateInstance(serviceProvider, type);
}
}
}
@@ -0,0 +1,69 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Moq;
using Podsync.Services.Builder;
using Podsync.Services.Links;
using Podsync.Services.Storage;
using Podsync.Services.Videos.YouTube;
using Xunit;
namespace Podsync.Tests.Services.Builder
{
public class YouTubeRssBuilderTests : TestBase
{
private readonly Mock<IStorageService> _storageService = new Mock<IStorageService>();
private readonly YouTubeRssBuilder _builder;
public YouTubeRssBuilderTests()
{
var linkService = new LinkService(Options);
var client = new YouTubeClient(linkService, Options);
_builder = new YouTubeRssBuilder(linkService, client, _storageService.Object);
}
[Theory]
[InlineData(LinkType.Channel, "UC0JB7TSe49lg56u6qH8y_MQ")]
[InlineData(LinkType.User, "fxigr1")]
[InlineData(LinkType.Playlist, "PL2e4mYbwSTbbiX2uwspn0xiYb8_P_cTAr")]
public async Task BuildRssTest(LinkType linkType, string id)
{
var feed = new FeedMetadata
{
Provider = Provider.YouTube,
LinkType = linkType,
Id = id
};
var feedId = DateTime.UtcNow.Ticks.ToString();
_storageService.Setup(x => x.Load(feedId)).ReturnsAsync(feed);
var rss = await _builder.Query(feedId);
Assert.NotEmpty(rss.Channels);
var channel = rss.Channels.Single();
Assert.NotNull(channel.Title);
Assert.NotNull(channel.Description);
Assert.NotNull(channel.Image);
Assert.NotNull(channel.Guid);
Assert.NotEmpty(channel.Items);
foreach (var item in channel.Items)
{
Assert.NotNull(item.Title);
Assert.NotNull(item.Link);
Assert.True(item.Duration.TotalSeconds > 0);
Assert.NotNull(item.Content);
Assert.True(item.Content.Length > 0);
Assert.NotNull(item.Content.MediaType);
Assert.NotNull(item.Content.Url);
Assert.NotNull(item.PubDate);
}
}
}
}
@@ -8,7 +8,7 @@ namespace Podsync.Tests.Services.Feed
{
public class FeedSerializationTests
{
private const string ImageUrl = "https://yt3.ggpht.com/-OOqFwHRjeMQ/AAAAAAAAAAI/AAAAAAAAAAA/0XQZ2NeGp_0/s88-c-k-no-mo-rj-c0xffffff/photo.jpg";
private static readonly Uri ImageUrl = new Uri("https://yt3.ggpht.com/-OOqFwHRjeMQ/AAAAAAAAAAI/AAAAAAAAAAA/0XQZ2NeGp_0/s88-c-k-no-mo-rj-c0xffffff/photo.jpg");
[Fact]
public void SerializeFeedTest()
@@ -18,11 +18,11 @@ namespace Podsync.Tests.Services.Feed
var item = new Item
{
Title = "Steve Gillespie - Getting Arrested (Stand up Comedy)",
Link = "https://youtube.com/watch?v=Jj22gfTnpAI",
Link = new Uri("https://youtube.com/watch?v=Jj22gfTnpAI"),
PubDate = DateTime.Parse("Mon, 07 Nov 2016 20:02:26 GMT"),
Content = new MediaContent
{
Url = "http://podsync.net/download/youtube/Jj22gfTnpAI.mp4",
Url = new Uri("http://podsync.net/download/youtube/Jj22gfTnpAI.mp4"),
Length = 52850000,
MediaType = "video/mp4"
},
@@ -33,7 +33,7 @@ namespace Podsync.Tests.Services.Feed
{
Title = "Laugh Factory",
Description = "The best stand up comedy clips online. That's it.",
Link = "https://youtube.com/channel/UCxyCzPY2pjAjrxoSYclpuLg",
Link = new Uri("https://youtube.com/channel/UCxyCzPY2pjAjrxoSYclpuLg"),
LastBuildDate = DateTime.Parse("Tue, 08 Nov 2016 05:55:25 GMT"),
PubDate = DateTime.Parse("Mon, 31 Jul 2006 22:18:05 GMT"),
Subtitle = "Laugh Factory",
@@ -4,9 +4,14 @@ using Xunit;
namespace Podsync.Tests.Services.Links
{
public class LinkServiceTests
public class LinkServiceTests : TestBase
{
private readonly ILinkService _linkService = new LinkService();
private readonly ILinkService _linkService;
public LinkServiceTests()
{
_linkService = new LinkService(Options);
}
[Theory]
[InlineData("http://youtu.be/jMeC7JFQ6801", Provider.YouTube, LinkType.Video, "jMeC7JFQ6801")]
@@ -33,5 +33,12 @@ namespace Podsync.Tests.Services.Resolver
{
Assert.NotNull(_resolver.Version);
}
[Fact]
public async Task ResolveOutputTest()
{
var downloadUrl = await _resolver.Resolve(new Uri("https://www.youtube.com/watch?v=-csRxRj_zcw&t=45s"), ResolveType.AudioHigh);
Assert.True(downloadUrl.IsAbsoluteUri);
}
}
}
@@ -14,7 +14,8 @@ namespace Podsync.Tests.Services.Videos.YouTube
public YouTubeClientTests()
{
_client = new YouTubeClient(new LinkService(), Options);
var linkService = new LinkService(Options);
_client = new YouTubeClient(linkService, Options);
}
[Fact]
@@ -52,7 +53,21 @@ namespace Podsync.Tests.Services.Videos.YouTube
Assert.Equal("GDC: Postmortems", playlist.Title);
Assert.Equal(new Uri("https://youtube.com/playlist?list=PL2e4mYbwSTbbiX2uwspn0xiYb8_P_cTAr"), playlist.Link);
Assert.NotNull(playlist.Thumbnail);
Assert.Equal(new DateTime(2015, 06, 29, 16, 58, 34), playlist.PublishedAt);
Assert.Equal(new DateTime(2015, 06, 29), playlist.PublishedAt.Date);
}
[Fact]
public async Task GetPlaylistIdsTest()
{
var query = new PlaylistItemsQuery
{
PlaylistId = "PL2e4mYbwSTbbiX2uwspn0xiYb8_P_cTAr",
Count = 3
};
var list = await _client.GetPlaylistItemIds(query);
Assert.NotEmpty(list);
}
[Fact]
@@ -75,7 +90,7 @@ namespace Podsync.Tests.Services.Videos.YouTube
Assert.Equal("Deus Ex: Mankind Divided - Full Soundtrack OST", last.Title);
Assert.False(string.IsNullOrEmpty(last.Description));
Assert.Equal(new TimeSpan(0, 1, 57, 28), last.Duration);
Assert.Equal(new DateTime(2016, 8, 24, 16, 50, 34), last.PublishedAt);
Assert.Equal(new DateTime(2016, 8, 24), last.PublishedAt.Date);
Assert.Equal(new Uri("https://youtube.com/watch?v=kkcKnWrCZ7k"), last.Link);
Assert.True(last.Size > 0);
}