1
0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00

Merge new podsync backend

This commit is contained in:
Maksym Pavlenko
2017-10-20 19:23:54 -07:00
145 changed files with 3030 additions and 4592 deletions

5
.dockerignore Normal file
View File

@@ -0,0 +1,5 @@
node_modules/
vendor/
package-lock.json
Gopkg.lock

273
.gitignore vendored
View File

@@ -1,258 +1,35 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# Folders
_obj
_test
vendor
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
_testmain.go
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
*.exe
*.test
*.prof
# NUNIT
*.VisualState.xml
TestResult.xml
glide.lock
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
Gopkg.lock
# DNX
project.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# Podsync
src/Podsync/wwwroot/lib/
src/Podsync/wwwroot/**/*.min.css
src/Podsync/wwwroot/**/*.min.js
src/Podsync/wwwroot/**/*.min.js.map
node_modules/
package-lock.json
dist/

19
.travis.yml Normal file
View File

@@ -0,0 +1,19 @@
dist: trusty
sudo: required
language: go
go_import_path: github.com/mxpv/podsync
go:
- 1.8.3
install:
- go get -u github.com/golang/dep/cmd/dep
- dep ensure
services:
- postgresql
addons:
postgresql: "9.6"
before_script:
- psql -a -c "CREATE DATABASE podsync;" -U postgres
script:
- set -e
- go test -v -short $(go list -e ./... | grep -v vendor)
- go tool vet -all ./pkg

26
Dockerfile Normal file
View File

@@ -0,0 +1,26 @@
FROM node:latest AS gulp
WORKDIR /app
COPY . .
RUN npm install
RUN npm link gulp
RUN gulp patch
FROM golang:1.8 AS build
WORKDIR /go/src/github.com/mxpv/podsync
COPY --from=gulp /app .
ENV GOOS=linux
ENV GOARCH=amd64
ENV CGO_ENABLED=0
RUN go get -u github.com/golang/dep/cmd/dep
RUN dep ensure
RUN go install -v ./cmd/app
FROM alpine
RUN apk --update --no-cache add ca-certificates
WORKDIR /app/
COPY --from=gulp /app/templates ./templates
COPY --from=gulp /app/dist ./assets
COPY --from=build /go/bin/app .
ENV ASSETS_PATH /app/assets
ENV TEMPLATES_PATH /app/templates
ENTRYPOINT ["/app/app"]

20
Gopkg.toml Normal file
View File

@@ -0,0 +1,20 @@
[[constraint]]
name = "github.com/mxpv/podcast"
branch = "release"
[[constraint]]
name = "github.com/speps/go-hashids"
revision = "c6ced3330f133cec79e144fb11b2e4c5154059ae"
[[constraint]]
name = "github.com/gin-gonic/gin"
version = "1.2"
[[constraint]]
name = "github.com/gin-contrib/sessions"
revision = "a71ea9167c616cf9f02bc928ed08bc390c8279bc"
[[constraint]]
name = "github.com/mxpv/patreon-go"
version = "1.2"

View File

@@ -1,62 +0,0 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26403.7
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{3EF2294C-C807-46F3-B3F4-98B7F30688D0}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{15355284-4B88-4EF3-B580-20907F343E43}"
ProjectSection(SolutionItems) = preProject
.gitignore = .gitignore
docker-compose.yml = docker-compose.yml
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Podsync", "src\Podsync\Podsync.csproj", "{C9283472-E95E-4517-AE3B-3E276B9851E4}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{D94416F2-31EA-4D38-A3D0-BB74180687D4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Podsync.Tests", "test\Podsync.Tests\Podsync.Tests.csproj", "{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x64.ActiveCfg = Debug|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x64.Build.0 = Debug|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x86.ActiveCfg = Debug|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x86.Build.0 = Debug|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|Any CPU.Build.0 = Release|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x64.ActiveCfg = Release|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x64.Build.0 = Release|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x86.ActiveCfg = Release|Any CPU
{C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x86.Build.0 = Release|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x64.ActiveCfg = Debug|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x64.Build.0 = Debug|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x86.ActiveCfg = Debug|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x86.Build.0 = Debug|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|Any CPU.Build.0 = Release|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x64.ActiveCfg = Release|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x64.Build.0 = Release|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x86.ActiveCfg = Release|Any CPU
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{C9283472-E95E-4517-AE3B-3E276B9851E4} = {3EF2294C-C807-46F3-B3F4-98B7F30688D0}
{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA} = {D94416F2-31EA-4D38-A3D0-BB74180687D4}
EndGlobalSection
EndGlobal

View File

@@ -1,8 +1,37 @@
# Podsync
Turn YouTube channels into podcast feeds
## Handling static files
## Redis configuration notes
- [Set overcommit_memory=1 on CoreOS](https://gist.github.com/PyYoshi/c7b271c8ba80ea350376)
- [Set somaxconn=1024 on CoreOS](https://gist.github.com/PyYoshi/55c6953b7e8267d81daf)
- [Tweak Redis](https://easyengine.io/tutorials/redis/)
- [Performance tips for Redis Cache Server](https://www.techandme.se/performance-tips-for-redis-cache-server/)
`ASSETS_PATH` should point to a directory with static files.
For debugging just 'Copy Path' to `assets` directory.
For production run `gulp patch`.
Gulp will generate `dist` directory with minified files and update templates to include these files.
`TEMPLATES_PATH` should just point to `templates` directory.
Docker will run `gulp` and include `dist` and `templates` directories during build as well as specify `ASSETS_PATH` and `TEMPLATES_PATH` environment variables.
## Patreon
In order to login via Patreon the following variables should be configured:
- `PATREON_REDIRECT_URL` should point to `http://yout_host_here/patreon`
- `PATREON_CLIENT_ID` and `PATREON_SECRET` should be copied from https://www.patreon.com/platform/documentation/clients
## Deploy Docker images
Build docker image:
```
docker build -t ytdl .
```
Deploy image to Container Registry:
```
docker tag ytdl gcr.io/pod-sync/ytdl
gcloud auth application-default login
gcloud docker -- push gcr.io/pod-sync/ytdl
```
or just use `build.sh`
## Access to Container Registry from docker
See https://cloud.google.com/container-registry/docs/advanced-authentication for details

View File

@@ -13,7 +13,7 @@ a {
}
.background-image {
background-image: url('/img/pc_bg.png');
background-image: url('/assets/img/pc_bg.png');
-ms-background-repeat: repeat-x;
background-repeat: repeat-x;
-ms-background-position: center bottom;
@@ -32,7 +32,7 @@ a {
/* Footer */
.footer {
background-image: url('/img/pc_footer.png');
background-image: url('/assets/img/pc_footer.png');
-ms-background-repeat: space;
background-repeat: space;
-ms-background-position: center bottom;
@@ -46,7 +46,7 @@ a {
margin: 0;
padding: 0;
-webkit-user-select: text;
z-index: 1;
z-index: -1;
}
.twitter-link {
@@ -180,7 +180,7 @@ a {
.man {
width: 210px;
height: 333px;
background-image: url('/img/man.png');
background-image: url('/assets/img/man.png');
-ms-background-repeat: no-repeat;
background-repeat: no-repeat;
-ms-background-position: center bottom;
@@ -267,11 +267,11 @@ a {
@media screen and (max-width: 640px) {
.background-image {
background-image: url('/img/mobile_bg.png')
background-image: url('/assets/img/mobile_bg.png')
}
.footer {
background-image: url('/img/mobile_footer.png');
background-image: url('/assets/img/mobile_footer.png');
height: 50px;
}

View File

Before

Width:  |  Height:  |  Size: 4.2 KiB

After

Width:  |  Height:  |  Size: 4.2 KiB

View File

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 13 KiB

View File

Before

Width:  |  Height:  |  Size: 34 KiB

After

Width:  |  Height:  |  Size: 34 KiB

View File

Before

Width:  |  Height:  |  Size: 54 KiB

After

Width:  |  Height:  |  Size: 54 KiB

View File

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

View File

Before

Width:  |  Height:  |  Size: 89 KiB

After

Width:  |  Height:  |  Size: 89 KiB

View File

Before

Width:  |  Height:  |  Size: 3.4 KiB

After

Width:  |  Height:  |  Size: 3.4 KiB

View File

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

View File

Before

Width:  |  Height:  |  Size: 2.7 KiB

After

Width:  |  Height:  |  Size: 2.7 KiB

View File

@@ -1,6 +1,4 @@
// Write your Javascript code.
$(function () {
$(function () {
function err(msg) {
alert(msg);
}
@@ -12,40 +10,33 @@ $(function () {
$.ajax({
dataType: 'text',
url: '/feed/create',
url: '/api/create',
method: 'POST',
data: JSON.stringify(data),
contentType: 'application/json; charset=utf-8',
success: function (feedLink) {
// HACK: remove quotes
feedLink = JSON.parse(feedLink);
done(feedLink);
success: function (resp) {
done(JSON.parse(resp));
},
error: function (xhr, status, error) {
if (xhr.status === 400) {
// Bad request
var text = '';
var text = '';
try {
var json = JSON.parse(xhr.responseText);
$.each(json, function (key, value) {
text += value + '\r\n';
});
} catch (e) {
text = xhr.responseText;
}
err(text);
} else {
// Generic error
err('Server sad \'' + error + '\': ' + xhr.responseText);
try {
var json = JSON.parse(xhr.responseText);
if (json['error']) {
text = json['error'];
}
} catch (e) {
text = xhr.responseText;
}
err(text);
}
});
}
function displayLink(link) {
showModal(link);
function displayLink(obj) {
var addr = location.protocol + '//' + location.hostname + '/' + obj.id;
showModal(addr);
}
/*
@@ -104,15 +95,14 @@ $(function () {
$('#best-quality, #worst-quality').toggleClass('selected-option');
}
function getQuality() {
function getFormat() {
var isAudio = $('#audio-format').hasClass('selected-option');
var isWorst = $('#worst-quality').hasClass('selected-option');
return isAudio ? 'audio' : 'video'
}
if (isAudio) {
return isWorst ? 'AudioLow' : 'AudioHigh';
} else {
return isWorst ? 'VideoLow' : 'VideoHigh';
}
function getQuality() {
var isWorst = $('#worst-quality').hasClass('selected-option');
return isWorst ? 'low' : 'high';
}
function pageSwitch(evt) {
@@ -195,7 +185,7 @@ $(function () {
$('#get-link').click(function(e) {
var url = $('#url-input').val();
createFeed({ url: url, quality: getQuality(), pageSize: getPageCount() }, displayLink);
createFeed({ url: url, format: getFormat(), quality: getQuality(), page_size: getPageCount() }, displayLink);
e.preventDefault();
});

View File

@@ -1,6 +0,0 @@
#!/bin/bash
BACKUP_DIR=${1:-"$HOME/PodsyncBackups"}
mkdir -p $BACKUP_DIR
docker-machine scp podsync:/data/redis/appendonly.aof $BACKUP_DIR/redis.$(date '+%Y_%m_%d__%H_%M_%S').aof

View File

@@ -1,19 +0,0 @@
{
"name": "Podsync",
"homepage": "Podsync.net",
"authors": [
"Maksym Pavlenko <pavlenko.maksym@gmail.com>"
],
"description": "",
"main": "",
"moduleType": [],
"license": "MIT",
"private": true,
"ignore": [
"**/.*",
"node_modules",
"bower_components",
"test",
"tests"
]
}

3
build.sh Executable file
View File

@@ -0,0 +1,3 @@
docker build -t app .
docker tag app gcr.io/pod-sync/app
gcloud docker -- push gcr.io/pod-sync/app

83
cmd/app/main.go Normal file
View File

@@ -0,0 +1,83 @@
package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"github.com/mxpv/podsync/pkg/api"
"github.com/mxpv/podsync/pkg/builders"
"github.com/mxpv/podsync/pkg/config"
"github.com/mxpv/podsync/pkg/feeds"
"github.com/mxpv/podsync/pkg/id"
"github.com/mxpv/podsync/pkg/server"
"github.com/mxpv/podsync/pkg/storage"
)
func main() {
stop := make(chan os.Signal)
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Create core sevices
cfg, err := config.ReadConfiguration()
if err != nil {
panic(err)
}
hashIds, err := id.NewIdGenerator()
if err != nil {
panic(err)
}
redis, err := storage.NewRedisStorage(cfg.RedisURL)
if err != nil {
panic(err)
}
// Builders
youtube, err := builders.NewYouTubeBuilder(cfg.YouTubeApiKey)
if err != nil {
panic(err)
}
vimeo, err := builders.NewVimeoBuilder(ctx, cfg.VimeoApiKey)
if err != nil {
panic(err)
}
feed := feeds.NewFeedService(
feeds.WithIdGen(hashIds),
feeds.WithStorage(redis),
feeds.WithBuilder(api.Youtube, youtube),
feeds.WithBuilder(api.Vimeo, vimeo),
)
srv := http.Server{
Addr: fmt.Sprintf(":%d", 5001),
Handler: server.MakeHandlers(feed, cfg),
}
go func() {
log.Println("running listener")
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
<-stop
log.Printf("shutting down server")
srv.Shutdown(ctx)
log.Printf("server gracefully stopped")
}

View File

16
cmd/nginx/nginx.conf Normal file
View File

@@ -0,0 +1,16 @@
server {
listen 80;
gzip on;
server_tokens off;
location /download {
proxy_pass http://ytdl:5002;
}
location / {
proxy_pass http://app:5001;
proxy_buffers 8 16k;
proxy_buffer_size 16k;
}
}

View File

3
cmd/ytdl/build.sh Executable file
View File

@@ -0,0 +1,3 @@
docker build -t ytdl .
docker tag ytdl gcr.io/pod-sync/ytdl
gcloud docker -- push gcr.io/pod-sync/ytdl

View File

@@ -1,3 +1,4 @@
requests
youtube_dl
sanic
redis

8
cmd/ytdl/test_ytdl.py Normal file
View File

@@ -0,0 +1,8 @@
import ytdl
import unittest
class TestYtdl(unittest.TestCase):
def test_resolve(self):
self.assertIsNotNone(ytdl._resolve('https://youtube.com/watch?v=ygIUF678y40', {'format': 'video', 'quality': 'low'}))
self.assertIsNotNone(ytdl._resolve('https://youtube.com/watch?v=WyaEiO4hyik', {'format': 'audio', 'quality': 'high'}))

View File

@@ -1,17 +1,16 @@
import youtube_dl
import redis
import os
from youtube_dl.utils import DownloadError
import requests
import youtube_dl
from sanic import Sanic
from sanic.exceptions import InvalidUsage, NotFound
from sanic.exceptions import InvalidUsage
from sanic.response import text, redirect
from datetime import timedelta
from youtube_dl.utils import DownloadError
METADATA_URL = os.getenv('METADATA_URL', 'http://app:5001/api/metadata/{feed_id}')
print('Using metadata URL template: ' + METADATA_URL)
app = Sanic()
db = redis.from_url(os.getenv('REDIS_CONNECTION_STRING', 'redis://localhost:6379'))
db.ping()
opts = {
'quiet': True,
'no_warnings': True,
@@ -29,7 +28,7 @@ url_formats = {
@app.route('/download/<feed_id>/<video_id>', methods=['GET'])
async def download(request, feed_id, video_id):
async def download(req, feed_id, video_id):
if not feed_id:
raise InvalidUsage('Invalid feed id')
@@ -38,44 +37,34 @@ async def download(request, feed_id, video_id):
if not video_id:
raise InvalidUsage('Invalid video id')
# Query redis
data = db.hgetall(feed_id)
if not data:
raise NotFound('Feed not found')
# Delete this feed if no requests within 90 days
db.expire(feed_id, timedelta(days=90))
entries = {k.decode().lower(): v.decode().lower() for k, v in data.items()}
# Pull metadata from API server
metadata_url = METADATA_URL.format(feed_id=feed_id, video_id=video_id)
r = requests.get(url=metadata_url)
json = r.json()
# Build URL
provider = entries.get('provider')
provider = json['provider']
tpl = url_formats[provider]
if not tpl:
raise InvalidUsage('Invalid feed')
url = tpl.format(video_id)
quality = entries.get('quality')
try:
redirect_url = _resolve(url, quality)
redirect_url = _resolve(url, json)
return redirect(redirect_url)
except DownloadError as e:
msg = str(e)
return text(msg, status=511)
def _resolve(url, quality):
def _resolve(url, metadata):
if not url:
raise InvalidUsage('Invalid URL')
if not quality:
quality = 'videohigh'
try:
with youtube_dl.YoutubeDL(opts) as ytdl:
info = ytdl.extract_info(url, download=False)
return _choose_url(info, quality)
return _choose_url(info, metadata)
except DownloadError:
raise
except Exception as e:
@@ -83,8 +72,8 @@ def _resolve(url, quality):
raise
def _choose_url(info, quality):
is_video = quality == 'videohigh' or quality == 'videolow'
def _choose_url(info, metadata):
is_video = metadata['format'] == 'video'
# Filter formats by file extension
ext = 'mp4' if is_video else 'm4a'
@@ -100,10 +89,10 @@ def _choose_url(info, quality):
ordered = sorted(fmt_list, key=lambda x: x[sort_field], reverse=True)
# Choose an item depending on quality, better at the beginning
is_high_quality = quality == 'videohigh' or quality == 'audiohigh'
is_high_quality = metadata['quality'] == 'high'
item = ordered[0] if is_high_quality else ordered[-1]
return item['url']
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5002, workers=32)
app.run(host='0.0.0.0', port=5002, workers=32)

View File

@@ -8,16 +8,13 @@ services:
ports:
- 5001
environment:
- ASPNETCORE_URLS=http://*:5001
- ASPNETCORE_ENVIRONMENT=Production
- Podsync:RedisConnectionString=redis
- Podsync:RemoteResolverUrl=http://ytdl:5002
- Podsync:YouTubeApiKey={YOUTUBE_API_KEY}
- Podsync:VimeoApiKey={VIMEO_API_KEY}
- Podsync:PatreonClientId={PATREON_CLIENT_ID}
- Podsync:PatreonSecret={PATREON_SECRET}
- Logging:LogLevel:Default=Information
- Logging:LogLevel:Microsoft=Warning
- REDIS_CONNECTION_URL=redis://redis
- YOUTUBE_API_KEY={YOUTUBE_API_KEY}
- VIMEO_API_KEY={VIMEO_API_KEY}
- PATREON_CLIENT_ID={PATREON_CLIENT_ID}
- PATREON_SECRET={PATREON_SECRET}
- PATREON_REDIRECT_URL=http://podsync.net/patreon
- COOKIE_SECRET={COOKIE_SECRET}
ytdl:
image: gcr.io/pod-sync/ytdl:latest
container_name: ytdl
@@ -25,7 +22,7 @@ services:
ports:
- 5002
environment:
- REDIS_CONNECTION_STRING=redis://redis:6379/0
- METADATA_URL=http://app:5001/api/metadata/{feed_id}
redis:
image: redis
container_name: redis

48
gulpfile.js Normal file
View File

@@ -0,0 +1,48 @@
var gulp = require('gulp'),
del = require('del'),
path = require('path'),
uglify = require('gulp-uglify'),
rev = require('gulp-rev'),
revreplace = require('gulp-rev-replace'),
cleancss = require('gulp-clean-css'),
autoprefixer = require('gulp-autoprefixer'),
size = require('gulp-size'),
gulpif = require('gulp-if'),
imagemin = require('gulp-imagemin');
abs = path.join(process.cwd(), 'assets');
gulp.task('clean', function () {
return del(['./dist/**/*'])
});
// Minify images and output to ./dist folder
gulp.task('img', ['clean'], function() {
return gulp.src('./assets/**/*.{png,ico}')
.pipe(imagemin())
.pipe(size())
.pipe(gulp.dest('./dist'))
});
// Minify scripts, build manifest.json and output to ./dist folder
gulp.task('js+css', ['clean', 'img'], function() {
return gulp.src(['./assets/js/**/*.js', './assets/css/**/*.css'], {base: abs})
.pipe(gulpif(/js$/, uglify()))
.pipe(gulpif(/css$/, autoprefixer()))
.pipe(gulpif(/css$/, cleancss()))
.pipe(rev())
.pipe(size())
.pipe(gulp.dest('./dist'))
.pipe(rev.manifest('manifest.json', {merge: true}))
.pipe(gulp.dest('./dist'));
});
// Rewrite occurrences of scripts in template files
gulp.task('patch', ['js+css'], function() {
var manifest = gulp.src('./dist/manifest.json');
return gulp.src('./templates/index.html')
.pipe(revreplace({manifest: manifest}))
.pipe(gulp.dest('./templates/'))
});
gulp.task('default', ['js+css']);

View File

@@ -1,21 +0,0 @@
server {
listen 80;
gzip on;
server_tokens off;
location /download {
proxy_pass http://ytdl:5002;
}
location / {
proxy_pass http://app:5001;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection keep-alive;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_cache_bypass $http_upgrade;
}
}

32
package.json Normal file
View File

@@ -0,0 +1,32 @@
{
"name": "podsync",
"version": "1.0.0",
"description": "",
"main": "gulpfile.js",
"dependencies": {},
"devDependencies": {
"del": "^3.0.0",
"gulp": "^3.9.1",
"gulp-autoprefixer": "^4.0.0",
"gulp-clean-css": "^3.7.0",
"gulp-if": "^2.0.2",
"gulp-imagemin": "^3.3.0",
"gulp-rev": "^8.0.0",
"gulp-rev-replace": "^0.4.3",
"gulp-size": "^2.1.0",
"gulp-uglify": "^3.0.0"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "git+https://github.com/mxpv/Podsync.git"
},
"author": "",
"license": "ISC",
"bugs": {
"url": "https://github.com/mxpv/Podsync/issues"
},
"homepage": "https://github.com/mxpv/Podsync#readme"
}

82
pkg/api/api.go Normal file
View File

@@ -0,0 +1,82 @@
package api
import (
"time"
"github.com/pkg/errors"
)
var (
ErrNotFound = errors.New("resource not found")
)
type Provider string
const (
Youtube = Provider("youtube")
Vimeo = Provider("vimeo")
)
type LinkType string
const (
Channel = LinkType("channel")
Playlist = LinkType("playlist")
User = LinkType("user")
Group = LinkType("group")
)
type Quality string
const (
HighQuality = Quality("high")
LowQuality = Quality("low")
)
type Format string
const (
AudioFormat = Format("audio")
VideoFormat = Format("video")
)
const (
DefaultPageSize = 50
DefaultFormat = VideoFormat
DefaultQuality = HighQuality
)
type Feed struct {
Id int64 `json:"id"`
HashId string `json:"hash_id"` // Short human readable feed id for users
UserId string `json:"user_id"` // Patreon user id
ItemId string `json:"item_id"`
Provider Provider `json:"provider"` // Youtube or Vimeo
LinkType LinkType `json:"link_type"` // Either group, channel or user
PageSize int `json:"page_size"` // The number of episodes to return
Format Format `json:"format"`
Quality Quality `json:"quality"`
FeatureLevel int `json:"feature_level"` // Available features
LastAccess time.Time `json:"last_access"`
}
const (
DefaultFeatures = iota
ExtendedFeatures
PodcasterFeature
)
type CreateFeedRequest struct {
URL string `json:"url" binding:"url,required"`
PageSize int `json:"page_size" binding:"min=10,max=150,required"`
Quality Quality `json:"quality" binding:"eq=high|eq=low"`
Format Format `json:"format" binding:"eq=video|eq=audio"`
}
type Identity struct {
UserId string `json:"user_id"`
FullName string `json:"full_name"`
Email string `json:"email"`
ProfileURL string `json:"profile_url"`
FeatureLevel int `json:"feature_level"`
}

25
pkg/builders/common.go Normal file
View File

@@ -0,0 +1,25 @@
package builders
import (
"fmt"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
)
const (
podsyncGenerator = "Podsync generator"
defaultCategory = "TV & Film"
)
func makeEnclosure(feed *api.Feed, id string, lengthInBytes int64) (string, itunes.EnclosureType, int64) {
ext := "mp4"
contentType := itunes.MP4
if feed.Format == api.AudioFormat {
ext = "m4a"
contentType = itunes.M4A
}
url := fmt.Sprintf("http://podsync.net/download/%s/%s.%s", feed.HashId, id, ext)
return url, contentType, lengthInBytes
}

190
pkg/builders/vimeo.go Normal file
View File

@@ -0,0 +1,190 @@
package builders
import (
"net/http"
"strconv"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
"github.com/pkg/errors"
"github.com/silentsokolov/go-vimeo"
"golang.org/x/net/context"
"golang.org/x/oauth2"
)
const (
vimeoDefaultPageSize = 50
)
type VimeoBuilder struct {
client *vimeo.Client
}
func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q api.Quality) string {
if p == nil || len(p.Sizes) == 0 {
return ""
}
if q == api.LowQuality {
return p.Sizes[0].Link
} else {
return p.Sizes[len(p.Sizes)-1].Link
}
}
func (v *VimeoBuilder) queryChannel(feed *api.Feed) (*itunes.Podcast, error) {
channelId := feed.ItemId
ch, resp, err := v.client.Channels.Get(channelId)
if err != nil {
if resp.StatusCode == http.StatusNotFound {
return nil, api.ErrNotFound
}
return nil, errors.Wrapf(err, "failed to query channel with channelId %s", channelId)
}
podcast := itunes.New(ch.Name, ch.Link, ch.Description, &ch.CreatedTime, nil)
podcast.Generator = podsyncGenerator
podcast.AddSubTitle(ch.Name)
podcast.AddImage(v.selectImage(ch.Pictures, feed.Quality))
podcast.AddCategory(defaultCategory, nil)
podcast.IAuthor = ch.User.Name
return &podcast, nil
}
func (v *VimeoBuilder) queryGroup(feed *api.Feed) (*itunes.Podcast, error) {
groupId := feed.ItemId
gr, resp, err := v.client.Groups.Get(groupId)
if err != nil {
if resp.StatusCode == http.StatusNotFound {
return nil, api.ErrNotFound
}
return nil, errors.Wrapf(err, "failed to query group with id %s", groupId)
}
podcast := itunes.New(gr.Name, gr.Link, gr.Description, &gr.CreatedTime, nil)
podcast.Generator = podsyncGenerator
podcast.AddSubTitle(gr.Name)
podcast.AddImage(v.selectImage(gr.Pictures, feed.Quality))
podcast.AddCategory(defaultCategory, nil)
podcast.IAuthor = gr.User.Name
return &podcast, nil
}
func (v *VimeoBuilder) queryUser(feed *api.Feed) (*itunes.Podcast, error) {
userId := feed.ItemId
user, resp, err := v.client.Users.Get(userId)
if err != nil {
if resp.StatusCode == http.StatusNotFound {
return nil, api.ErrNotFound
}
return nil, errors.Wrapf(err, "failed to query user with id %s", userId)
}
podcast := itunes.New(user.Name, user.Link, user.Bio, &user.CreatedTime, nil)
podcast.Generator = podsyncGenerator
podcast.AddSubTitle(user.Name)
podcast.AddImage(v.selectImage(user.Pictures, feed.Quality))
podcast.AddCategory(defaultCategory, nil)
podcast.IAuthor = user.Name
return &podcast, nil
}
func (v *VimeoBuilder) getVideoSize(video *vimeo.Video) int64 {
// Very approximate video file size
return int64(float64(video.Duration*video.Width*video.Height) * 0.38848958333)
}
type getVideosFunc func(id string, opt *vimeo.ListVideoOptions) ([]*vimeo.Video, *vimeo.Response, error)
func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, podcast *itunes.Podcast, feed *api.Feed) error {
opt := vimeo.ListVideoOptions{}
opt.Page = 1
opt.PerPage = vimeoDefaultPageSize
added := 0
for {
videos, response, err := getVideos(feed.ItemId, &opt)
if err != nil {
return errors.Wrapf(err, "failed to query videos (error %d %s)", response.StatusCode, response.Status)
}
for _, video := range videos {
item := itunes.Item{}
item.GUID = strconv.Itoa(video.GetID())
item.Link = video.Link
item.Title = video.Name
item.Description = video.Description
if item.Description == "" {
item.Description = " " // Videos can be without description, workaround for AddItem
}
item.AddDuration(int64(video.Duration))
item.AddPubDate(&video.CreatedTime)
item.AddImage(v.selectImage(video.Pictures, feed.Quality))
size := v.getVideoSize(video)
item.AddEnclosure(makeEnclosure(feed, item.GUID, size))
_, err = podcast.AddItem(item)
if err != nil {
return errors.Wrapf(err, "failed to add episode %s (%s)", item.GUID, item.Title)
}
added++
}
if added >= feed.PageSize || response.NextPage == "" {
return nil
}
opt.Page++
}
}
func (v *VimeoBuilder) Build(feed *api.Feed) (podcast *itunes.Podcast, err error) {
if feed.LinkType == api.Channel {
if podcast, err = v.queryChannel(feed); err == nil {
err = v.queryVideos(v.client.Channels.ListVideo, podcast, feed)
}
return
}
if feed.LinkType == api.Group {
if podcast, err = v.queryGroup(feed); err == nil {
err = v.queryVideos(v.client.Groups.ListVideo, podcast, feed)
}
return
}
if feed.LinkType == api.User {
if podcast, err = v.queryUser(feed); err == nil {
err = v.queryVideos(v.client.Users.ListVideo, podcast, feed)
}
return
}
err = errors.New("unsupported feed type")
return
}
func NewVimeoBuilder(ctx context.Context, token string) (*VimeoBuilder, error) {
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
tc := oauth2.NewClient(ctx, ts)
client := vimeo.NewClient(tc)
return &VimeoBuilder{client}, nil
}

View File

@@ -0,0 +1,80 @@
package builders
import (
"context"
"os"
"testing"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
"github.com/stretchr/testify/require"
)
var (
vimeoKey = os.Getenv("VIMEO_TEST_API_KEY")
)
func TestQueryVimeoChannel(t *testing.T) {
builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
require.NoError(t, err)
podcast, err := builder.queryChannel(&api.Feed{ItemId: "staffpicks", Quality: api.HighQuality})
require.NoError(t, err)
require.Equal(t, "https://vimeo.com/channels/staffpicks", podcast.Link)
require.Equal(t, "Vimeo Staff Picks", podcast.Title)
require.Equal(t, "Vimeo Curation", podcast.IAuthor)
require.NotEmpty(t, podcast.Description)
require.NotEmpty(t, podcast.Image)
require.NotEmpty(t, podcast.IImage)
}
func TestQueryVimeoGroup(t *testing.T) {
builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
require.NoError(t, err)
podcast, err := builder.queryGroup(&api.Feed{ItemId: "motion", Quality: api.HighQuality})
require.NoError(t, err)
require.Equal(t, "https://vimeo.com/groups/motion", podcast.Link)
require.Equal(t, "Motion Graphic Artists", podcast.Title)
require.Equal(t, "Danny Garcia", podcast.IAuthor)
require.NotEmpty(t, podcast.Description)
require.NotEmpty(t, podcast.Image)
require.NotEmpty(t, podcast.IImage)
}
func TestQueryVimeoUser(t *testing.T) {
builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
require.NoError(t, err)
podcast, err := builder.queryUser(&api.Feed{ItemId: "motionarray", Quality: api.HighQuality})
require.NoError(t, err)
require.Equal(t, "https://vimeo.com/motionarray", podcast.Link)
require.Equal(t, "Motion Array", podcast.Title)
require.Equal(t, "Motion Array", podcast.IAuthor)
require.NotEmpty(t, podcast.Description)
}
func TestQueryVimeoVideos(t *testing.T) {
builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
require.NoError(t, err)
feed := &itunes.Podcast{}
err = builder.queryVideos(builder.client.Channels.ListVideo, feed, &api.Feed{ItemId: "staffpicks"})
require.NoError(t, err)
require.Equal(t, vimeoDefaultPageSize, len(feed.Items))
for _, item := range feed.Items {
require.NotEmpty(t, item.Title)
require.NotEmpty(t, item.Link)
require.NotEmpty(t, item.GUID)
require.NotEmpty(t, item.IDuration)
require.NotNil(t, item.Enclosure)
require.NotEmpty(t, item.Enclosure.URL)
require.True(t, item.Enclosure.Length > 0)
}
}

324
pkg/builders/youtube.go Normal file
View File

@@ -0,0 +1,324 @@
package builders
import (
"fmt"
"net/http"
"strings"
"time"
"github.com/BrianHicks/finch/duration"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
"github.com/pkg/errors"
"google.golang.org/api/youtube/v3"
)
const (
maxYoutubeResults = 50
hdBytesPerSecond = 350000
ldBytesPerSecond = 100000
)
type apiKey string
func (key apiKey) Get() (string, string) {
return "key", string(key)
}
type YouTubeBuilder struct {
client *youtube.Service
key apiKey
}
func (yt *YouTubeBuilder) listChannels(linkType api.LinkType, id string) (*youtube.Channel, error) {
req := yt.client.Channels.List("id,snippet,contentDetails")
if linkType == api.Channel {
req = req.Id(id)
} else if linkType == api.User {
req = req.ForUsername(id)
} else {
return nil, errors.New("unsupported link type")
}
resp, err := req.Do(yt.key)
if err != nil {
return nil, errors.Wrapf(err, "failed to query channel")
}
if len(resp.Items) == 0 {
return nil, api.ErrNotFound
}
item := resp.Items[0]
return item, nil
}
func (yt *YouTubeBuilder) listPlaylists(id, channelId string) (*youtube.Playlist, error) {
req := yt.client.Playlists.List("id,snippet")
if id != "" {
req = req.Id(id)
} else {
req = req.ChannelId(channelId)
}
resp, err := req.Do(yt.key)
if err != nil {
return nil, errors.Wrapf(err, "failed to query playlist")
}
if len(resp.Items) == 0 {
return nil, api.ErrNotFound
}
item := resp.Items[0]
return item, nil
}
func (yt *YouTubeBuilder) listPlaylistItems(itemId string, pageToken string) ([]*youtube.PlaylistItem, string, error) {
req := yt.client.PlaylistItems.List("id,snippet").MaxResults(maxYoutubeResults).PlaylistId(itemId)
if pageToken != "" {
req = req.PageToken(pageToken)
}
resp, err := req.Do(yt.key)
if err != nil {
return nil, "", errors.Wrap(err, "failed to query playlist items")
}
return resp.Items, resp.NextPageToken, nil
}
func (yt *YouTubeBuilder) parseDate(s string) (time.Time, error) {
date, err := time.Parse(time.RFC3339, s)
if err != nil {
return time.Time{}, errors.Wrapf(err, "failed to parse date: %s", s)
}
return date, nil
}
func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, quality api.Quality) string {
// Use high resolution thumbnails for high quality mode
// https://github.com/mxpv/Podsync/issues/14
if quality == api.HighQuality {
if snippet.Maxres != nil {
return snippet.Maxres.Url
}
if snippet.High != nil {
return snippet.High.Url
}
if snippet.Medium != nil {
return snippet.Medium.Url
}
}
return snippet.Default.Url
}
func (yt *YouTubeBuilder) queryFeed(feed *api.Feed) (*itunes.Podcast, string, error) {
now := time.Now()
if feed.LinkType == api.Channel || feed.LinkType == api.User {
channel, err := yt.listChannels(feed.LinkType, feed.ItemId)
if err != nil {
return nil, "", err
}
itemId := channel.ContentDetails.RelatedPlaylists.Uploads
link := ""
if feed.LinkType == api.Channel {
link = fmt.Sprintf("https://youtube.com/channel/%s", itemId)
} else {
link = fmt.Sprintf("https://youtube.com/user/%s", itemId)
}
pubDate, err := yt.parseDate(channel.Snippet.PublishedAt)
if err != nil {
return nil, "", err
}
title := channel.Snippet.Title
podcast := itunes.New(title, link, channel.Snippet.Description, &pubDate, &now)
podcast.Generator = podsyncGenerator
podcast.AddSubTitle(title)
podcast.AddCategory(defaultCategory, nil)
podcast.AddImage(yt.selectThumbnail(channel.Snippet.Thumbnails, feed.Quality))
return &podcast, itemId, nil
}
if feed.LinkType == api.Playlist {
playlist, err := yt.listPlaylists(feed.ItemId, "")
if err != nil {
return nil, "", err
}
link := fmt.Sprintf("https://youtube.com/playlist?list=%s", playlist.Id)
snippet := playlist.Snippet
pubDate, err := yt.parseDate(snippet.PublishedAt)
if err != nil {
return nil, "", err
}
title := fmt.Sprintf("%s: %s", snippet.ChannelTitle, snippet.Title)
podcast := itunes.New(title, link, snippet.Description, &pubDate, &now)
podcast.Generator = podsyncGenerator
podcast.AddSubTitle(title)
podcast.AddCategory(defaultCategory, nil)
podcast.AddImage(yt.selectThumbnail(snippet.Thumbnails, feed.Quality))
return &podcast, playlist.Id, nil
}
return nil, "", errors.New("unsupported link format")
}
func (yt *YouTubeBuilder) getVideoSize(definition string, duration int64, fmt api.Format) int64 {
// Video size information requires 1 additional call for each video (1 feed = 50 videos = 50 calls),
// which is too expensive, so get approximated size depending on duration and definition params
var size int64 = 0
if definition == "hd" {
size = duration * hdBytesPerSecond
} else {
size = duration * ldBytesPerSecond
}
// Some podcasts are coming in with exactly double the actual runtime and with the second half just silence.
// https://github.com/mxpv/Podsync/issues/6
if fmt == api.AudioFormat {
size /= 2
}
return size
}
func (yt *YouTubeBuilder) queryVideoDescriptions(ids []string, feed *api.Feed, podcast *itunes.Podcast) error {
req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(strings.Join(ids, ",")).Do(yt.key)
if err != nil {
return errors.Wrap(err, "failed to query video descriptions")
}
for _, video := range req.Items {
snippet := video.Snippet
item := itunes.Item{}
item.GUID = video.Id
item.Link = fmt.Sprintf("https://youtube.com/watch?v=%s", video.Id)
item.Title = snippet.Title
item.Description = snippet.Description
item.ISubtitle = snippet.Title
// Select thumbnail
item.AddImage(yt.selectThumbnail(snippet.Thumbnails, feed.Quality))
// Parse publication date
pubDate, err := yt.parseDate(snippet.PublishedAt)
if err != nil {
return errors.Wrapf(err, "failed to parse video publish date: %s", snippet.PublishedAt)
}
item.AddPubDate(&pubDate)
// Parse duration
d, err := duration.FromString(video.ContentDetails.Duration)
if err != nil {
return errors.Wrapf(err, "failed to parse duration %s", video.ContentDetails.Duration)
}
seconds := int64(d.ToDuration().Seconds())
item.AddDuration(seconds)
// Add download links
size := yt.getVideoSize(video.ContentDetails.Definition, seconds, feed.Format)
item.AddEnclosure(makeEnclosure(feed, video.Id, size))
// podcast.AddItem requires description to be not empty, use workaround
if item.Description == "" {
item.Description = " "
}
_, err = podcast.AddItem(item)
if err != nil {
return errors.Wrapf(err, "failed to add item to podcast (id '%s')", video.Id)
}
}
return nil
}
func (yt *YouTubeBuilder) queryItems(itemId string, feed *api.Feed, podcast *itunes.Podcast) error {
pageToken := ""
count := 0
for {
items, pageToken, err := yt.listPlaylistItems(itemId, pageToken)
if err != nil {
return err
}
if len(items) == 0 {
return nil
}
// Extract video ids
ids := make([]string, len(items))
for index, item := range items {
ids[index] = item.Snippet.ResourceId.VideoId
count++
}
// Query video descriptions from the list of ids
if err := yt.queryVideoDescriptions(ids, feed, podcast); err != nil {
return err
}
if count >= feed.PageSize || pageToken == "" {
return nil
}
}
}
func (yt *YouTubeBuilder) Build(feed *api.Feed) (*itunes.Podcast, error) {
// Query general information about feed (title, description, lang, etc)
podcast, itemId, err := yt.queryFeed(feed)
if err != nil {
return nil, err
}
// Get video descriptions
if err := yt.queryItems(itemId, feed, podcast); err != nil {
return nil, err
}
return podcast, nil
}
func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) {
yt, err := youtube.New(&http.Client{})
if err != nil {
return nil, errors.Wrap(err, "failed to create youtube client")
}
return &YouTubeBuilder{client: yt, key: apiKey(key)}, nil
}

View File

@@ -0,0 +1,56 @@
package builders
import (
"os"
"testing"
"github.com/mxpv/podsync/pkg/api"
"github.com/stretchr/testify/require"
)
var ytKey = os.Getenv("YOUTUBE_TEST_API_KEY")
func TestQueryYTChannel(t *testing.T) {
if testing.Short() {
t.Skip("skipping YT test in short mode")
}
builder, err := NewYouTubeBuilder(ytKey)
require.NoError(t, err)
channel, err := builder.listChannels(api.Channel, "UC2yTVSttx7lxAOAzx1opjoA")
require.NoError(t, err)
require.Equal(t, "UC2yTVSttx7lxAOAzx1opjoA", channel.Id)
channel, err = builder.listChannels(api.User, "fxigr1")
require.NoError(t, err)
require.Equal(t, "UCr_fwF-n-2_olTYd-m3n32g", channel.Id)
}
func TestBuildYTFeed(t *testing.T) {
if testing.Short() {
t.Skip("skipping YT test in short mode")
}
builder, err := NewYouTubeBuilder(ytKey)
require.NoError(t, err)
podcast, err := builder.Build(&api.Feed{
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "UCupvZG-5ko_eiXAupbDfxWw",
PageSize: maxYoutubeResults,
})
require.NoError(t, err)
require.Equal(t, "CNN", podcast.Title)
require.NotEmpty(t, podcast.Description)
require.Equal(t, 50, len(podcast.Items))
for _, item := range podcast.Items {
require.NotEmpty(t, item.Title)
require.NotEmpty(t, item.Link)
require.NotEmpty(t, item.IDuration)
}
}

66
pkg/config/config.go Normal file
View File

@@ -0,0 +1,66 @@
package config
import (
"strings"
"github.com/spf13/viper"
)
const FileName = "podsync"
type AppConfig struct {
YouTubeApiKey string `yaml:"youtubeApiKey"`
VimeoApiKey string `yaml:"vimeoApiKey"`
PatreonClientId string `yaml:"patreonClientId"`
PatreonSecret string `yaml:"patreonSecret"`
PatreonRedirectURL string `yaml:"patreonRedirectUrl"`
PostgresConnectionURL string `yaml:"postgresConnectionUrl"`
RedisURL string `yaml:"redisUrl"`
CookieSecret string `yaml:"cookieSecret"`
AssetsPath string `yaml:"assetsPath"`
TemplatesPath string `yaml:"templatesPath"`
}
func ReadConfiguration() (cfg *AppConfig, err error) {
viper.SetConfigName(FileName)
// Configuration file
viper.AddConfigPath(".")
viper.AddConfigPath("/app/config/")
// Env variables
viper.AutomaticEnv()
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
envmap := map[string]string{
"youtubeApiKey": "YOUTUBE_API_KEY",
"vimeoApiKey": "VIMEO_API_KEY",
"patreonClientId": "PATREON_CLIENT_ID",
"patreonSecret": "PATREON_SECRET",
"patreonRedirectUrl": "PATREON_REDIRECT_URL",
"postgresConnectionUrl": "POSTGRES_CONNECTION_URL",
"redisUrl": "REDIS_CONNECTION_URL",
"cookieSecret": "COOKIE_SECRET",
"assetsPath": "ASSETS_PATH",
"templatesPath": "TEMPLATES_PATH",
}
for k, v := range envmap {
viper.BindEnv(k, v)
}
err = viper.ReadInConfig()
if err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
return
}
// Ignore file not found error
err = nil
}
cfg = &AppConfig{}
viper.Unmarshal(cfg)
return
}

71
pkg/config/config_test.go Normal file
View File

@@ -0,0 +1,71 @@
package config
import (
"io/ioutil"
"os"
"testing"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
const YamlConfig = `
youtubeApiKey: "1"
vimeoApiKey: "2"
patreonClientId: "3"
patreonSecret: "4"
postgresConnectionUrl: "5"
cookieSecret: "6"
patreonRedirectUrl: "7"
assetsPath: "8"
templatesPath: "9"
`
func TestReadYaml(t *testing.T) {
defer viper.Reset()
err := ioutil.WriteFile("./podsync.yaml", []byte(YamlConfig), 0644)
defer os.Remove("./podsync.yaml")
require.NoError(t, err)
cfg, err := ReadConfiguration()
require.NoError(t, err)
require.Equal(t, "1", cfg.YouTubeApiKey)
require.Equal(t, "2", cfg.VimeoApiKey)
require.Equal(t, "3", cfg.PatreonClientId)
require.Equal(t, "4", cfg.PatreonSecret)
require.Equal(t, "5", cfg.PostgresConnectionURL)
require.Equal(t, "6", cfg.CookieSecret)
require.Equal(t, "7", cfg.PatreonRedirectURL)
require.Equal(t, "8", cfg.AssetsPath)
require.Equal(t, "9", cfg.TemplatesPath)
}
func TestReadEnv(t *testing.T) {
defer viper.Reset()
defer os.Clearenv()
os.Setenv("YOUTUBE_API_KEY", "11")
os.Setenv("VIMEO_API_KEY", "22")
os.Setenv("PATREON_CLIENT_ID", "33")
os.Setenv("PATREON_SECRET", "44")
os.Setenv("POSTGRES_CONNECTION_URL", "55")
os.Setenv("COOKIE_SECRET", "66")
os.Setenv("PATREON_REDIRECT_URL", "77")
os.Setenv("ASSETS_PATH", "88")
os.Setenv("TEMPLATES_PATH", "99")
cfg, err := ReadConfiguration()
require.NoError(t, err)
require.Equal(t, "11", cfg.YouTubeApiKey)
require.Equal(t, "22", cfg.VimeoApiKey)
require.Equal(t, "33", cfg.PatreonClientId)
require.Equal(t, "44", cfg.PatreonSecret)
require.Equal(t, "55", cfg.PostgresConnectionURL)
require.Equal(t, "66", cfg.CookieSecret)
require.Equal(t, "77", cfg.PatreonRedirectURL)
require.Equal(t, "88", cfg.AssetsPath)
require.Equal(t, "99", cfg.TemplatesPath)
}

114
pkg/feeds/feeds.go Normal file
View File

@@ -0,0 +1,114 @@
package feeds
import (
"fmt"
"time"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
"github.com/pkg/errors"
)
const (
maxPageSize = 150
)
type service struct {
id id
storage storage
builders map[api.Provider]builder
}
func (s *service) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) {
feed, err := parseURL(req.URL)
if err != nil {
return "", errors.Wrapf(err, "failed to create feed for URL: %s", req.URL)
}
// Make sure builder exists for this provider
_, ok := s.builders[feed.Provider]
if !ok {
return "", fmt.Errorf("failed to get builder for URL: %s", req.URL)
}
// Set default fields
feed.PageSize = api.DefaultPageSize
feed.Format = api.VideoFormat
feed.Quality = api.HighQuality
feed.FeatureLevel = api.DefaultFeatures
feed.LastAccess = time.Now().UTC()
if identity.FeatureLevel > 0 {
feed.Quality = req.Quality
feed.Format = req.Format
feed.FeatureLevel = identity.FeatureLevel
feed.PageSize = req.PageSize
if feed.PageSize > maxPageSize {
feed.PageSize = maxPageSize
}
}
// Generate short id
hashId, err := s.id.Generate(feed)
if err != nil {
return "", errors.Wrap(err, "failed to generate id for feed")
}
feed.HashId = hashId
// Save to database
if err := s.storage.CreateFeed(feed); err != nil {
return "", errors.Wrap(err, "failed to save feed to database")
}
return hashId, nil
}
func (s *service) GetFeed(hashId string) (*itunes.Podcast, error) {
feed, err := s.GetMetadata(hashId)
if err != nil {
return nil, err
}
builder, ok := s.builders[feed.Provider]
if !ok {
return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashId)
}
return builder.Build(feed)
}
func (s *service) GetMetadata(hashId string) (*api.Feed, error) {
return s.storage.GetFeed(hashId)
}
type feedOption func(*service)
func WithStorage(storage storage) feedOption {
return func(service *service) {
service.storage = storage
}
}
func WithIdGen(id id) feedOption {
return func(service *service) {
service.id = id
}
}
func WithBuilder(provider api.Provider, builder builder) feedOption {
return func(service *service) {
service.builders[provider] = builder
}
}
func NewFeedService(opts ...feedOption) *service {
svc := &service{}
svc.builders = make(map[api.Provider]builder)
for _, fn := range opts {
fn(svc)
}
return svc
}

72
pkg/feeds/feeds_test.go Normal file
View File

@@ -0,0 +1,72 @@
//go:generate mockgen -source=interfaces.go -destination=interfaces_mock_test.go -package=feeds
package feeds
import (
"testing"
"github.com/golang/mock/gomock"
"github.com/mxpv/podsync/pkg/api"
"github.com/stretchr/testify/require"
)
func TestService_CreateFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
id := NewMockid(ctrl)
id.EXPECT().Generate(gomock.Any()).Times(1).Return("123", nil)
storage := NewMockstorage(ctrl)
storage.EXPECT().CreateFeed(gomock.Any()).Times(1).Return(nil)
s := service{
id: id,
storage: storage,
builders: map[api.Provider]builder{api.Youtube: nil},
}
req := &api.CreateFeedRequest{
URL: "youtube.com/channel/123",
PageSize: 50,
Quality: api.HighQuality,
Format: api.VideoFormat,
}
hashId, err := s.CreateFeed(req, &api.Identity{})
require.NoError(t, err)
require.Equal(t, "123", hashId)
}
func TestService_GetFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
feed := &api.Feed{Provider: api.Youtube}
storage := NewMockstorage(ctrl)
storage.EXPECT().GetFeed("123").Times(1).Return(feed, nil)
bld := NewMockbuilder(ctrl)
bld.EXPECT().Build(feed).Return(nil, nil)
s := service{
storage: storage,
builders: map[api.Provider]builder{api.Youtube: bld},
}
_, err := s.GetFeed("123")
require.NoError(t, err)
}
func TestService_GetMetadata(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
storage := NewMockstorage(ctrl)
storage.EXPECT().GetFeed("123").Times(1).Return(&api.Feed{}, nil)
s := service{storage: storage}
_, err := s.GetMetadata("123")
require.NoError(t, err)
}

19
pkg/feeds/interfaces.go Normal file
View File

@@ -0,0 +1,19 @@
package feeds
import (
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
)
type id interface {
Generate(feed *api.Feed) (string, error)
}
type storage interface {
CreateFeed(feed *api.Feed) error
GetFeed(hashId string) (*api.Feed, error)
}
type builder interface {
Build(feed *api.Feed) (podcast *itunes.Podcast, err error)
}

View File

@@ -0,0 +1,131 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: interfaces.go
package feeds
import (
gomock "github.com/golang/mock/gomock"
podcast "github.com/mxpv/podcast"
api "github.com/mxpv/podsync/pkg/api"
reflect "reflect"
)
// Mockid is a mock of id interface
type Mockid struct {
ctrl *gomock.Controller
recorder *MockidMockRecorder
}
// MockidMockRecorder is the mock recorder for Mockid
type MockidMockRecorder struct {
mock *Mockid
}
// NewMockid creates a new mock instance
func NewMockid(ctrl *gomock.Controller) *Mockid {
mock := &Mockid{ctrl: ctrl}
mock.recorder = &MockidMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (_m *Mockid) EXPECT() *MockidMockRecorder {
return _m.recorder
}
// Generate mocks base method
func (_m *Mockid) Generate(feed *api.Feed) (string, error) {
ret := _m.ctrl.Call(_m, "Generate", feed)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Generate indicates an expected call of Generate
func (_mr *MockidMockRecorder) Generate(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Generate", reflect.TypeOf((*Mockid)(nil).Generate), arg0)
}
// Mockstorage is a mock of storage interface
type Mockstorage struct {
ctrl *gomock.Controller
recorder *MockstorageMockRecorder
}
// MockstorageMockRecorder is the mock recorder for Mockstorage
type MockstorageMockRecorder struct {
mock *Mockstorage
}
// NewMockstorage creates a new mock instance
func NewMockstorage(ctrl *gomock.Controller) *Mockstorage {
mock := &Mockstorage{ctrl: ctrl}
mock.recorder = &MockstorageMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (_m *Mockstorage) EXPECT() *MockstorageMockRecorder {
return _m.recorder
}
// CreateFeed mocks base method
func (_m *Mockstorage) CreateFeed(feed *api.Feed) error {
ret := _m.ctrl.Call(_m, "CreateFeed", feed)
ret0, _ := ret[0].(error)
return ret0
}
// CreateFeed indicates an expected call of CreateFeed
func (_mr *MockstorageMockRecorder) CreateFeed(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "CreateFeed", reflect.TypeOf((*Mockstorage)(nil).CreateFeed), arg0)
}
// GetFeed mocks base method
func (_m *Mockstorage) GetFeed(hashId string) (*api.Feed, error) {
ret := _m.ctrl.Call(_m, "GetFeed", hashId)
ret0, _ := ret[0].(*api.Feed)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetFeed indicates an expected call of GetFeed
func (_mr *MockstorageMockRecorder) GetFeed(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), arg0)
}
// Mockbuilder is a mock of builder interface
type Mockbuilder struct {
ctrl *gomock.Controller
recorder *MockbuilderMockRecorder
}
// MockbuilderMockRecorder is the mock recorder for Mockbuilder
type MockbuilderMockRecorder struct {
mock *Mockbuilder
}
// NewMockbuilder creates a new mock instance
func NewMockbuilder(ctrl *gomock.Controller) *Mockbuilder {
mock := &Mockbuilder{ctrl: ctrl}
mock.recorder = &MockbuilderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (_m *Mockbuilder) EXPECT() *MockbuilderMockRecorder {
return _m.recorder
}
// Build mocks base method
func (_m *Mockbuilder) Build(feed *api.Feed) (*podcast.Podcast, error) {
ret := _m.ctrl.Call(_m, "Build", feed)
ret0, _ := ret[0].(*podcast.Podcast)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Build indicates an expected call of Build
func (_mr *MockbuilderMockRecorder) Build(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Build", reflect.TypeOf((*Mockbuilder)(nil).Build), arg0)
}

150
pkg/feeds/url.go Normal file
View File

@@ -0,0 +1,150 @@
package feeds
import (
"net/url"
"strings"
"github.com/mxpv/podsync/pkg/api"
"github.com/pkg/errors"
)
func parseURL(link string) (*api.Feed, error) {
if !strings.HasPrefix(link, "http") {
link = "https://" + link
}
parsed, err := url.Parse(link)
if err != nil {
err = errors.Wrapf(err, "failed to parse url: %s", link)
return nil, err
}
feed := &api.Feed{}
if strings.HasSuffix(parsed.Host, "youtube.com") {
kind, id, err := parseYoutubeURL(parsed)
if err != nil {
return nil, err
}
feed.Provider = api.Youtube
feed.LinkType = kind
feed.ItemId = id
return feed, nil
}
if strings.HasSuffix(parsed.Host, "vimeo.com") {
kind, id, err := parseVimeoURL(parsed)
if err != nil {
return nil, err
}
feed.Provider = api.Vimeo
feed.LinkType = kind
feed.ItemId = id
return feed, nil
}
return nil, errors.New("unsupported URL host")
}
func parseYoutubeURL(parsed *url.URL) (kind api.LinkType, id string, err error) {
path := parsed.EscapedPath()
// https://www.youtube.com/playlist?list=PLCB9F975ECF01953C
if strings.HasPrefix(path, "/playlist") {
kind = api.Playlist
id = parsed.Query().Get("list")
if id != "" {
return
}
err = errors.New("invalid playlist link")
return
}
// - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og
// - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos
if strings.HasPrefix(path, "/channel") {
kind = api.Channel
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 2 {
err = errors.New("invalid youtube channel link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
// - https://www.youtube.com/user/fxigr1
if strings.HasPrefix(path, "/user") {
kind = api.User
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 2 {
err = errors.New("invalid user link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
err = errors.New("unsupported link format")
return
}
func parseVimeoURL(parsed *url.URL) (kind api.LinkType, id string, err error) {
parts := strings.Split(parsed.EscapedPath(), "/")
if len(parts) <= 1 {
err = errors.New("invalid vimeo link path")
return
}
if parts[1] == "groups" {
kind = api.Group
} else if parts[1] == "channels" {
kind = api.Channel
} else {
kind = api.User
}
if kind == api.Group || kind == api.Channel {
if len(parts) <= 2 {
err = errors.New("invalid channel link")
return
}
id = parts[2]
if id == "" {
err = errors.New("invalid id")
}
return
}
if kind == api.User {
id = parts[1]
if id == "" {
err = errors.New("invalid id")
}
return
}
err = errors.New("unsupported link format")
return
}

107
pkg/feeds/url_test.go Normal file
View File

@@ -0,0 +1,107 @@
package feeds
import (
"net/url"
"testing"
"github.com/mxpv/podsync/pkg/api"
"github.com/stretchr/testify/require"
)
func TestParseYoutubeURL_Playlist(t *testing.T) {
link, _ := url.ParseRequestURI("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C")
kind, id, err := parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.Playlist, kind)
require.Equal(t, "PLCB9F975ECF01953C", id)
}
func TestParseYoutubeURL_Channel(t *testing.T) {
link, _ := url.ParseRequestURI("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og")
kind, id, err := parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "UC5XPnUk8Vvv_pWslhwom6Og", id)
link, _ = url.ParseRequestURI("https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos")
kind, id, err = parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id)
}
func TestParseYoutubeURL_User(t *testing.T) {
link, _ := url.ParseRequestURI("https://youtube.com/user/fxigr1")
kind, id, err := parseYoutubeURL(link)
require.NoError(t, err)
require.Equal(t, api.User, kind)
require.Equal(t, "fxigr1", id)
}
func TestParseYoutubeURL_InvalidLink(t *testing.T) {
link, _ := url.ParseRequestURI("https://www.youtube.com/user///")
_, _, err := parseYoutubeURL(link)
require.Error(t, err)
link, _ = url.ParseRequestURI("https://www.youtube.com/channel//videos")
_, _, err = parseYoutubeURL(link)
require.Error(t, err)
}
func TestParseVimeoURL_Group(t *testing.T) {
link, _ := url.ParseRequestURI("https://vimeo.com/groups/109")
kind, id, err := parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
link, _ = url.ParseRequestURI("http://vimeo.com/groups/109")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
link, _ = url.ParseRequestURI("http://www.vimeo.com/groups/109")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
link, _ = url.ParseRequestURI("https://vimeo.com/groups/109/videos/")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Group, kind)
require.Equal(t, "109", id)
}
func TestParseVimeoURL_Channel(t *testing.T) {
link, _ := url.ParseRequestURI("https://vimeo.com/channels/staffpicks")
kind, id, err := parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "staffpicks", id)
link, _ = url.ParseRequestURI("http://vimeo.com/channels/staffpicks/146224925")
kind, id, err = parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.Channel, kind)
require.Equal(t, "staffpicks", id)
}
func TestParseVimeoURL_User(t *testing.T) {
link, _ := url.ParseRequestURI("https://vimeo.com/awhitelabelproduct")
kind, id, err := parseVimeoURL(link)
require.NoError(t, err)
require.Equal(t, api.User, kind)
require.Equal(t, "awhitelabelproduct", id)
}
func TestParseVimeoURL_InvalidLink(t *testing.T) {
link, _ := url.ParseRequestURI("http://www.apple.com")
_, _, err := parseVimeoURL(link)
require.Error(t, err)
link, _ = url.ParseRequestURI("http://www.vimeo.com")
_, _, err = parseVimeoURL(link)
require.Error(t, err)
}

32
pkg/id/hashids.go Normal file
View File

@@ -0,0 +1,32 @@
package id
import (
"hash/fnv"
"time"
"github.com/mxpv/podsync/pkg/api"
"github.com/ventu-io/go-shortid"
)
type hashId struct {
sid *shortid.Shortid
}
func hashString(s string) int {
h := fnv.New32a()
h.Write([]byte(s))
return int(h.Sum32())
}
func (h *hashId) Generate(feed *api.Feed) (string, error) {
return h.sid.Generate()
}
func NewIdGenerator() (*hashId, error) {
sid, err := shortid.New(1, shortid.DefaultABC, uint64(time.Now().UnixNano()))
if err != nil {
return nil, err
}
return &hashId{sid}, nil
}

19
pkg/id/hashids_test.go Normal file
View File

@@ -0,0 +1,19 @@
package id
import (
"testing"
"github.com/mxpv/podsync/pkg/api"
"github.com/stretchr/testify/require"
)
func TestEncode(t *testing.T) {
hid, err := NewIdGenerator()
require.NoError(t, err)
feed := &api.Feed{}
hash, err := hid.Generate(feed)
require.NoError(t, err)
require.NotEmpty(t, hash)
}

263
pkg/server/server.go Normal file
View File

@@ -0,0 +1,263 @@
package server
import (
"crypto/rand"
"encoding/base64"
"encoding/json"
"log"
"net/http"
"path"
"github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin"
"github.com/mxpv/patreon-go"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
"github.com/mxpv/podsync/pkg/config"
"golang.org/x/oauth2"
)
const (
campaignId = "278915"
identitySessionKey = "identity"
)
type feed interface {
CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error)
GetFeed(hashId string) (*itunes.Podcast, error)
GetMetadata(hashId string) (*api.Feed, error)
}
func MakeHandlers(feed feed, cfg *config.AppConfig) http.Handler {
r := gin.New()
r.Use(gin.Recovery())
store := sessions.NewCookieStore([]byte(cfg.CookieSecret))
r.Use(sessions.Sessions("podsync", store))
// Static files + HTML
log.Printf("using assets path: %s", cfg.AssetsPath)
if cfg.AssetsPath != "" {
r.Static("/assets", cfg.AssetsPath)
}
log.Printf("using templates path: %s", cfg.TemplatesPath)
if cfg.TemplatesPath != "" {
r.LoadHTMLGlob(path.Join(cfg.TemplatesPath, "*.html"))
}
conf := &oauth2.Config{
ClientID: cfg.PatreonClientId,
ClientSecret: cfg.PatreonSecret,
RedirectURL: cfg.PatreonRedirectURL,
Scopes: []string{"users", "pledges-to-me", "my-campaign"},
Endpoint: oauth2.Endpoint{
AuthURL: patreon.AuthorizationURL,
TokenURL: patreon.AccessTokenURL,
},
}
r.GET("/", func(c *gin.Context) {
s := sessions.Default(c)
identity := &api.Identity{
FeatureLevel: api.DefaultFeatures,
}
buf, ok := s.Get(identitySessionKey).(string)
if ok {
// We are failed to deserialize Identity structure, do cleanup, force user to login again
if err := json.Unmarshal([]byte(buf), identity); err != nil {
s.Clear()
s.Save()
}
}
c.HTML(http.StatusOK, "index.html", identity)
})
r.GET("/login", func(c *gin.Context) {
state := randToken()
s := sessions.Default(c)
s.Set("state", state)
s.Save()
authURL := conf.AuthCodeURL(state)
c.Redirect(http.StatusFound, authURL)
})
r.GET("/logout", func(c *gin.Context) {
s := sessions.Default(c)
s.Clear()
s.Save()
c.Redirect(http.StatusFound, "/")
})
r.GET("/patreon", func(c *gin.Context) {
// Validate session state
s := sessions.Default(c)
state := s.Get("state")
if state != c.Query("state") {
c.String(http.StatusUnauthorized, "invalid state")
return
}
// Exchange code with tokens
token, err := conf.Exchange(c.Request.Context(), c.Query("code"))
if err != nil {
c.String(http.StatusBadRequest, err.Error())
return
}
// Create Patreon client
tc := conf.Client(c.Request.Context(), token)
client := patreon.NewClient(tc)
// Query user info from Patreon
user, err := client.FetchUser()
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
}
// Determine feature level
level := api.DefaultFeatures
amount := 0
for _, item := range user.Included.Items {
pledge, ok := item.(*patreon.Pledge)
if ok {
amount += pledge.Attributes.AmountCents
}
}
if amount >= 100 {
level = api.ExtendedFeatures
}
identity := &api.Identity{
UserId: user.Data.Id,
FullName: user.Data.Attributes.FullName,
Email: user.Data.Attributes.Email,
ProfileURL: user.Data.Attributes.URL,
FeatureLevel: level,
}
// Serialize identity and return cookies
buf, err := json.Marshal(identity)
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
}
s.Clear()
s.Set(identitySessionKey, string(buf))
s.Save()
c.Redirect(http.StatusFound, "/")
})
// GET /robots.txt
r.GET("/robots.txt", func(c *gin.Context) {
c.String(http.StatusOK, `User-agent: *
Allow: /$
Disallow: /
Host: www.podsync.net`)
})
// REST API
r.GET("/api/ping", func(c *gin.Context) {
c.String(http.StatusOK, "ok")
})
r.POST("/api/create", func(c *gin.Context) {
req := &api.CreateFeedRequest{}
if err := c.BindJSON(req); err != nil {
c.JSON(badRequest(err))
return
}
s := sessions.Default(c)
identity := &api.Identity{
FeatureLevel: api.DefaultFeatures,
}
buf, ok := s.Get(identitySessionKey).(string)
if ok {
// We are failed to deserialize Identity structure, do cleanup, force user to login again
if err := json.Unmarshal([]byte(buf), identity); err != nil {
s.Clear()
s.Save()
}
}
hashId, err := feed.CreateFeed(req, identity)
if err != nil {
c.JSON(internalError(err))
return
}
c.JSON(http.StatusOK, gin.H{"id": hashId})
})
r.NoRoute(func(c *gin.Context) {
hashId := c.Request.URL.Path[1:]
if hashId == "" || len(hashId) > 12 {
c.String(http.StatusBadRequest, "invalid feed id")
return
}
podcast, err := feed.GetFeed(hashId)
if err != nil {
code := http.StatusInternalServerError
if err == api.ErrNotFound {
code = http.StatusNotFound
} else {
log.Printf("server error (hash id: %s): %v", hashId, err)
}
c.String(code, err.Error())
return
}
c.Data(http.StatusOK, "application/rss+xml; charset=UTF-8", podcast.Bytes())
})
r.GET("/api/metadata/:hashId", func(c *gin.Context) {
hashId := c.Param("hashId")
if hashId == "" || len(hashId) > 12 {
c.String(http.StatusBadRequest, "invalid feed id")
return
}
feed, err := feed.GetMetadata(hashId)
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
}
c.JSON(http.StatusOK, feed)
})
return r
}
func badRequest(err error) (int, interface{}) {
return http.StatusBadRequest, gin.H{"error": err.Error()}
}
func internalError(err error) (int, interface{}) {
log.Printf("server error: %v", err)
return http.StatusInternalServerError, gin.H{"error": err.Error()}
}
func randToken() string {
b := make([]byte, 32)
rand.Read(b)
return base64.StdEncoding.EncodeToString(b)
}

View File

@@ -0,0 +1,73 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: server.go
package server
import (
gomock "github.com/golang/mock/gomock"
podcast "github.com/mxpv/podcast"
api "github.com/mxpv/podsync/pkg/api"
reflect "reflect"
)
// Mockfeed is a mock of feed interface
type Mockfeed struct {
ctrl *gomock.Controller
recorder *MockfeedMockRecorder
}
// MockfeedMockRecorder is the mock recorder for Mockfeed
type MockfeedMockRecorder struct {
mock *Mockfeed
}
// NewMockfeed creates a new mock instance
func NewMockfeed(ctrl *gomock.Controller) *Mockfeed {
mock := &Mockfeed{ctrl: ctrl}
mock.recorder = &MockfeedMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (_m *Mockfeed) EXPECT() *MockfeedMockRecorder {
return _m.recorder
}
// CreateFeed mocks base method
func (_m *Mockfeed) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) {
ret := _m.ctrl.Call(_m, "CreateFeed", req, identity)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// CreateFeed indicates an expected call of CreateFeed
func (_mr *MockfeedMockRecorder) CreateFeed(arg0, arg1 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "CreateFeed", reflect.TypeOf((*Mockfeed)(nil).CreateFeed), arg0, arg1)
}
// GetFeed mocks base method
func (_m *Mockfeed) GetFeed(hashId string) (*podcast.Podcast, error) {
ret := _m.ctrl.Call(_m, "GetFeed", hashId)
ret0, _ := ret[0].(*podcast.Podcast)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetFeed indicates an expected call of GetFeed
func (_mr *MockfeedMockRecorder) GetFeed(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetFeed", reflect.TypeOf((*Mockfeed)(nil).GetFeed), arg0)
}
// GetMetadata mocks base method
func (_m *Mockfeed) GetMetadata(hashId string) (*api.Feed, error) {
ret := _m.ctrl.Call(_m, "GetMetadata", hashId)
ret0, _ := ret[0].(*api.Feed)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetMetadata indicates an expected call of GetMetadata
func (_mr *MockfeedMockRecorder) GetMetadata(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetMetadata", reflect.TypeOf((*Mockfeed)(nil).GetMetadata), arg0)
}

133
pkg/server/server_test.go Normal file
View File

@@ -0,0 +1,133 @@
//go:generate mockgen -source=server.go -destination=server_mock_test.go -package=server
package server
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/golang/mock/gomock"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
"github.com/mxpv/podsync/pkg/config"
"github.com/stretchr/testify/require"
)
var cfg = &config.AppConfig{}
func TestCreateFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
req := &api.CreateFeedRequest{
URL: "https://youtube.com/channel/123",
PageSize: 55,
Quality: api.LowQuality,
Format: api.AudioFormat,
}
feed := NewMockfeed(ctrl)
feed.EXPECT().CreateFeed(gomock.Eq(req), gomock.Any()).Times(1).Return("456", nil)
srv := httptest.NewServer(MakeHandlers(feed, cfg))
defer srv.Close()
query := `{"url": "https://youtube.com/channel/123", "page_size": 55, "quality": "low", "format": "audio"}`
resp, err := http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
require.JSONEq(t, `{"id": "456"}`, readBody(t, resp))
}
func TestCreateInvalidFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
srv := httptest.NewServer(MakeHandlers(NewMockfeed(ctrl), cfg))
defer srv.Close()
query := `{}`
resp, err := http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "not a url", "page_size": 55, "quality": "low", "format": "audio"}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "https://youtube.com/channel/123", "page_size": 1, "quality": "low", "format": "audio"}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "https://youtube.com/channel/123", "page_size": 151, "quality": "low", "format": "audio"}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "xyz", "format": "audio"}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "low", "format": "xyz"}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "low", "format": ""}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "", "format": "audio"}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
}
func TestGetFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
podcast := itunes.New("", "", "", nil, nil)
feed := NewMockfeed(ctrl)
feed.EXPECT().GetFeed("123").Return(&podcast, nil)
srv := httptest.NewServer(MakeHandlers(feed, cfg))
defer srv.Close()
resp, err := http.Get(srv.URL + "/123")
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
}
func TestGetMetadata(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
feed := NewMockfeed(ctrl)
feed.EXPECT().GetMetadata("123").Times(1).Return(&api.Feed{}, nil)
srv := httptest.NewServer(MakeHandlers(feed, cfg))
defer srv.Close()
resp, err := http.Get(srv.URL + "/api/metadata/123")
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
}
func readBody(t *testing.T, resp *http.Response) string {
buf, err := ioutil.ReadAll(resp.Body)
defer resp.Body.Close()
require.NoError(t, err)
return string(buf)
}

76
pkg/storage/pg.go Normal file
View File

@@ -0,0 +1,76 @@
package storage
import (
"log"
"net"
"strings"
"time"
"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
"github.com/go-pg/pg"
"github.com/mxpv/podsync/pkg/api"
"github.com/pkg/errors"
)
type PgConfig struct {
ConnectionUrl string `yaml:"connectionUrl"`
}
type PgStorage struct {
db *pg.DB
}
func (p *PgStorage) CreateFeed(feed *api.Feed) error {
feed.LastAccess = time.Now().UTC()
_, err := p.db.Model(feed).OnConflict("DO NOTHING").Insert()
if err != nil {
return errors.Wrap(err, "failed to create feed")
}
return nil
}
func (p *PgStorage) GetFeed(hashId string) (*api.Feed, error) {
lastAccess := time.Now().UTC()
feed := &api.Feed{}
_, err := p.db.Model(feed).
Set("last_access = ?", lastAccess).
Where("hash_id = ?", hashId).
Returning("*").
Update()
return feed, err
}
func NewPgStorage(config *PgConfig) (*PgStorage, error) {
opts, err := pg.ParseURL(config.ConnectionUrl)
if err != nil {
return nil, err
}
// If host format is "projection:region:host", than use Google SQL Proxy
// See https://github.com/go-pg/pg/issues/576
if strings.Count(opts.Addr, ":") == 2 {
log.Print("using GCP SQL proxy")
opts.Dialer = func(network, addr string) (net.Conn, error) {
return proxy.Dial(addr)
}
}
db := pg.Connect(opts)
// Check database connectivity
if _, err := db.ExecOne("SELECT 1"); err != nil {
db.Close()
return nil, errors.Wrap(err, "failed to check database connectivity")
}
log.Print("running update script")
if _, err := db.Exec(installScript); err != nil {
return nil, errors.Wrap(err, "failed to upgrade database structure")
}
storage := &PgStorage{db: db}
return storage, nil
}

38
pkg/storage/pg_sql.go Normal file
View File

@@ -0,0 +1,38 @@
package storage
const installScript = `
BEGIN;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'provider') THEN
CREATE TYPE provider AS ENUM ('youtube', 'vimeo');
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'link_type') THEN
CREATE TYPE link_type AS ENUM ('channel', 'playlist', 'user', 'group');
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'quality') THEN
CREATE TYPE quality AS ENUM ('high', 'low');
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'format') THEN
CREATE TYPE format AS ENUM ('audio', 'video');
END IF;
END
$$;
CREATE TABLE IF NOT EXISTS feeds (
id BIGSERIAL PRIMARY KEY,
hash_id VARCHAR(12) NOT NULL CHECK (hash_id <> '') UNIQUE,
user_id VARCHAR(32) NULL,
item_id VARCHAR(32) NOT NULL CHECK (item_id <> ''),
provider provider NOT NULL,
link_type link_type NOT NULL,
page_size INT NOT NULL DEFAULT 50,
format format NOT NULL DEFAULT 'video',
quality quality NOT NULL DEFAULT 'high',
feature_level INT NOT NULL DEFAULT 0,
last_access timestamp WITHOUT TIME ZONE NOT NULL
);
COMMIT;
`

106
pkg/storage/pg_test.go Normal file
View File

@@ -0,0 +1,106 @@
package storage
import (
"testing"
"github.com/mxpv/podsync/pkg/api"
"github.com/stretchr/testify/require"
)
func TestPgStorage_CreateFeed(t *testing.T) {
feed := &api.Feed{
HashId: "xyz",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
err := client.CreateFeed(feed)
require.NoError(t, err)
require.True(t, feed.Id > 0)
}
func TestPgStorage_CreateFeedWithDuplicate(t *testing.T) {
feed := &api.Feed{
HashId: "123",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
err := client.CreateFeed(feed)
require.NoError(t, err)
// Ensure 1 record
count, err := client.db.Model(&api.Feed{}).Count()
require.NoError(t, err)
require.Equal(t, 1, count)
// Insert duplicated feed
err = client.CreateFeed(feed)
require.NoError(t, err)
// Check no duplicates inserted
count, err = client.db.Model(&api.Feed{}).Count()
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestPgStorage_GetFeed(t *testing.T) {
feed := &api.Feed{
HashId: "xyz",
UserId: "123",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
client.CreateFeed(feed)
out, err := client.GetFeed("xyz")
require.NoError(t, err)
require.Equal(t, feed.Id, out.Id)
}
func TestPgStorage_UpdateLastAccess(t *testing.T) {
feed := &api.Feed{
HashId: "xyz",
UserId: "123",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
}
client := createClient(t)
err := client.CreateFeed(feed)
require.NoError(t, err)
lastAccess := feed.LastAccess
require.True(t, lastAccess.Unix() > 0)
last, err := client.GetFeed("xyz")
require.NoError(t, err)
require.NotEmpty(t, last.HashId)
require.NotEmpty(t, last.UserId)
require.NotEmpty(t, last.Provider)
require.NotEmpty(t, last.LinkType)
require.NotEmpty(t, last.ItemId)
require.True(t, last.LastAccess.UnixNano() > lastAccess.UnixNano())
}
const TestDatabaseConnectionUrl = "postgres://postgres:@localhost/podsync?sslmode=disable"
func createClient(t *testing.T) *PgStorage {
pg, err := NewPgStorage(&PgConfig{ConnectionUrl: TestDatabaseConnectionUrl})
require.NoError(t, err)
_, err = pg.db.Model(&api.Feed{}).Where("1=1").Delete()
require.NoError(t, err)
return pg
}

206
pkg/storage/redis.go Normal file
View File

@@ -0,0 +1,206 @@
package storage
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/go-redis/redis"
"github.com/mxpv/podsync/pkg/api"
"github.com/pkg/errors"
)
const expiration = 24 * time.Hour * 90
// Backward compatible Redis storage for feeds
type RedisStorage struct {
client *redis.Client
}
func (r *RedisStorage) parsePageSize(m map[string]string) (int, error) {
str, ok := m["pagesize"]
if !ok {
return 50, nil
}
size, err := strconv.ParseInt(str, 10, 32)
if err != nil {
return 0, err
}
if size > 150 {
return 0, errors.New("invalid page size")
}
return int(size), nil
}
func (r *RedisStorage) parseFormat(m map[string]string) (api.Format, api.Quality, error) {
quality, ok := m["quality"]
if !ok {
return api.VideoFormat, api.HighQuality, nil
}
if quality == "VideoHigh" {
return api.VideoFormat, api.HighQuality, nil
} else if quality == "VideoLow" {
return api.VideoFormat, api.LowQuality, nil
} else if quality == "AudioHigh" {
return api.AudioFormat, api.HighQuality, nil
} else if quality == "AudioLow" {
return api.AudioFormat, api.LowQuality, nil
}
return "", "", fmt.Errorf("unsupported formmat %s", quality)
}
func (r *RedisStorage) GetFeed(hashId string) (*api.Feed, error) {
result, err := r.client.HGetAll(hashId).Result()
if err != nil {
return nil, errors.Wrapf(err, "failed to query feed with id %s", hashId)
}
if len(result) == 0 {
return nil, api.ErrNotFound
}
// Expire after 3 month if no use
if err := r.client.Expire(hashId, expiration).Err(); err != nil {
return nil, errors.Wrap(err, "failed query update feed")
}
feed := &api.Feed{
PageSize: api.DefaultPageSize,
Quality: api.DefaultQuality,
Format: api.DefaultFormat,
HashId: hashId,
LastAccess: time.Now().UTC(),
}
m := make(map[string]string, len(result))
for key, val := range result {
m[strings.ToLower(key)] = val
}
// Unpack provider and link type
provider := m["provider"]
linkType := m["type"]
if strings.EqualFold(provider, "youtube") {
feed.Provider = api.Youtube
if strings.EqualFold(linkType, "channel") {
feed.LinkType = api.Channel
} else if strings.EqualFold(linkType, "playlist") {
feed.LinkType = api.Playlist
} else if strings.EqualFold(linkType, "user") {
feed.LinkType = api.User
} else {
return nil, fmt.Errorf("unsupported yt link type %s", linkType)
}
} else if strings.EqualFold(provider, "vimeo") {
feed.Provider = api.Vimeo
if strings.EqualFold(linkType, "channel") {
feed.LinkType = api.Channel
} else if strings.EqualFold(linkType, "user") {
feed.LinkType = api.User
} else if strings.EqualFold(linkType, "group") {
feed.LinkType = api.Group
} else {
return nil, fmt.Errorf("unsupported vimeo link type %s", linkType)
}
} else {
return nil, errors.New("unsupported provider")
}
// Unpack item id
id, ok := m["id"]
if !ok || id == "" {
return nil, errors.New("failed to unpack item id")
}
feed.ItemId = id
// Fetch user id
patreonId, ok := m["patreonid"]
if ok {
feed.UserId = patreonId
}
// Unpack page size
pageSize, err := r.parsePageSize(m)
if err != nil {
return nil, err
}
if patreonId == "" && pageSize > 50 {
return nil, errors.New("wrong feed data")
}
// Parse feed's format and quality
format, quality, err := r.parseFormat(m)
if err != nil {
return nil, err
}
feed.PageSize = pageSize
feed.Format = format
feed.Quality = quality
return feed, nil
}
func (r *RedisStorage) CreateFeed(feed *api.Feed) error {
fields := map[string]interface{}{
"provider": string(feed.Provider),
"type": string(feed.LinkType),
"id": feed.ItemId,
"patreonid": feed.UserId,
"pagesize": feed.PageSize,
}
if feed.Format == api.VideoFormat {
if feed.Quality == api.HighQuality {
fields["quality"] = "VideoHigh"
} else {
fields["quality"] = "VideoLow"
}
} else {
if feed.Quality == api.HighQuality {
fields["quality"] = "AudioHigh"
} else {
fields["quality"] = "AudioLow"
}
}
if err := r.client.HMSet(feed.HashId, fields).Err(); err != nil {
return errors.Wrap(err, "failed to save feed")
}
return r.client.Expire(feed.HashId, expiration).Err()
}
func (r *RedisStorage) keys() ([]string, error) {
return r.client.Keys("*").Result()
}
func NewRedisStorage(redisUrl string) (*RedisStorage, error) {
opts, err := redis.ParseURL(redisUrl)
if err != nil {
return nil, err
}
client := redis.NewClient(opts)
if err := client.Ping().Err(); err != nil {
return nil, err
}
return &RedisStorage{client}, nil
}

71
pkg/storage/redis_test.go Normal file
View File

@@ -0,0 +1,71 @@
package storage
import (
"strconv"
"testing"
"time"
"github.com/mxpv/podsync/pkg/api"
"github.com/stretchr/testify/require"
)
func TestRedisStorage_GetFeed(t *testing.T) {
t.Skip("run redis tests manually")
client := createRedisClient(t)
keys, err := client.keys()
require.NoError(t, err)
require.True(t, len(keys) > 0)
for idx, key := range keys {
if key == "keygen" {
continue
}
feed, err := client.GetFeed(key)
require.NoError(t, err, "feed %s (id = %d) failed", key, idx)
require.NotNil(t, feed)
}
}
func TestRedisStorage_CreateFeed(t *testing.T) {
t.Skip("run redis tests manually")
client := createRedisClient(t)
hashId := strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
err := client.CreateFeed(&api.Feed{
HashId: hashId,
UserId: "321",
Provider: api.Youtube,
LinkType: api.Channel,
ItemId: "123",
PageSize: 45,
Quality: api.LowQuality,
Format: api.AudioFormat,
})
require.NoError(t, err)
feed, err := client.GetFeed(hashId)
require.NoError(t, err)
require.Equal(t, hashId, feed.HashId)
require.Equal(t, "321", feed.UserId)
require.Equal(t, api.Youtube, feed.Provider)
require.Equal(t, api.Channel, feed.LinkType)
require.Equal(t, "123", feed.ItemId)
require.Equal(t, 45, feed.PageSize)
require.Equal(t, api.LowQuality, feed.Quality)
require.Equal(t, api.AudioFormat, feed.Format)
}
func createRedisClient(t *testing.T) *RedisStorage {
client, err := NewRedisStorage("redis://localhost")
require.NoError(t, err)
return client
}

View File

@@ -1,3 +0,0 @@
{
"directory": "wwwroot/lib"
}

View File

@@ -1,104 +0,0 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Security.Claims;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Podsync.Helpers;
using Podsync.Services;
using Podsync.Services.Links;
using Podsync.Services.Rss;
using Podsync.Services.Rss.Contracts;
using Podsync.Services.Storage;
namespace Podsync.Controllers
{
[Route("feed")]
[ServiceFilter(typeof(HandleExceptionAttribute), IsReusable = true)]
public class FeedController : Controller
{
private static readonly IDictionary<string, string> Extensions = new Dictionary<string, string>
{
["video/mp4"] = "mp4",
["audio/mp4"] = "m4a"
};
private readonly ILinkService _linkService;
private readonly IFeedService _feedService;
public FeedController(IRssBuilder rssBuilder, ILinkService linkService, IFeedService feedService)
{
_linkService = linkService;
_feedService = feedService;
}
[HttpPost]
[Route("create")]
[ValidateModelState]
public async Task<Uri> Create([FromBody] CreateFeedRequest request)
{
var linkInfo = _linkService.Parse(new Uri(request.Url));
var feed = new FeedMetadata
{
Provider = linkInfo.Provider,
LinkType = linkInfo.LinkType,
Id = linkInfo.Id,
Quality = request.Quality ?? Constants.DefaultFormat,
PageSize = request.PageSize ?? Constants.DefaultPageSize
};
// Check if user eligible for Patreon features
var allowFeatures = User.EnablePatreonFeatures();
if (allowFeatures)
{
feed.PatreonId = User.GetClaim(ClaimTypes.NameIdentifier);
}
else
{
feed.Quality = Constants.DefaultFormat;
feed.PageSize = Constants.DefaultPageSize;
}
var feedId = await _feedService.Create(feed);
var url = _linkService.Feed(Request.GetBaseUrl(), feedId);
return url;
}
[HttpGet]
[Route("~/{feedId:length(4, 6)}")]
[ValidateModelState]
public async Task<IActionResult> Feed([Required] string feedId)
{
try
{
var body = await _feedService.Get(feedId, FixLinks);
return Content(body, "application/rss+xml; charset=UTF-8");
}
catch (KeyNotFoundException)
{
return NotFound($"ERROR: No feed with id {feedId}");
}
}
[NonAction]
private void FixLinks(string feedId, Feed feed)
{
var selfHost = Request.GetBaseUrl();
// Set atom link to this feed
// See https://validator.w3.org/feed/docs/warning/MissingAtomSelfLink.html
var selfLink = new Uri(selfHost, Request.Path);
feed.Channels.ForEach(x => x.AtomLink = selfLink);
// No magic here, just make download links to DownloadController.Download
feed.Channels.SelectMany(x => x.Items).ForEach(item =>
{
var ext = Extensions[item.ContentType];
item.DownloadLink = new Uri(selfHost, $"download/{feedId}/{item.Id}.{ext}");
});
}
}
}

View File

@@ -1,12 +0,0 @@
using Microsoft.AspNetCore.Mvc;
namespace Podsync.Controllers
{
public class HomeController : Controller
{
public IActionResult Index()
{
return View();
}
}
}

View File

@@ -1,45 +0,0 @@
using System;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Podsync.Services.Resolver;
using Podsync.Services.Storage;
namespace Podsync.Controllers
{
public class StatusController : Controller
{
private const string ErrorStatus = "ERROR";
private readonly IStorageService _storageService;
private readonly IResolverService _resolverService;
public StatusController(IStorageService storageService, IResolverService resolverService)
{
_storageService = storageService;
_resolverService = resolverService;
}
[HttpGet]
[Route("~/status")]
public async Task<string> Index()
{
var storageStatus = ErrorStatus;
try
{
var time = await _storageService.Ping();
storageStatus = time.ToString();
}
catch (Exception)
{
// Nothing to do
}
var resolverStatus = _resolverService.Version ?? ErrorStatus;
return $"Path: {Request.Path}\r\n" +
$"Redis: {storageStatus}\r\n" +
$"Resolve: {resolverStatus}";
}
}
}

View File

@@ -1,13 +0,0 @@
FROM microsoft/aspnetcore-build AS build
WORKDIR /workspace
COPY . .
RUN dotnet restore
RUN dotnet publish --configuration release --output ./bin/
FROM microsoft/aspnetcore:1.1
ENV ASPNETCORE_URLS http://*:8080
ENV ASPNETCORE_ENVIRONMENT Production
WORKDIR /app
COPY --from=build /workspace/bin/ .
EXPOSE 8080
ENTRYPOINT ["dotnet", "Podsync.dll"]

View File

@@ -1,76 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace Podsync.Helpers
{
internal static class EnumerableExtensions
{
public static IEnumerable<IEnumerable<T>> Chunk<T>(this IEnumerable<T> source, int chunkSize)
{
if (chunkSize < 1)
{
throw new ArgumentException("Chunk size must be positive", nameof(chunkSize));
}
while (source.Any())
{
yield return source.Take(chunkSize);
source = source.Skip(chunkSize);
}
}
public static bool SafeAny<T>(this IEnumerable<T> source)
{
if (source == null)
{
return false;
}
if (source.Any())
{
return true;
}
return false;
}
public static void ForEach<T>(this IEnumerable<T> source, Action<T> action)
{
if (source == null)
{
return;
}
foreach (var item in source)
{
action(item);
}
}
public static void AddTo<T>(this IEnumerable<T> collection, List<T> target)
{
target.AddRange(collection);
}
public static void SafeForEach<T>(this IEnumerable<T> source, Action<T> action)
{
if (source == null)
{
return;
}
foreach (var item in source)
{
try
{
action(item);
}
catch
{
// Nothing to do
}
}
}
}
}

View File

@@ -1,82 +0,0 @@
using System;
using System.Linq;
using System.Security.Claims;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Podsync.Services;
using Podsync.Services.Resolver;
namespace Podsync.Helpers
{
public static class Extensions
{
/// <summary>
/// Generates a fully qualified URL to the specified content by using the specified content path.
/// Converts a virtual (relative) path to an application absolute path.
/// See http://stackoverflow.com/questions/30755827/getting-absolute-urls-using-asp-net-core-mvc-6
/// </summary>
/// <param name="url"></param>
/// <param name="contentPath"></param>
/// <returns></returns>
public static string AbsoluteContent(this IUrlHelper url, string contentPath)
{
var request = url.ActionContext.HttpContext.Request;
var baseUri = new Uri($"{request.Scheme}://{request.Host.Value}");
var fullUri = new Uri(baseUri, url.Content(contentPath));
return fullUri.ToString();
}
public static Uri GetBaseUrl(this HttpRequest request)
{
return new Uri($"{request.Scheme}://{request.Host}");
}
private const string OwnerId = "2822191";
/// <summary>
/// Check if user eligible for advanced features
/// </summary>
/// <param name="user"></param>
/// <returns></returns>
public static bool EnablePatreonFeatures(this ClaimsPrincipal user)
{
if (!user.Identity.IsAuthenticated)
{
return false;
}
if (user.GetClaim(ClaimTypes.NameIdentifier) == OwnerId)
{
return true;
}
const int MinAmountCents = 100;
int amount;
if (int.TryParse(user.GetClaim(Constants.Patreon.AmountDonated), out amount))
{
return amount >= MinAmountCents;
}
return false;
}
public static string GetName(this ClaimsPrincipal claimsPrincipal)
{
return claimsPrincipal.GetClaim(ClaimTypes.Name)
?? claimsPrincipal.GetClaim(ClaimTypes.Email)
?? "noname :(";
}
public static string GetClaim(this ClaimsPrincipal claimsPrincipal, string type)
{
return claimsPrincipal.Claims.FirstOrDefault(x => x.Type == type)?.Value;
}
public static bool IsAudio(this ResolveFormat format)
{
return format == ResolveFormat.AudioHigh || format == ResolveFormat.AudioLow;
}
}
}

View File

@@ -1,34 +0,0 @@
using System;
using System.Net;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Filters;
using Microsoft.Extensions.Logging;
using Podsync.Services;
namespace Podsync.Helpers
{
public class HandleExceptionAttribute : ExceptionFilterAttribute
{
private readonly ILogger _logger;
public HandleExceptionAttribute(ILogger<HandleExceptionAttribute> logger)
{
_logger = logger;
}
public override void OnException(ExceptionContext context)
{
var exception = context.Exception;
if (exception is ArgumentNullException || exception is ArgumentException)
{
context.Result = new BadRequestObjectResult(exception.Message);
}
else
{
context.Result = new StatusCodeResult((int)HttpStatusCode.InternalServerError);
_logger.LogCritical(Constants.Events.UnhandledError, context.Exception, "Unhandled exception");
}
}
}
}

View File

@@ -1,73 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using Microsoft.Extensions.DependencyInjection;
namespace Podsync.Helpers
{
internal static class ServiceProviderExtensions
{
public static T CreateInstance<T>(this IServiceProvider serviceProvider)
{
return serviceProvider.CreateInstance<T>(typeof(T));
}
public static T CreateInstance<T>(this IServiceProvider serviceProvider, Type implementation)
{
return (T)serviceProvider.CreateInstance(implementation);
}
public static object CreateInstance(this IServiceProvider serviceProvider, Type type)
{
return ActivatorUtilities.CreateInstance(serviceProvider, type);
}
public static IEnumerable<Type> FindAllImplementationsOf(this IServiceProvider serviceProvider, Type interfaceType, Assembly assembly)
{
if (!interfaceType.GetTypeInfo().IsInterface)
{
throw new ArgumentException("T should be an interface");
}
return GetLoadableTypes(assembly).Where(type => IsAssignableFrom(interfaceType, type));
}
public static IEnumerable<Type> FindAllImplementationsOf<T>(this IServiceProvider serviceProvider, Assembly assembly)
{
return serviceProvider.FindAllImplementationsOf(typeof(T), assembly);
}
private static bool IsAssignableFrom(Type interfaceType, Type serviceType)
{
var serviceTypeInfo = serviceType.GetTypeInfo();
if (serviceTypeInfo.IsInterface || serviceTypeInfo.IsAbstract)
{
return false;
}
var interfaceTypeInfo = interfaceType.GetTypeInfo();
if (!interfaceTypeInfo.IsGenericType)
{
return interfaceType.IsAssignableFrom(serviceType);
}
return serviceType
.GetInterfaces()
.Where(type => type.GetTypeInfo().IsGenericType)
.Any(type => type.GetGenericTypeDefinition() == interfaceType);
}
private static IEnumerable<Type> GetLoadableTypes(Assembly assembly)
{
try
{
return assembly.GetTypes();
}
catch (ReflectionTypeLoadException e)
{
return e.Types.Where(x => x != null);
}
}
}
}

View File

@@ -1,28 +0,0 @@
using System.Linq;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Filters;
namespace Podsync.Helpers
{
/// <summary>
/// Validate the model state prior to action execution.
/// See http://www.strathweb.com/2015/06/action-filters-service-filters-type-filters-asp-net-5-mvc-6/
/// </summary>
public class ValidateModelStateAttribute : ActionFilterAttribute
{
private readonly bool _validateNotNull;
public ValidateModelStateAttribute(bool validateNotNull = true)
{
_validateNotNull = validateNotNull;
}
public override void OnActionExecuting(ActionExecutingContext context)
{
if (!context.ModelState.IsValid || _validateNotNull && context.ActionArguments.Any(p => p.Value == null))
{
context.Result = new BadRequestObjectResult(context.ModelState);
}
}
}
}

View File

@@ -1,90 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>netcoreapp1.1</TargetFramework>
<PreserveCompilationContext>true</PreserveCompilationContext>
<AssemblyName>Podsync</AssemblyName>
<OutputType>Exe</OutputType>
<PackageId>Podsync</PackageId>
<UserSecretsId>aspnet-Podsync-20161004104901</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<Compile Include="..\Shared\*.cs" Exclude="bin\**;obj\**;**\*.xproj;packages\**" />
<Content Update="wwwroot\**\*;Views\**\*;Areas\**\Views;appsettings.json;web.config;Dockerfile;youtube-dl">
<CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Google.Apis.YouTube.v3">
<Version>1.25.0.760</Version>
</PackageReference>
<PackageReference Include="Hashids.net" Version="1.2.2" />
<PackageReference Include="MedallionShell" Version="1.2.1" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Cookies">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.Authentication.OAuth">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.Diagnostics">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.HttpOverrides">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.Mvc">
<Version>1.1.3</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.Server.IISIntegration">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.StaticFiles">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.WebUtilities">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration.Json">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging.Console">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging.Debug">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions">
<Version>1.1.2</Version>
</PackageReference>
<PackageReference Include="Microsoft.NETCore.App">
<Version>1.1.1</Version>
</PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Web.BrowserLink.Loader" Version="14.1.0" />
<PackageReference Include="StackExchange.Redis">
<Version>1.2.3</Version>
</PackageReference>
<PackageReference Include="System.Xml.XmlSerializer" Version="4.3.0" />
</ItemGroup>
<Target Name="PrecompileScript" BeforeTargets="BeforeBuild">
<Exec Command="dotnet bundle" />
</Target>
<Target Name="PrepublishScript" BeforeTargets="PrepareForPublish">
<Exec Command="bower install" />
<Exec Command="dotnet bundle" />
</Target>
<ItemGroup>
<DotNetCliToolReference Include="BundlerMinifier.Core" Version="2.2.301" />
<DotNetCliToolReference Include="Microsoft.Extensions.SecretManager.Tools" Version="1.0.0-msbuild3-final" />
</ItemGroup>
</Project>

View File

@@ -1,20 +0,0 @@
using System.IO;
using Microsoft.AspNetCore.Hosting;
namespace Podsync
{
public class Program
{
public static void Main(string[] args)
{
var host = new WebHostBuilder()
.UseKestrel()
.UseContentRoot(Directory.GetCurrentDirectory())
.UseIISIntegration()
.UseStartup<Startup>()
.Build();
host.Run();
}
}
}

View File

@@ -1,27 +0,0 @@
{
"iisSettings": {
"windowsAuthentication": false,
"anonymousAuthentication": true,
"iisExpress": {
"applicationUrl": "http://localhost:56247/",
"sslPort": 0
}
},
"profiles": {
"IIS Express": {
"commandName": "IISExpress",
"launchBrowser": true,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"Podsync": {
"commandName": "Project",
"launchBrowser": true,
"launchUrl": "http://localhost:5000",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}

View File

@@ -1,38 +0,0 @@
using Podsync.Services.Resolver;
namespace Podsync.Services
{
public static class Constants
{
public const int DefaultPageSize = 50;
public const ResolveFormat DefaultFormat = ResolveFormat.VideoHigh;
public static class Patreon
{
public const string AuthenticationScheme = "Patreon";
public const string AuthorizationEndpoint = "https://www.patreon.com/oauth2/authorize";
public const string TokenEndpoint = "https://api.patreon.com/oauth2/token";
public const string AmountDonated = "Patreon/" + nameof(AmountDonated);
}
public static class Events
{
public const int RssError = 1;
public const int YtdlError = 2;
public const int UnhandledError = 3;
}
public static class Cache
{
public const string VideosPrefix = "video_urls";
public const string FeedsPrefix = "feeds";
}
}
}

View File

@@ -1,17 +0,0 @@
using System.ComponentModel.DataAnnotations;
using Podsync.Services.Resolver;
namespace Podsync.Services
{
public class CreateFeedRequest
{
[Required]
[Url]
public string Url { get; set; }
public ResolveFormat? Quality { get; set; }
[Range(50, 150)]
public int? PageSize { get; set; }
}
}

View File

@@ -1,13 +0,0 @@
using System;
namespace Podsync.Services.Links
{
public interface ILinkService
{
LinkInfo Parse(Uri link);
Uri Make(LinkInfo info);
Uri Feed(Uri baseUrl, string feedId);
}
}

View File

@@ -1,11 +0,0 @@
namespace Podsync.Services.Links
{
public struct LinkInfo
{
public string Id { get; set; }
public LinkType LinkType { get; set; }
public Provider Provider { get; set; }
}
}

View File

@@ -1,169 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using Microsoft.AspNetCore.WebUtilities;
using Microsoft.Extensions.Primitives;
namespace Podsync.Services.Links
{
public class LinkService : ILinkService
{
private static readonly IDictionary<Provider, IDictionary<LinkType, string>> LinkFormats = new Dictionary<Provider, IDictionary<LinkType, string>>
{
[Provider.YouTube] = new Dictionary<LinkType, string>
{
[LinkType.Video] = "https://youtube.com/watch?v={0}",
[LinkType.Channel] = "https://youtube.com/channel/{0}",
[LinkType.Playlist] = "https://youtube.com/playlist?list={0}"
},
[Provider.Vimeo] = new Dictionary<LinkType, string>
{
[LinkType.Channel] = "https://vimeo.com/channels/{0}",
[LinkType.Group] = "https://vimeo.com/groups/{0}",
[LinkType.User] = "https://vimeo.com/user{0}",
[LinkType.Video] = "https://vimeo.com/{0}"
}
};
/*
YouTube users, channels and playlists
Test input:
https://www.youtube.com/playlist?list=PLCB9F975ECF01953C
https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og
https://www.youtube.com/user/fxigr1
https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos
*/
private static readonly Regex YouTubeRegex = new Regex(@"^(?:https?://)?(?:www\.|m\.)?(?:youtube.com/)(?<type>user|channel|playlist|watch)/?(?<id>[-\w]+)?", RegexOptions.Compiled);
/*
Vimeo groups, channels and users
Test input:
https://vimeo.com/groups/109
http://vimeo.com/groups/109
http://www.vimeo.com/groups/109
https://vimeo.com/groups/109/videos/
https://vimeo.com/channels/staffpicks
https://vimeo.com/channels/staffpicks/146224925
https://vimeo.com/awhitelabelproduct
*/
private static readonly Regex VimeoRegex = new Regex(@"^(?:https?://)?(?:www\.)?(?:vimeo.com/)(?<type>groups|channels)?/?(?<id>\w+)", RegexOptions.Compiled);
public LinkInfo Parse(Uri link)
{
if (link == null)
{
throw new ArgumentNullException(nameof(link), "Link can't be null");
}
var provider = Provider.Unknown;
var linkType = LinkType.Unknown;
var id = string.Empty;
// YouTube
var match = YouTubeRegex.Match(link.AbsoluteUri);
if (match.Success)
{
provider = Provider.YouTube;
var type = match.Groups["type"]?.ToString();
if (type == "user")
{
// https://www.youtube.com/user/fxigr1
id = match.Groups["id"]?.ToString();
linkType = LinkType.User;
}
else if (type == "channel")
{
// https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og
id = match.Groups["id"]?.ToString();
linkType = LinkType.Channel;
}
else if (type == "playlist" || type == "watch")
{
// https://www.youtube.com/playlist?list=PLCB9F975ECF01953C
// https://www.youtube.com/watch?v=otm9NaT9OWU&list=PLCB9F975ECF01953C
var qs = QueryHelpers.ParseQuery(link.Query);
StringValues list;
if (qs.TryGetValue("list", out list))
{
id = list;
}
linkType = LinkType.Playlist;
}
}
else
{
// Vimeo
match = VimeoRegex.Match(link.AbsoluteUri);
if (match.Success)
{
provider = Provider.Vimeo;
id = match.Groups["id"]?.ToString();
var type = match.Groups["type"]?.ToString();
if (type == "groups")
{
// https://vimeo.com/groups/109
linkType = LinkType.Group;
}
else if (type == "channels")
{
// https://vimeo.com/channels/staffpicks
linkType = LinkType.Channel;
}
else
{
// https://vimeo.com/awhitelabelproduct
linkType = LinkType.User;
}
}
}
if (string.IsNullOrWhiteSpace(id) || linkType == LinkType.Unknown || provider == Provider.Unknown)
{
throw new ArgumentException("Not supported provider or link format");
}
return new LinkInfo
{
Id = id,
LinkType = linkType,
Provider = provider
};
}
public Uri Make(LinkInfo info)
{
if (info.Id == null)
{
throw new ArgumentNullException(nameof(info.Id), "Id can't be empty");
}
try
{
var format = LinkFormats[info.Provider][info.LinkType];
return new Uri(string.Format(format, info.Id));
}
catch (KeyNotFoundException ex)
{
throw new ArgumentException("Unsupported provider or link type", nameof(info), ex);
}
}
public Uri Feed(Uri baseUrl, string feedId)
{
return new Uri(baseUrl, feedId);
}
}
}

View File

@@ -1,12 +0,0 @@
namespace Podsync.Services.Links
{
public enum LinkType
{
Unknown = 0,
Video,
Playlist,
User,
Channel,
Group
}
}

View File

@@ -1,9 +0,0 @@
namespace Podsync.Services.Links
{
public enum Provider
{
Unknown = 0,
YouTube,
Vimeo,
}
}

View File

@@ -1,17 +0,0 @@
using System;
namespace Podsync.Services.Patreon.Contracts
{
public class Pledge
{
public string Id { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime DeclinedSince { get; set; }
public int AmountCents { get; set; }
public int PledgeCapCents { get; set; }
}
}

View File

@@ -1,23 +0,0 @@
using System.Collections.Generic;
using System.Linq;
namespace Podsync.Services.Patreon.Contracts
{
public class User
{
public User()
{
Pledges = Enumerable.Empty<Pledge>();
}
public string Id { get; set; }
public string Email { get; set; }
public string Name { get; set; }
public string Url { get; set; }
public IEnumerable<Pledge> Pledges { get; set; }
}
}

View File

@@ -1,10 +0,0 @@
using System;
using System.Threading.Tasks;
namespace Podsync.Services.Patreon
{
public interface IPatreonApi : IDisposable
{
Task<dynamic> FetchProfile(Tokens tokens);
}
}

View File

@@ -1,46 +0,0 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
namespace Podsync.Services.Patreon
{
public sealed class PatreonApi : IPatreonApi
{
private readonly HttpClient _client;
public PatreonApi()
{
_client = new HttpClient
{
BaseAddress = new Uri("https://api.patreon.com/oauth2/api/")
};
_client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer");
}
public Task<dynamic> FetchProfile(Tokens tokens)
{
return Query("current_user", tokens);
}
public void Dispose()
{
_client.Dispose();
}
private async Task<dynamic> Query(string path, Tokens tokens)
{
var request = new HttpRequestMessage(HttpMethod.Get, path);
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", tokens.AccessToken);
var response = await _client.SendAsync(request);
response.EnsureSuccessStatusCode();
var json = await response.Content.ReadAsStringAsync();
return JObject.Parse(json);
}
}
}

View File

@@ -1,72 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Podsync.Services.Patreon.Contracts;
namespace Podsync.Services.Patreon
{
public static class PatreonExtensions
{
public static async Task<User> FetchUserAndPledges(this IPatreonApi api, Tokens tokens)
{
var resp = await api.FetchProfile(tokens);
dynamic userAttrs = resp.data.attributes;
var user = new User
{
Id = resp.data.id,
Email = userAttrs.email,
Name = userAttrs.first_name ?? userAttrs.full_name,
Url = userAttrs.url,
Pledges = ParsePledges(resp)
};
return user;
}
private static IEnumerable<Pledge> ParsePledges(dynamic resp)
{
dynamic pledges = resp.data.relationships.pledges.data;
foreach (var pledge in pledges)
{
var id = pledge.id;
var type = pledge.type;
foreach (var include in resp.included)
{
if (include.id == id && include.type == type)
{
dynamic attrs = include.attributes;
yield return new Pledge
{
Id = include.id,
CreatedAt = ParseDate(attrs.created_at),
DeclinedSince = ParseDate(attrs.declined_since),
AmountCents = attrs.amount_cents,
PledgeCapCents = attrs.pledge_cap_cents
};
break;
}
}
}
}
private static DateTime ParseDate(object obj)
{
var date = obj?.ToString();
if (string.IsNullOrWhiteSpace(date))
{
return DateTime.MinValue;
}
var dateTime = DateTime.Parse(date);
return DateTime.SpecifyKind(dateTime, DateTimeKind.Utc);
}
}
}

View File

@@ -1,9 +0,0 @@
namespace Podsync.Services.Patreon
{
public struct Tokens
{
public string AccessToken { get; set; }
public string RefreshToken { get; set; }
}
}

View File

@@ -1,21 +0,0 @@
using Podsync.Services.Patreon;
namespace Podsync.Services
{
public class PodsyncConfiguration
{
public string YouTubeApiKey { get; set; }
public string VimeoApiKey { get; set; }
public string RedisConnectionString { get; set; }
public string PatreonClientId { get; set; }
public string PatreonSecret { get; set; }
public Tokens CreatorTokens { get; set; }
public string RemoteResolverUrl { get; set; }
}
}

View File

@@ -1,39 +0,0 @@
using System;
using System.Threading.Tasks;
using Podsync.Services.Storage;
namespace Podsync.Services.Resolver
{
public abstract class CachedResolver : IResolverService
{
private readonly TimeSpan UrlExpiration = TimeSpan.FromHours(3);
private readonly IStorageService _storageService;
protected CachedResolver(IStorageService storageService)
{
_storageService = storageService;
}
public abstract string Version { get; }
public async Task<Uri> Resolve(Uri videoUrl, ResolveFormat format)
{
var id = videoUrl.GetHashCode().ToString();
// Check if this video URL was resolved within last 3 hours
var value = await _storageService.GetCached(Constants.Cache.VideosPrefix, id);
if (!string.IsNullOrWhiteSpace(value))
{
return new Uri(value);
}
// Resolve and save to cache
var uri = await ResolveInternal(videoUrl, format);
await _storageService.Cache(Constants.Cache.VideosPrefix, id, uri.ToString(), UrlExpiration);
return uri;
}
protected abstract Task<Uri> ResolveInternal(Uri videoUrl, ResolveFormat format);
}
}

View File

@@ -1,12 +0,0 @@
using System;
using System.Threading.Tasks;
namespace Podsync.Services.Resolver
{
public interface IResolverService
{
string Version { get; }
Task<Uri> Resolve(Uri videoUrl, ResolveFormat format = ResolveFormat.VideoHigh);
}
}

View File

@@ -1,42 +0,0 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Podsync.Services.Storage;
namespace Podsync.Services.Resolver
{
public class RemoteResolver : CachedResolver
{
private readonly ILogger _logger;
private readonly HttpClient _client = new HttpClient();
public RemoteResolver(IStorageService storageService, IOptions<PodsyncConfiguration> options, ILogger<RemoteResolver> logger) : base(storageService)
{
_logger = logger;
_client.BaseAddress = new Uri(options.Value.RemoteResolverUrl);
_logger.LogInformation($"Remote resolver URL: {_client.BaseAddress}");
}
public override string Version { get; } = "Remote";
protected override async Task<Uri> ResolveInternal(Uri videoUrl, ResolveFormat format)
{
using(var response = await _client.GetAsync($"/resolve?url={videoUrl}&quality={format}"))
{
using(response.Content)
{
var body = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException(body);
}
return new Uri(body);
}
}
}
}
}

View File

@@ -1,10 +0,0 @@
namespace Podsync.Services.Resolver
{
public enum ResolveFormat
{
VideoHigh = 0,
VideoLow,
AudioHigh,
AudioLow
}
}

View File

@@ -1,54 +0,0 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Podsync.Helpers;
using Podsync.Services.Links;
using Podsync.Services.Storage;
namespace Podsync.Services.Rss.Builders
{
// ReSharper disable once ClassNeverInstantiated.Global
public class CompositeRssBuilder : RssBuilderBase
{
private readonly IDictionary<Provider, IRssBuilder> _builders;
private readonly ILogger _logger;
public CompositeRssBuilder(IServiceProvider serviceProvider, IStorageService storageService, ILogger<CompositeRssBuilder> logger) : base(storageService)
{
_logger = logger;
// Find all RSS builders (all implementations of IRssBuilder), create instances and make dictionary for fast search by Provider type
var buildTypes = serviceProvider.FindAllImplementationsOf<IRssBuilder>(Assembly.GetEntryAssembly()).Where(x => x != typeof(CompositeRssBuilder));
var builders = buildTypes.Select(builderType => (IRssBuilder)serviceProvider.CreateInstance(builderType)).ToDictionary(builder => builder.Provider);
_logger.LogInformation($"Found {builders.Count} RSS builders");
_builders = new ReadOnlyDictionary<Provider, IRssBuilder>(builders);
}
public override Provider Provider => throw new NotSupportedException();
public override Task<Contracts.Feed> Query(FeedMetadata feed)
{
try
{
IRssBuilder builder;
if (_builders.TryGetValue(feed.Provider, out builder))
{
return builder.Query(feed);
}
throw new NotSupportedException("Not supported provider");
}
catch (Exception ex)
{
_logger.LogError(Constants.Events.RssError, ex, "Failed to query RSS feed (id: {ID})", feed.Id);
throw;
}
}
}
}

View File

@@ -1,28 +0,0 @@
using System.Threading.Tasks;
using Podsync.Services.Links;
using Podsync.Services.Rss.Contracts;
using Podsync.Services.Storage;
namespace Podsync.Services.Rss.Builders
{
public abstract class RssBuilderBase : IRssBuilder
{
private readonly IStorageService _storageService;
protected RssBuilderBase(IStorageService storageService)
{
_storageService = storageService;
}
public abstract Provider Provider { get; }
public async Task<Contracts.Feed> Query(string feedId)
{
var metadata = await _storageService.Load(feedId);
return await Query(metadata);
}
public abstract Task<Feed> Query(FeedMetadata metadata);
}
}

View File

@@ -1,114 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Podsync.Services.Links;
using Podsync.Services.Rss.Contracts;
using Podsync.Services.Storage;
using Podsync.Services.Videos.Vimeo;
namespace Podsync.Services.Rss.Builders
{
// ReSharper disable once ClassNeverInstantiated.Global
public class VimeoRssBuilder : RssBuilderBase
{
private readonly IVimeoClient _client;
public VimeoRssBuilder(IStorageService storageService, IVimeoClient client) : base(storageService)
{
_client = client;
}
public override Provider Provider { get; } = Provider.Vimeo;
public override async Task<Feed> Query(FeedMetadata metadata)
{
var linkType = metadata.LinkType;
var id = metadata.Id;
var pageSize = metadata.PageSize;
if (pageSize == 0)
{
pageSize = Constants.DefaultPageSize;
}
Channel channel;
if (linkType == LinkType.Channel)
{
channel = CreateChannel(await _client.Channel(id));
channel.Items = CreateItems(await _client.ChannelVideos(id, pageSize));
}
else if (linkType == LinkType.Group)
{
channel = CreateChannel(await _client.Group(id));
channel.Items = CreateItems(await _client.GroupVideos(id, pageSize));
}
else if (linkType == LinkType.User)
{
channel = CreateChannel(await _client.User(id));
channel.Items = CreateItems(await _client.UserVideos(id, pageSize));
}
else
{
throw new NotSupportedException("URL type is not supported");
}
var rss = new Feed
{
Channels = new[] { channel }
};
return rss;
}
private static Channel CreateChannel(Group group)
{
return new Channel
{
Title = group.Name,
Description = group.Description,
Link = group.Link,
PubDate = group.CreatedAt,
Image = group.Thumbnail,
Thumbnail = group.Thumbnail,
Guid = group.Link.ToString()
};
}
private static Channel CreateChannel(User user)
{
return new Channel
{
Title = user.Name,
Description = user.Bio,
Link = user.Link,
PubDate = user.CreatedAt,
Image = user.Thumbnail,
Thumbnail = user.Thumbnail,
Guid = user.Link.ToString()
};
}
private static Item CreateItem(Video video)
{
return new Item
{
Id = video.Id,
Title = video.Title,
Description = video.Description,
PubDate = video.CreatedAt,
Link = video.Link,
Duration = video.Duration,
FileSize = video.Size,
ContentType = "video/mp4",
Author = video.Author
};
}
private static Item[] CreateItems(IEnumerable<Video> videos)
{
return videos.Select(CreateItem).ToArray();
}
}
}

View File

@@ -1,160 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Podsync.Services.Links;
using Podsync.Services.Resolver;
using Podsync.Services.Rss.Contracts;
using Podsync.Services.Storage;
using Podsync.Services.Videos.YouTube;
using Channel = Podsync.Services.Rss.Contracts.Channel;
using Video = Podsync.Services.Videos.YouTube.Video;
namespace Podsync.Services.Rss.Builders
{
public class YouTubeRssBuilder : RssBuilderBase
{
private static readonly Item[] NoVideos = new Item[0];
private readonly IYouTubeClient _youTube;
public YouTubeRssBuilder(IYouTubeClient youTube, IStorageService storageService) : base(storageService)
{
_youTube = youTube;
}
public override Provider Provider { get; } = Provider.YouTube;
public override async Task<Feed> Query(FeedMetadata metadata)
{
if (metadata.Provider != Provider.YouTube)
{
throw new ArgumentException("Invalid provider");
}
if (metadata.PageSize == 0)
{
metadata.PageSize = Constants.DefaultPageSize;
}
var linkType = metadata.LinkType;
Channel channel;
if (linkType == LinkType.Channel)
{
channel = await GetChannel(new ChannelQuery { ChannelId = metadata.Id });
}
else if (linkType == LinkType.User)
{
channel = await GetChannel(new ChannelQuery { Username = metadata.Id });
}
else if (linkType == LinkType.Playlist)
{
channel = await GetPlaylist(metadata.Id);
}
else
{
throw new NotSupportedException("URL type is not supported");
}
if (channel == null)
{
throw new ArgumentException("Invalid channel or playlist id");
}
channel.Items = NoVideos;
// Get video ids from this playlist
var ids = await _youTube.GetPlaylistItemIds(new PlaylistItemsQuery { PlaylistId = channel.Guid, Count = metadata.PageSize });
if (ids.Count > 0)
{
// Get video descriptions
var videos = await _youTube.GetVideos(new VideoQuery { Ids = ids });
channel.Items = videos.Select(youtubeVideo => MakeItem(youtubeVideo, metadata)).ToArray();
}
var rss = new Feed
{
Channels = new[] { channel }
};
return rss;
}
private async Task<Channel> GetChannel(ChannelQuery query)
{
var list = await _youTube.GetChannels(query);
var item = list.FirstOrDefault();
if (item == null)
{
return null;
}
var channel = MakeChannel(item);
channel.Guid = item.PlaylistId;
return channel;
}
private async Task<Channel> GetPlaylist(string playlistId)
{
var list = await _youTube.GetPlaylists(new PlaylistQuery { PlaylistId = playlistId });
var item = list.FirstOrDefault();
if (item == null)
{
return null;
}
var channel = MakeChannel(item);
channel.Guid = item.PlaylistId;
return channel;
}
private static Channel MakeChannel(YouTubeItem item)
{
return new Channel
{
Title = item.Title,
Description = item.Description,
Link = item.Link,
PubDate = item.PublishedAt,
Image = item.Thumbnail,
Thumbnail = item.Thumbnail,
};
}
private Item MakeItem(Video video, FeedMetadata feed)
{
string contentType = GetContentType(feed.Quality);
return new Item
{
Id = video.VideoId,
Title = video.Title,
Description = video.Description,
PubDate = video.PublishedAt,
Link = video.Link,
Duration = video.Duration,
FileSize = video.Size,
ContentType = contentType
};
}
private static string GetContentType(ResolveFormat format)
{
if (format == ResolveFormat.VideoHigh || format == ResolveFormat.VideoLow)
{
return "video/mp4";
}
if (format == ResolveFormat.AudioHigh || format == ResolveFormat.AudioLow)
{
return "audio/mp4";
}
throw new ArgumentException("Unsupported resolve type");
}
}
}

View File

@@ -1,117 +0,0 @@
using System;
using System.Xml;
using System.Xml.Schema;
using System.Xml.Serialization;
using Podsync.Helpers;
namespace Podsync.Services.Rss.Contracts
{
[XmlRoot("channel")]
public class Channel : IXmlSerializable
{
private const string PodsyncGeneratorName = "Podsync Generator";
private const string DefaultItunesCategory = "TV & Film";
public Channel()
{
Category = DefaultItunesCategory;
LastBuildDate = DateTime.Now;
}
public string Guid { get; set; }
public string Title { get; set; }
public string Description { get; set; }
public Uri Link { get; set; }
public DateTime LastBuildDate { get; set; }
public DateTime PubDate { get; set; }
public string Subtitle { get; set; }
public string Summary { get; set; }
public string Category { get; set; }
public Uri Image { get; set; }
public Uri Thumbnail { get; set; }
public Item[] Items { get; set; }
public bool Explicit { get; set; }
public Uri AtomLink { get; set; }
public XmlSchema GetSchema()
{
return null;
}
public void ReadXml(XmlReader reader)
{
throw new NotSupportedException("Reading is not supported");
}
public void WriteXml(XmlWriter writer)
{
writer.WriteElementString("title", Title);
writer.WriteElementString("description", Description);
writer.WriteElementString("link", Link.ToString());
writer.WriteElementString("generator", PodsyncGeneratorName);
writer.WriteElementString("lastBuildDate", LastBuildDate.ToString("R"));
writer.WriteElementString("pubDate", PubDate.ToString("R"));
/*
<itunes:subtitle>Laugh Factory</itunes:subtitle>
<itunes:summary>The best stand up comedy clips online. That's it.</itunes:summary>
*/
writer.WriteElementString("subtitle", Namespaces.Itunes, Title);
writer.WriteElementString("summary", Namespaces.Itunes, Summary);
writer.WriteElementString("explicit", Namespaces.Itunes, Explicit ? "yes" : "no");
if (AtomLink != null)
{
writer.WriteStartElement("link", Namespaces.Atom);
writer.WriteAttributeString("href", AtomLink.ToString());
writer.WriteAttributeString("rel", "self");
writer.WriteAttributeString("type", "application/rss+xml");
writer.WriteEndElement();
}
/*
<itunes:category text="TV & Film"/>
*/
writer.WriteStartElement("category", Namespaces.Itunes);
writer.WriteAttributeString("text", Category);
writer.WriteEndElement();
/*
<itunes:image href="https://yt3.ggpht.com/photo.jpg"/>
*/
writer.WriteStartElement("image", Namespaces.Itunes);
writer.WriteAttributeString("href", Image.ToString());
writer.WriteEndElement();
/*
<media:thumbnail url="https://yt3.ggpht.com//photo.jpg"/>
*/
writer.WriteStartElement("thumbnail", Namespaces.Media);
writer.WriteAttributeString("url", Thumbnail.ToString());
writer.WriteEndElement();
// Items
var serializer = new XmlSerializer(typeof(Item));
Items.ForEach(item => serializer.Serialize(writer, item));
}
}
}

View File

@@ -1,59 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml;
using System.Xml.Schema;
using System.Xml.Serialization;
using Podsync.Helpers;
namespace Podsync.Services.Rss.Contracts
{
[XmlRoot("rss")]
public class Feed : IXmlSerializable
{
public const string Version = "2.0";
public IEnumerable<Channel> Channels { get; set; }
public Feed()
{
Channels = Enumerable.Empty<Channel>();
}
public XmlSchema GetSchema()
{
return null;
}
public void ReadXml(XmlReader reader)
{
throw new NotSupportedException("Reading is not supported");
}
public void WriteXml(XmlWriter writer)
{
writer.WriteAttributeString("xmlns", "dc", null, Namespaces.Dc);
writer.WriteAttributeString("xmlns", "content", null, Namespaces.Content);
writer.WriteAttributeString("xmlns", "atom", null, Namespaces.Atom);
writer.WriteAttributeString("xmlns", "itunes", null, Namespaces.Itunes);
writer.WriteAttributeString("xmlns", "media", null, Namespaces.Media);
writer.WriteAttributeString("version", Version);
var serializer = new XmlSerializer(typeof(Channel));
Channels.ForEach(channel => serializer.Serialize(writer, channel));
}
public override string ToString()
{
var serializer = new XmlSerializer(typeof(Feed));
// Serialize feed to XML string
using (var writer = new Utf8StringWriter())
{
serializer.Serialize(writer, this);
return writer.ToString();
}
}
}
}

View File

@@ -1,101 +0,0 @@
using System;
using System.IO;
using System.Xml;
using System.Xml.Schema;
using System.Xml.Serialization;
namespace Podsync.Services.Rss.Contracts
{
[XmlRoot("item")]
public class Item : IXmlSerializable
{
public string Title { get; set; }
public string Description { get; set; }
public string Author { get; set; }
public Uri Link { get; set; }
public DateTime PubDate { get; set; }
public string Summary { get; set; }
public TimeSpan Duration { get; set; }
public string Id { get; set; }
public long FileSize { get; set; }
public Uri DownloadLink { get; set; }
public string ContentType { get; set; }
public XmlSchema GetSchema()
{
return null;
}
public void ReadXml(XmlReader reader)
{
throw new NotSupportedException("Reading is not supported");
}
public void WriteXml(XmlWriter writer)
{
writer.WriteElementString("title", Title);
writer.WriteElementString("description", Description);
writer.WriteElementString("link", Link.ToString());
writer.WriteElementString("pubDate", PubDate.ToString("R"));
if (!string.IsNullOrWhiteSpace(Author))
{
writer.WriteElementString("author", Author);
}
/*
<guid isPermaLink="true">https://youtube.com/watch?v=yp202t46OIE</guid>
*/
writer.WriteStartElement("guid");
writer.WriteAttributeString("isPermaLink", "true");
writer.WriteString(Link?.ToString() ?? Id);
writer.WriteEndElement();
/*
<enclosure url="http://podsync.net/download/youtube/yp202t46OIE.mp4" length="48300000" type="video/mp4"/>
*/
if (DownloadLink == null)
{
throw new InvalidDataException("Can't generate RSS item with no download link");
}
writer.WriteStartElement("enclosure");
writer.WriteAttributeString("url", DownloadLink.ToString());
writer.WriteAttributeString("length", FileSize.ToString());
writer.WriteAttributeString("type", ContentType);
writer.WriteEndElement();
/*
<media:content url="http://podsync.net/download/youtube/yp202t46OIE.mp4" fileSize="48300000" type="video/mp4"/>
*/
writer.WriteStartElement("content", Namespaces.Media);
writer.WriteAttributeString("url", DownloadLink.ToString());
writer.WriteAttributeString("fileSize", FileSize.ToString());
writer.WriteAttributeString("type", ContentType);
writer.WriteEndElement();
/*
<itunes:subtitle>Mike E. Winfield - Cheating (Stand up comedy)</itunes:subtitle>
<itunes:summary>...</itunes:summary>
<itunes:duration>00:02:18</itunes:duration>
*/
writer.WriteElementString("subtitle", Namespaces.Itunes, Title);
writer.WriteElementString("summary", Namespaces.Itunes, Summary);
writer.WriteElementString("duration", Namespaces.Itunes, Duration.ToString());
}
}
}

View File

@@ -1,15 +0,0 @@
namespace Podsync.Services.Rss.Contracts
{
public static class Namespaces
{
public const string Itunes = "http://www.itunes.com/dtds/podcast-1.0.dtd";
public const string Media = "http://search.yahoo.com/mrss/";
public const string Atom = "http://www.w3.org/2005/Atom";
public const string Content = "http://purl.org/rss/1.0/modules/content/";
public const string Dc = "http://purl.org/dc/elements/1.1/";
}
}

View File

@@ -1,55 +0,0 @@
using System;
using System.Threading.Tasks;
using Podsync.Helpers;
using Podsync.Services.Links;
using Podsync.Services.Rss.Contracts;
using Podsync.Services.Storage;
namespace Podsync.Services.Rss
{
public class FeedService : IFeedService
{
private readonly IStorageService _storageService;
private readonly IRssBuilder _rssBuilder;
public FeedService(IStorageService storageService, IRssBuilder rssBuilder)
{
_storageService = storageService;
_rssBuilder = rssBuilder;
}
public Task<string> Create(FeedMetadata metadata)
{
if (metadata.Provider != Provider.YouTube && metadata.Quality.IsAudio())
{
throw new ArgumentException("Only YouTube supports audio feeds");
}
return _storageService.Save(metadata);
}
public Task<Feed> Get(string id)
{
return _rssBuilder.Query(id);
}
public async Task<string> Get(string id, Action<string, Feed> fixup)
{
var serializedFeed = await _storageService.GetCached(Constants.Cache.FeedsPrefix, id);
if (string.IsNullOrEmpty(serializedFeed))
{
var feed = await Get(id);
// Fix download links
fixup(id, feed);
// Add to cache
serializedFeed = feed.ToString();
await _storageService.Cache(Constants.Cache.FeedsPrefix, id, serializedFeed, TimeSpan.FromMinutes(3));
}
return serializedFeed;
}
}
}

View File

@@ -1,16 +0,0 @@
using System;
using System.Threading.Tasks;
using Podsync.Services.Rss.Contracts;
using Podsync.Services.Storage;
namespace Podsync.Services.Rss
{
public interface IFeedService
{
Task<string> Create(FeedMetadata metadata);
Task<Feed> Get(string id);
Task<string> Get(string id, Action<string, Feed> fixup);
}
}

View File

@@ -1,16 +0,0 @@
using System.Threading.Tasks;
using Podsync.Services.Links;
using Podsync.Services.Rss.Contracts;
using Podsync.Services.Storage;
namespace Podsync.Services.Rss
{
public interface IRssBuilder
{
Provider Provider { get; }
Task<Feed> Query(string feedId);
Task<Feed> Query(FeedMetadata metadata);
}
}

View File

@@ -1,10 +0,0 @@
using System.IO;
using System.Text;
namespace Podsync.Services.Rss
{
public class Utf8StringWriter : StringWriter
{
public override Encoding Encoding { get; } = Encoding.UTF8;
}
}

Some files were not shown because too many files have changed in this diff Show More