diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..be004c8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +node_modules/ +vendor/ + +package-lock.json +Gopkg.lock \ No newline at end of file diff --git a/.gitignore b/.gitignore index 78341d2..1d7a2eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,258 +1,35 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so -# User-specific files -*.suo -*.user -*.userosscache -*.sln.docstates +# Folders +_obj +_test +vendor -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* -# Visual Studio 2015 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ +_testmain.go -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* +*.exe +*.test +*.prof -# NUNIT -*.VisualState.xml -TestResult.xml +glide.lock -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c +Gopkg.lock -# DNX -project.lock.json -artifacts/ - -*_i.c -*_p.c -*_i.h -*.ilk -*.meta -*.obj -*.pch -*.pdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# JustCode is a .NET coding add-in -.JustCode - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# TODO: Comment the next line if you want to checkin your web deploy settings -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# The packages folder can be ignored because of Package Restore -**/packages/* -# except build/, which is used as an MSBuild target. -!**/packages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/packages/repositories.config -# NuGet v3's project.json files produces more ignoreable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.pfx -*.publishsettings -node_modules/ -orleans.codegen.cs - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm - -# SQL Server files -*.mdf -*.ldf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# JetBrains Rider .idea/ -*.sln.iml -# Podsync -src/Podsync/wwwroot/lib/ -src/Podsync/wwwroot/**/*.min.css -src/Podsync/wwwroot/**/*.min.js -src/Podsync/wwwroot/**/*.min.js.map \ No newline at end of file +node_modules/ +package-lock.json +dist/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3edb6e3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +dist: trusty +sudo: required +language: go +go_import_path: github.com/mxpv/podsync +go: + - 1.8.3 +install: + - go get -u github.com/golang/dep/cmd/dep + - dep ensure +services: + - postgresql +addons: + postgresql: "9.6" +before_script: + - psql -a -c "CREATE DATABASE podsync;" -U postgres +script: + - set -e + - go test -v -short $(go list -e ./... | grep -v vendor) + - go tool vet -all ./pkg diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9e966b5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM node:latest AS gulp +WORKDIR /app +COPY . . +RUN npm install +RUN npm link gulp +RUN gulp patch + +FROM golang:1.8 AS build +WORKDIR /go/src/github.com/mxpv/podsync +COPY --from=gulp /app . +ENV GOOS=linux +ENV GOARCH=amd64 +ENV CGO_ENABLED=0 +RUN go get -u github.com/golang/dep/cmd/dep +RUN dep ensure +RUN go install -v ./cmd/app + +FROM alpine +RUN apk --update --no-cache add ca-certificates +WORKDIR /app/ +COPY --from=gulp /app/templates ./templates +COPY --from=gulp /app/dist ./assets +COPY --from=build /go/bin/app . +ENV ASSETS_PATH /app/assets +ENV TEMPLATES_PATH /app/templates +ENTRYPOINT ["/app/app"] \ No newline at end of file diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..15d11ca --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,20 @@ + +[[constraint]] + name = "github.com/mxpv/podcast" + branch = "release" + +[[constraint]] + name = "github.com/speps/go-hashids" + revision = "c6ced3330f133cec79e144fb11b2e4c5154059ae" + +[[constraint]] + name = "github.com/gin-gonic/gin" + version = "1.2" + +[[constraint]] + name = "github.com/gin-contrib/sessions" + revision = "a71ea9167c616cf9f02bc928ed08bc390c8279bc" + +[[constraint]] + name = "github.com/mxpv/patreon-go" + version = "1.2" \ No newline at end of file diff --git a/Podsync.sln b/Podsync.sln deleted file mode 100644 index 8d733f4..0000000 --- a/Podsync.sln +++ /dev/null @@ -1,62 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26403.7 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{3EF2294C-C807-46F3-B3F4-98B7F30688D0}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{15355284-4B88-4EF3-B580-20907F343E43}" - ProjectSection(SolutionItems) = preProject - .gitignore = .gitignore - docker-compose.yml = docker-compose.yml - EndProjectSection -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Podsync", "src\Podsync\Podsync.csproj", "{C9283472-E95E-4517-AE3B-3E276B9851E4}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{D94416F2-31EA-4D38-A3D0-BB74180687D4}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Podsync.Tests", "test\Podsync.Tests\Podsync.Tests.csproj", "{81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x64.ActiveCfg = Debug|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x64.Build.0 = Debug|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x86.ActiveCfg = Debug|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Debug|x86.Build.0 = Debug|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|Any CPU.Build.0 = Release|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x64.ActiveCfg = Release|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x64.Build.0 = Release|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x86.ActiveCfg = Release|Any CPU - {C9283472-E95E-4517-AE3B-3E276B9851E4}.Release|x86.Build.0 = Release|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x64.ActiveCfg = Debug|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x64.Build.0 = Debug|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x86.ActiveCfg = Debug|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Debug|x86.Build.0 = Debug|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|Any CPU.Build.0 = Release|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x64.ActiveCfg = Release|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x64.Build.0 = Release|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x86.ActiveCfg = Release|Any CPU - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {C9283472-E95E-4517-AE3B-3E276B9851E4} = {3EF2294C-C807-46F3-B3F4-98B7F30688D0} - {81ABAF5F-4CB6-42A6-8EAA-19EB9BAF66AA} = {D94416F2-31EA-4D38-A3D0-BB74180687D4} - EndGlobalSection -EndGlobal diff --git a/README.md b/README.md index ee4a9fd..8498b97 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,37 @@ -# Podsync -Turn YouTube channels into podcast feeds +## Handling static files -## Redis configuration notes -- [Set overcommit_memory=1 on CoreOS](https://gist.github.com/PyYoshi/c7b271c8ba80ea350376) -- [Set somaxconn=1024 on CoreOS](https://gist.github.com/PyYoshi/55c6953b7e8267d81daf) -- [Tweak Redis](https://easyengine.io/tutorials/redis/) -- [Performance tips for Redis Cache Server](https://www.techandme.se/performance-tips-for-redis-cache-server/) +`ASSETS_PATH` should point to a directory with static files. +For debugging just 'Copy Path' to `assets` directory. + +For production run `gulp patch`. +Gulp will generate `dist` directory with minified files and update templates to include these files. + +`TEMPLATES_PATH` should just point to `templates` directory. + +Docker will run `gulp` and include `dist` and `templates` directories during build as well as specify `ASSETS_PATH` and `TEMPLATES_PATH` environment variables. + +## Patreon + +In order to login via Patreon the following variables should be configured: +- `PATREON_REDIRECT_URL` should point to `http://yout_host_here/patreon` +- `PATREON_CLIENT_ID` and `PATREON_SECRET` should be copied from https://www.patreon.com/platform/documentation/clients + +## Deploy Docker images + +Build docker image: +``` +docker build -t ytdl . +``` + +Deploy image to Container Registry: +``` +docker tag ytdl gcr.io/pod-sync/ytdl +gcloud auth application-default login +gcloud docker -- push gcr.io/pod-sync/ytdl +``` + +or just use `build.sh` + +## Access to Container Registry from docker + +See https://cloud.google.com/container-registry/docs/advanced-authentication for details \ No newline at end of file diff --git a/src/Podsync/wwwroot/css/site.css b/assets/css/site.css similarity index 94% rename from src/Podsync/wwwroot/css/site.css rename to assets/css/site.css index 3881801..8f6e42e 100644 --- a/src/Podsync/wwwroot/css/site.css +++ b/assets/css/site.css @@ -13,7 +13,7 @@ a { } .background-image { - background-image: url('/img/pc_bg.png'); + background-image: url('/assets/img/pc_bg.png'); -ms-background-repeat: repeat-x; background-repeat: repeat-x; -ms-background-position: center bottom; @@ -32,7 +32,7 @@ a { /* Footer */ .footer { - background-image: url('/img/pc_footer.png'); + background-image: url('/assets/img/pc_footer.png'); -ms-background-repeat: space; background-repeat: space; -ms-background-position: center bottom; @@ -46,7 +46,7 @@ a { margin: 0; padding: 0; -webkit-user-select: text; - z-index: 1; + z-index: -1; } .twitter-link { @@ -180,7 +180,7 @@ a { .man { width: 210px; height: 333px; - background-image: url('/img/man.png'); + background-image: url('/assets/img/man.png'); -ms-background-repeat: no-repeat; background-repeat: no-repeat; -ms-background-position: center bottom; @@ -267,11 +267,11 @@ a { @media screen and (max-width: 640px) { .background-image { - background-image: url('/img/mobile_bg.png') + background-image: url('/assets/img/mobile_bg.png') } .footer { - background-image: url('/img/mobile_footer.png'); + background-image: url('/assets/img/mobile_footer.png'); height: 50px; } diff --git a/src/Podsync/wwwroot/favicon.ico b/assets/favicon.ico similarity index 100% rename from src/Podsync/wwwroot/favicon.ico rename to assets/favicon.ico diff --git a/src/Podsync/wwwroot/img/become_patreon.png b/assets/img/become_patreon.png similarity index 100% rename from src/Podsync/wwwroot/img/become_patreon.png rename to assets/img/become_patreon.png diff --git a/src/Podsync/wwwroot/img/man.png b/assets/img/man.png similarity index 100% rename from src/Podsync/wwwroot/img/man.png rename to assets/img/man.png diff --git a/src/Podsync/wwwroot/img/mobile_bg.png b/assets/img/mobile_bg.png similarity index 100% rename from src/Podsync/wwwroot/img/mobile_bg.png rename to assets/img/mobile_bg.png diff --git a/src/Podsync/wwwroot/img/mobile_footer.png b/assets/img/mobile_footer.png similarity index 100% rename from src/Podsync/wwwroot/img/mobile_footer.png rename to assets/img/mobile_footer.png diff --git a/src/Podsync/wwwroot/img/og_image.png b/assets/img/og_image.png similarity index 100% rename from src/Podsync/wwwroot/img/og_image.png rename to assets/img/og_image.png diff --git a/src/Podsync/wwwroot/img/patreon_logo.png b/assets/img/patreon_logo.png similarity index 100% rename from src/Podsync/wwwroot/img/patreon_logo.png rename to assets/img/patreon_logo.png diff --git a/src/Podsync/wwwroot/img/pc_bg.png b/assets/img/pc_bg.png similarity index 100% rename from src/Podsync/wwwroot/img/pc_bg.png rename to assets/img/pc_bg.png diff --git a/src/Podsync/wwwroot/img/pc_footer.png b/assets/img/pc_footer.png similarity index 100% rename from src/Podsync/wwwroot/img/pc_footer.png rename to assets/img/pc_footer.png diff --git a/src/Podsync/wwwroot/js/site.js b/assets/js/site.js similarity index 77% rename from src/Podsync/wwwroot/js/site.js rename to assets/js/site.js index bd8a3fe..460ba8d 100644 --- a/src/Podsync/wwwroot/js/site.js +++ b/assets/js/site.js @@ -1,6 +1,4 @@ -// Write your Javascript code. - -$(function () { +$(function () { function err(msg) { alert(msg); } @@ -12,40 +10,33 @@ $(function () { $.ajax({ dataType: 'text', - url: '/feed/create', + url: '/api/create', method: 'POST', data: JSON.stringify(data), contentType: 'application/json; charset=utf-8', - success: function (feedLink) { - // HACK: remove quotes - feedLink = JSON.parse(feedLink); - done(feedLink); + success: function (resp) { + done(JSON.parse(resp)); }, error: function (xhr, status, error) { - if (xhr.status === 400) { - // Bad request - var text = ''; + var text = ''; - try { - var json = JSON.parse(xhr.responseText); - $.each(json, function (key, value) { - text += value + '\r\n'; - }); - } catch (e) { - text = xhr.responseText; - } - - err(text); - } else { - // Generic error - err('Server sad \'' + error + '\': ' + xhr.responseText); + try { + var json = JSON.parse(xhr.responseText); + if (json['error']) { + text = json['error']; + } + } catch (e) { + text = xhr.responseText; } + + err(text); } }); } - function displayLink(link) { - showModal(link); + function displayLink(obj) { + var addr = location.protocol + '//' + location.hostname + '/' + obj.id; + showModal(addr); } /* @@ -104,15 +95,14 @@ $(function () { $('#best-quality, #worst-quality').toggleClass('selected-option'); } - function getQuality() { + function getFormat() { var isAudio = $('#audio-format').hasClass('selected-option'); - var isWorst = $('#worst-quality').hasClass('selected-option'); + return isAudio ? 'audio' : 'video' + } - if (isAudio) { - return isWorst ? 'AudioLow' : 'AudioHigh'; - } else { - return isWorst ? 'VideoLow' : 'VideoHigh'; - } + function getQuality() { + var isWorst = $('#worst-quality').hasClass('selected-option'); + return isWorst ? 'low' : 'high'; } function pageSwitch(evt) { @@ -195,7 +185,7 @@ $(function () { $('#get-link').click(function(e) { var url = $('#url-input').val(); - createFeed({ url: url, quality: getQuality(), pageSize: getPageCount() }, displayLink); + createFeed({ url: url, format: getFormat(), quality: getQuality(), page_size: getPageCount() }, displayLink); e.preventDefault(); }); diff --git a/backup.sh b/backup.sh deleted file mode 100755 index 3f044bd..0000000 --- a/backup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -BACKUP_DIR=${1:-"$HOME/PodsyncBackups"} - -mkdir -p $BACKUP_DIR -docker-machine scp podsync:/data/redis/appendonly.aof $BACKUP_DIR/redis.$(date '+%Y_%m_%d__%H_%M_%S').aof \ No newline at end of file diff --git a/bower.json b/bower.json deleted file mode 100644 index 55b76ac..0000000 --- a/bower.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "Podsync", - "homepage": "Podsync.net", - "authors": [ - "Maksym Pavlenko " - ], - "description": "", - "main": "", - "moduleType": [], - "license": "MIT", - "private": true, - "ignore": [ - "**/.*", - "node_modules", - "bower_components", - "test", - "tests" - ] -} diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..3c90c0b --- /dev/null +++ b/build.sh @@ -0,0 +1,3 @@ +docker build -t app . +docker tag app gcr.io/pod-sync/app +gcloud docker -- push gcr.io/pod-sync/app \ No newline at end of file diff --git a/cmd/app/main.go b/cmd/app/main.go new file mode 100644 index 0000000..a187158 --- /dev/null +++ b/cmd/app/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + + "github.com/mxpv/podsync/pkg/api" + "github.com/mxpv/podsync/pkg/builders" + "github.com/mxpv/podsync/pkg/config" + "github.com/mxpv/podsync/pkg/feeds" + "github.com/mxpv/podsync/pkg/id" + "github.com/mxpv/podsync/pkg/server" + "github.com/mxpv/podsync/pkg/storage" +) + +func main() { + stop := make(chan os.Signal) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create core sevices + + cfg, err := config.ReadConfiguration() + if err != nil { + panic(err) + } + + hashIds, err := id.NewIdGenerator() + if err != nil { + panic(err) + } + + redis, err := storage.NewRedisStorage(cfg.RedisURL) + if err != nil { + panic(err) + } + + // Builders + + youtube, err := builders.NewYouTubeBuilder(cfg.YouTubeApiKey) + if err != nil { + panic(err) + } + + vimeo, err := builders.NewVimeoBuilder(ctx, cfg.VimeoApiKey) + if err != nil { + panic(err) + } + + feed := feeds.NewFeedService( + feeds.WithIdGen(hashIds), + feeds.WithStorage(redis), + feeds.WithBuilder(api.Youtube, youtube), + feeds.WithBuilder(api.Vimeo, vimeo), + ) + + srv := http.Server{ + Addr: fmt.Sprintf(":%d", 5001), + Handler: server.MakeHandlers(feed, cfg), + } + + go func() { + log.Println("running listener") + if err := srv.ListenAndServe(); err != nil { + log.Fatal(err) + } + }() + + <-stop + + log.Printf("shutting down server") + + srv.Shutdown(ctx) + + log.Printf("server gracefully stopped") +} diff --git a/nginx/Dockerfile b/cmd/nginx/Dockerfile similarity index 100% rename from nginx/Dockerfile rename to cmd/nginx/Dockerfile diff --git a/cmd/nginx/nginx.conf b/cmd/nginx/nginx.conf new file mode 100644 index 0000000..43d9eb7 --- /dev/null +++ b/cmd/nginx/nginx.conf @@ -0,0 +1,16 @@ +server { + listen 80; + + gzip on; + server_tokens off; + + location /download { + proxy_pass http://ytdl:5002; + } + + location / { + proxy_pass http://app:5001; + proxy_buffers 8 16k; + proxy_buffer_size 16k; + } +} \ No newline at end of file diff --git a/ytdl/Dockerfile b/cmd/ytdl/Dockerfile similarity index 100% rename from ytdl/Dockerfile rename to cmd/ytdl/Dockerfile diff --git a/cmd/ytdl/build.sh b/cmd/ytdl/build.sh new file mode 100755 index 0000000..0939b87 --- /dev/null +++ b/cmd/ytdl/build.sh @@ -0,0 +1,3 @@ +docker build -t ytdl . +docker tag ytdl gcr.io/pod-sync/ytdl +gcloud docker -- push gcr.io/pod-sync/ytdl \ No newline at end of file diff --git a/ytdl/requirements.txt b/cmd/ytdl/requirements.txt similarity index 54% rename from ytdl/requirements.txt rename to cmd/ytdl/requirements.txt index 6646873..1293e62 100644 --- a/ytdl/requirements.txt +++ b/cmd/ytdl/requirements.txt @@ -1,3 +1,4 @@ +requests youtube_dl sanic redis \ No newline at end of file diff --git a/cmd/ytdl/test_ytdl.py b/cmd/ytdl/test_ytdl.py new file mode 100644 index 0000000..1cbc349 --- /dev/null +++ b/cmd/ytdl/test_ytdl.py @@ -0,0 +1,8 @@ +import ytdl +import unittest + + +class TestYtdl(unittest.TestCase): + def test_resolve(self): + self.assertIsNotNone(ytdl._resolve('https://youtube.com/watch?v=ygIUF678y40', {'format': 'video', 'quality': 'low'})) + self.assertIsNotNone(ytdl._resolve('https://youtube.com/watch?v=WyaEiO4hyik', {'format': 'audio', 'quality': 'high'})) diff --git a/ytdl/ytdl.py b/cmd/ytdl/ytdl.py similarity index 67% rename from ytdl/ytdl.py rename to cmd/ytdl/ytdl.py index 9808221..e30e549 100644 --- a/ytdl/ytdl.py +++ b/cmd/ytdl/ytdl.py @@ -1,17 +1,16 @@ -import youtube_dl -import redis import os -from youtube_dl.utils import DownloadError +import requests +import youtube_dl from sanic import Sanic -from sanic.exceptions import InvalidUsage, NotFound +from sanic.exceptions import InvalidUsage from sanic.response import text, redirect -from datetime import timedelta +from youtube_dl.utils import DownloadError + +METADATA_URL = os.getenv('METADATA_URL', 'http://app:5001/api/metadata/{feed_id}') +print('Using metadata URL template: ' + METADATA_URL) app = Sanic() -db = redis.from_url(os.getenv('REDIS_CONNECTION_STRING', 'redis://localhost:6379')) -db.ping() - opts = { 'quiet': True, 'no_warnings': True, @@ -29,7 +28,7 @@ url_formats = { @app.route('/download//', methods=['GET']) -async def download(request, feed_id, video_id): +async def download(req, feed_id, video_id): if not feed_id: raise InvalidUsage('Invalid feed id') @@ -38,44 +37,34 @@ async def download(request, feed_id, video_id): if not video_id: raise InvalidUsage('Invalid video id') - # Query redis - data = db.hgetall(feed_id) - if not data: - raise NotFound('Feed not found') - - # Delete this feed if no requests within 90 days - db.expire(feed_id, timedelta(days=90)) - - entries = {k.decode().lower(): v.decode().lower() for k, v in data.items()} + # Pull metadata from API server + metadata_url = METADATA_URL.format(feed_id=feed_id, video_id=video_id) + r = requests.get(url=metadata_url) + json = r.json() # Build URL - provider = entries.get('provider') + provider = json['provider'] tpl = url_formats[provider] if not tpl: raise InvalidUsage('Invalid feed') - url = tpl.format(video_id) - quality = entries.get('quality') try: - redirect_url = _resolve(url, quality) + redirect_url = _resolve(url, json) return redirect(redirect_url) except DownloadError as e: msg = str(e) return text(msg, status=511) -def _resolve(url, quality): +def _resolve(url, metadata): if not url: raise InvalidUsage('Invalid URL') - if not quality: - quality = 'videohigh' - try: with youtube_dl.YoutubeDL(opts) as ytdl: info = ytdl.extract_info(url, download=False) - return _choose_url(info, quality) + return _choose_url(info, metadata) except DownloadError: raise except Exception as e: @@ -83,8 +72,8 @@ def _resolve(url, quality): raise -def _choose_url(info, quality): - is_video = quality == 'videohigh' or quality == 'videolow' +def _choose_url(info, metadata): + is_video = metadata['format'] == 'video' # Filter formats by file extension ext = 'mp4' if is_video else 'm4a' @@ -100,10 +89,10 @@ def _choose_url(info, quality): ordered = sorted(fmt_list, key=lambda x: x[sort_field], reverse=True) # Choose an item depending on quality, better at the beginning - is_high_quality = quality == 'videohigh' or quality == 'audiohigh' + is_high_quality = metadata['quality'] == 'high' item = ordered[0] if is_high_quality else ordered[-1] return item['url'] if __name__ == '__main__': - app.run(host='0.0.0.0', port=5002, workers=32) + app.run(host='0.0.0.0', port=5002, workers=32) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 78f7a3d..ea491f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,16 +8,13 @@ services: ports: - 5001 environment: - - ASPNETCORE_URLS=http://*:5001 - - ASPNETCORE_ENVIRONMENT=Production - - Podsync:RedisConnectionString=redis - - Podsync:RemoteResolverUrl=http://ytdl:5002 - - Podsync:YouTubeApiKey={YOUTUBE_API_KEY} - - Podsync:VimeoApiKey={VIMEO_API_KEY} - - Podsync:PatreonClientId={PATREON_CLIENT_ID} - - Podsync:PatreonSecret={PATREON_SECRET} - - Logging:LogLevel:Default=Information - - Logging:LogLevel:Microsoft=Warning + - REDIS_CONNECTION_URL=redis://redis + - YOUTUBE_API_KEY={YOUTUBE_API_KEY} + - VIMEO_API_KEY={VIMEO_API_KEY} + - PATREON_CLIENT_ID={PATREON_CLIENT_ID} + - PATREON_SECRET={PATREON_SECRET} + - PATREON_REDIRECT_URL=http://podsync.net/patreon + - COOKIE_SECRET={COOKIE_SECRET} ytdl: image: gcr.io/pod-sync/ytdl:latest container_name: ytdl @@ -25,7 +22,7 @@ services: ports: - 5002 environment: - - REDIS_CONNECTION_STRING=redis://redis:6379/0 + - METADATA_URL=http://app:5001/api/metadata/{feed_id} redis: image: redis container_name: redis diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000..8114d58 --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,48 @@ +var gulp = require('gulp'), + del = require('del'), + path = require('path'), + uglify = require('gulp-uglify'), + rev = require('gulp-rev'), + revreplace = require('gulp-rev-replace'), + cleancss = require('gulp-clean-css'), + autoprefixer = require('gulp-autoprefixer'), + size = require('gulp-size'), + gulpif = require('gulp-if'), + imagemin = require('gulp-imagemin'); + +abs = path.join(process.cwd(), 'assets'); + +gulp.task('clean', function () { + return del(['./dist/**/*']) +}); + +// Minify images and output to ./dist folder +gulp.task('img', ['clean'], function() { + return gulp.src('./assets/**/*.{png,ico}') + .pipe(imagemin()) + .pipe(size()) + .pipe(gulp.dest('./dist')) +}); + +// Minify scripts, build manifest.json and output to ./dist folder +gulp.task('js+css', ['clean', 'img'], function() { + return gulp.src(['./assets/js/**/*.js', './assets/css/**/*.css'], {base: abs}) + .pipe(gulpif(/js$/, uglify())) + .pipe(gulpif(/css$/, autoprefixer())) + .pipe(gulpif(/css$/, cleancss())) + .pipe(rev()) + .pipe(size()) + .pipe(gulp.dest('./dist')) + .pipe(rev.manifest('manifest.json', {merge: true})) + .pipe(gulp.dest('./dist')); +}); + +// Rewrite occurrences of scripts in template files +gulp.task('patch', ['js+css'], function() { + var manifest = gulp.src('./dist/manifest.json'); + return gulp.src('./templates/index.html') + .pipe(revreplace({manifest: manifest})) + .pipe(gulp.dest('./templates/')) +}); + +gulp.task('default', ['js+css']); \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf deleted file mode 100644 index deec43a..0000000 --- a/nginx/nginx.conf +++ /dev/null @@ -1,21 +0,0 @@ -server { - listen 80; - - gzip on; - server_tokens off; - - location /download { - proxy_pass http://ytdl:5002; - } - - location / { - proxy_pass http://app:5001; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection keep-alive; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_cache_bypass $http_upgrade; - } -} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..62422ff --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "name": "podsync", + "version": "1.0.0", + "description": "", + "main": "gulpfile.js", + "dependencies": {}, + "devDependencies": { + "del": "^3.0.0", + "gulp": "^3.9.1", + "gulp-autoprefixer": "^4.0.0", + "gulp-clean-css": "^3.7.0", + "gulp-if": "^2.0.2", + "gulp-imagemin": "^3.3.0", + "gulp-rev": "^8.0.0", + "gulp-rev-replace": "^0.4.3", + "gulp-size": "^2.1.0", + "gulp-uglify": "^3.0.0" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/mxpv/Podsync.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/mxpv/Podsync/issues" + }, + "homepage": "https://github.com/mxpv/Podsync#readme" +} diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..4ba658a --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,82 @@ +package api + +import ( + "time" + + "github.com/pkg/errors" +) + +var ( + ErrNotFound = errors.New("resource not found") +) + +type Provider string + +const ( + Youtube = Provider("youtube") + Vimeo = Provider("vimeo") +) + +type LinkType string + +const ( + Channel = LinkType("channel") + Playlist = LinkType("playlist") + User = LinkType("user") + Group = LinkType("group") +) + +type Quality string + +const ( + HighQuality = Quality("high") + LowQuality = Quality("low") +) + +type Format string + +const ( + AudioFormat = Format("audio") + VideoFormat = Format("video") +) + +const ( + DefaultPageSize = 50 + DefaultFormat = VideoFormat + DefaultQuality = HighQuality +) + +type Feed struct { + Id int64 `json:"id"` + HashId string `json:"hash_id"` // Short human readable feed id for users + UserId string `json:"user_id"` // Patreon user id + ItemId string `json:"item_id"` + Provider Provider `json:"provider"` // Youtube or Vimeo + LinkType LinkType `json:"link_type"` // Either group, channel or user + PageSize int `json:"page_size"` // The number of episodes to return + Format Format `json:"format"` + Quality Quality `json:"quality"` + FeatureLevel int `json:"feature_level"` // Available features + LastAccess time.Time `json:"last_access"` +} + +const ( + DefaultFeatures = iota + ExtendedFeatures + PodcasterFeature +) + +type CreateFeedRequest struct { + URL string `json:"url" binding:"url,required"` + PageSize int `json:"page_size" binding:"min=10,max=150,required"` + Quality Quality `json:"quality" binding:"eq=high|eq=low"` + Format Format `json:"format" binding:"eq=video|eq=audio"` +} + +type Identity struct { + UserId string `json:"user_id"` + FullName string `json:"full_name"` + Email string `json:"email"` + ProfileURL string `json:"profile_url"` + FeatureLevel int `json:"feature_level"` +} diff --git a/pkg/builders/common.go b/pkg/builders/common.go new file mode 100644 index 0000000..fc412e7 --- /dev/null +++ b/pkg/builders/common.go @@ -0,0 +1,25 @@ +package builders + +import ( + "fmt" + + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" +) + +const ( + podsyncGenerator = "Podsync generator" + defaultCategory = "TV & Film" +) + +func makeEnclosure(feed *api.Feed, id string, lengthInBytes int64) (string, itunes.EnclosureType, int64) { + ext := "mp4" + contentType := itunes.MP4 + if feed.Format == api.AudioFormat { + ext = "m4a" + contentType = itunes.M4A + } + + url := fmt.Sprintf("http://podsync.net/download/%s/%s.%s", feed.HashId, id, ext) + return url, contentType, lengthInBytes +} diff --git a/pkg/builders/vimeo.go b/pkg/builders/vimeo.go new file mode 100644 index 0000000..252c0b5 --- /dev/null +++ b/pkg/builders/vimeo.go @@ -0,0 +1,190 @@ +package builders + +import ( + "net/http" + "strconv" + + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" + "github.com/pkg/errors" + "github.com/silentsokolov/go-vimeo" + "golang.org/x/net/context" + "golang.org/x/oauth2" +) + +const ( + vimeoDefaultPageSize = 50 +) + +type VimeoBuilder struct { + client *vimeo.Client +} + +func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q api.Quality) string { + if p == nil || len(p.Sizes) == 0 { + return "" + } + + if q == api.LowQuality { + return p.Sizes[0].Link + } else { + return p.Sizes[len(p.Sizes)-1].Link + } +} + +func (v *VimeoBuilder) queryChannel(feed *api.Feed) (*itunes.Podcast, error) { + channelId := feed.ItemId + + ch, resp, err := v.client.Channels.Get(channelId) + if err != nil { + if resp.StatusCode == http.StatusNotFound { + return nil, api.ErrNotFound + } + + return nil, errors.Wrapf(err, "failed to query channel with channelId %s", channelId) + } + + podcast := itunes.New(ch.Name, ch.Link, ch.Description, &ch.CreatedTime, nil) + podcast.Generator = podsyncGenerator + podcast.AddSubTitle(ch.Name) + podcast.AddImage(v.selectImage(ch.Pictures, feed.Quality)) + podcast.AddCategory(defaultCategory, nil) + podcast.IAuthor = ch.User.Name + + return &podcast, nil +} + +func (v *VimeoBuilder) queryGroup(feed *api.Feed) (*itunes.Podcast, error) { + groupId := feed.ItemId + + gr, resp, err := v.client.Groups.Get(groupId) + if err != nil { + if resp.StatusCode == http.StatusNotFound { + return nil, api.ErrNotFound + } + + return nil, errors.Wrapf(err, "failed to query group with id %s", groupId) + } + + podcast := itunes.New(gr.Name, gr.Link, gr.Description, &gr.CreatedTime, nil) + podcast.Generator = podsyncGenerator + podcast.AddSubTitle(gr.Name) + podcast.AddImage(v.selectImage(gr.Pictures, feed.Quality)) + podcast.AddCategory(defaultCategory, nil) + podcast.IAuthor = gr.User.Name + + return &podcast, nil +} + +func (v *VimeoBuilder) queryUser(feed *api.Feed) (*itunes.Podcast, error) { + userId := feed.ItemId + + user, resp, err := v.client.Users.Get(userId) + if err != nil { + if resp.StatusCode == http.StatusNotFound { + return nil, api.ErrNotFound + } + + return nil, errors.Wrapf(err, "failed to query user with id %s", userId) + } + + podcast := itunes.New(user.Name, user.Link, user.Bio, &user.CreatedTime, nil) + podcast.Generator = podsyncGenerator + podcast.AddSubTitle(user.Name) + podcast.AddImage(v.selectImage(user.Pictures, feed.Quality)) + podcast.AddCategory(defaultCategory, nil) + podcast.IAuthor = user.Name + + return &podcast, nil +} + +func (v *VimeoBuilder) getVideoSize(video *vimeo.Video) int64 { + // Very approximate video file size + return int64(float64(video.Duration*video.Width*video.Height) * 0.38848958333) +} + +type getVideosFunc func(id string, opt *vimeo.ListVideoOptions) ([]*vimeo.Video, *vimeo.Response, error) + +func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, podcast *itunes.Podcast, feed *api.Feed) error { + opt := vimeo.ListVideoOptions{} + opt.Page = 1 + opt.PerPage = vimeoDefaultPageSize + + added := 0 + + for { + videos, response, err := getVideos(feed.ItemId, &opt) + if err != nil { + return errors.Wrapf(err, "failed to query videos (error %d %s)", response.StatusCode, response.Status) + } + + for _, video := range videos { + item := itunes.Item{} + + item.GUID = strconv.Itoa(video.GetID()) + item.Link = video.Link + item.Title = video.Name + item.Description = video.Description + if item.Description == "" { + item.Description = " " // Videos can be without description, workaround for AddItem + } + + item.AddDuration(int64(video.Duration)) + item.AddPubDate(&video.CreatedTime) + item.AddImage(v.selectImage(video.Pictures, feed.Quality)) + + size := v.getVideoSize(video) + item.AddEnclosure(makeEnclosure(feed, item.GUID, size)) + + _, err = podcast.AddItem(item) + if err != nil { + return errors.Wrapf(err, "failed to add episode %s (%s)", item.GUID, item.Title) + } + + added++ + } + + if added >= feed.PageSize || response.NextPage == "" { + return nil + } + + opt.Page++ + } +} + +func (v *VimeoBuilder) Build(feed *api.Feed) (podcast *itunes.Podcast, err error) { + if feed.LinkType == api.Channel { + if podcast, err = v.queryChannel(feed); err == nil { + err = v.queryVideos(v.client.Channels.ListVideo, podcast, feed) + } + + return + } + + if feed.LinkType == api.Group { + if podcast, err = v.queryGroup(feed); err == nil { + err = v.queryVideos(v.client.Groups.ListVideo, podcast, feed) + } + + return + } + + if feed.LinkType == api.User { + if podcast, err = v.queryUser(feed); err == nil { + err = v.queryVideos(v.client.Users.ListVideo, podcast, feed) + } + + return + } + + err = errors.New("unsupported feed type") + return +} + +func NewVimeoBuilder(ctx context.Context, token string) (*VimeoBuilder, error) { + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + tc := oauth2.NewClient(ctx, ts) + + client := vimeo.NewClient(tc) + return &VimeoBuilder{client}, nil +} diff --git a/pkg/builders/vimeo_test.go b/pkg/builders/vimeo_test.go new file mode 100644 index 0000000..3a112f2 --- /dev/null +++ b/pkg/builders/vimeo_test.go @@ -0,0 +1,80 @@ +package builders + +import ( + "context" + "os" + "testing" + + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" + "github.com/stretchr/testify/require" +) + +var ( + vimeoKey = os.Getenv("VIMEO_TEST_API_KEY") +) + +func TestQueryVimeoChannel(t *testing.T) { + builder, err := NewVimeoBuilder(context.Background(), vimeoKey) + require.NoError(t, err) + + podcast, err := builder.queryChannel(&api.Feed{ItemId: "staffpicks", Quality: api.HighQuality}) + require.NoError(t, err) + + require.Equal(t, "https://vimeo.com/channels/staffpicks", podcast.Link) + require.Equal(t, "Vimeo Staff Picks", podcast.Title) + require.Equal(t, "Vimeo Curation", podcast.IAuthor) + require.NotEmpty(t, podcast.Description) + require.NotEmpty(t, podcast.Image) + require.NotEmpty(t, podcast.IImage) +} + +func TestQueryVimeoGroup(t *testing.T) { + builder, err := NewVimeoBuilder(context.Background(), vimeoKey) + require.NoError(t, err) + + podcast, err := builder.queryGroup(&api.Feed{ItemId: "motion", Quality: api.HighQuality}) + require.NoError(t, err) + + require.Equal(t, "https://vimeo.com/groups/motion", podcast.Link) + require.Equal(t, "Motion Graphic Artists", podcast.Title) + require.Equal(t, "Danny Garcia", podcast.IAuthor) + require.NotEmpty(t, podcast.Description) + require.NotEmpty(t, podcast.Image) + require.NotEmpty(t, podcast.IImage) +} + +func TestQueryVimeoUser(t *testing.T) { + builder, err := NewVimeoBuilder(context.Background(), vimeoKey) + require.NoError(t, err) + + podcast, err := builder.queryUser(&api.Feed{ItemId: "motionarray", Quality: api.HighQuality}) + require.NoError(t, err) + + require.Equal(t, "https://vimeo.com/motionarray", podcast.Link) + require.Equal(t, "Motion Array", podcast.Title) + require.Equal(t, "Motion Array", podcast.IAuthor) + require.NotEmpty(t, podcast.Description) +} + +func TestQueryVimeoVideos(t *testing.T) { + builder, err := NewVimeoBuilder(context.Background(), vimeoKey) + require.NoError(t, err) + + feed := &itunes.Podcast{} + + err = builder.queryVideos(builder.client.Channels.ListVideo, feed, &api.Feed{ItemId: "staffpicks"}) + require.NoError(t, err) + + require.Equal(t, vimeoDefaultPageSize, len(feed.Items)) + + for _, item := range feed.Items { + require.NotEmpty(t, item.Title) + require.NotEmpty(t, item.Link) + require.NotEmpty(t, item.GUID) + require.NotEmpty(t, item.IDuration) + require.NotNil(t, item.Enclosure) + require.NotEmpty(t, item.Enclosure.URL) + require.True(t, item.Enclosure.Length > 0) + } +} diff --git a/pkg/builders/youtube.go b/pkg/builders/youtube.go new file mode 100644 index 0000000..6375c0f --- /dev/null +++ b/pkg/builders/youtube.go @@ -0,0 +1,324 @@ +package builders + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/BrianHicks/finch/duration" + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" + "github.com/pkg/errors" + "google.golang.org/api/youtube/v3" +) + +const ( + maxYoutubeResults = 50 + hdBytesPerSecond = 350000 + ldBytesPerSecond = 100000 +) + +type apiKey string + +func (key apiKey) Get() (string, string) { + return "key", string(key) +} + +type YouTubeBuilder struct { + client *youtube.Service + key apiKey +} + +func (yt *YouTubeBuilder) listChannels(linkType api.LinkType, id string) (*youtube.Channel, error) { + req := yt.client.Channels.List("id,snippet,contentDetails") + + if linkType == api.Channel { + req = req.Id(id) + } else if linkType == api.User { + req = req.ForUsername(id) + } else { + return nil, errors.New("unsupported link type") + } + + resp, err := req.Do(yt.key) + if err != nil { + return nil, errors.Wrapf(err, "failed to query channel") + } + + if len(resp.Items) == 0 { + return nil, api.ErrNotFound + } + + item := resp.Items[0] + return item, nil +} + +func (yt *YouTubeBuilder) listPlaylists(id, channelId string) (*youtube.Playlist, error) { + req := yt.client.Playlists.List("id,snippet") + + if id != "" { + req = req.Id(id) + } else { + req = req.ChannelId(channelId) + } + + resp, err := req.Do(yt.key) + if err != nil { + return nil, errors.Wrapf(err, "failed to query playlist") + } + + if len(resp.Items) == 0 { + return nil, api.ErrNotFound + } + + item := resp.Items[0] + return item, nil +} + +func (yt *YouTubeBuilder) listPlaylistItems(itemId string, pageToken string) ([]*youtube.PlaylistItem, string, error) { + req := yt.client.PlaylistItems.List("id,snippet").MaxResults(maxYoutubeResults).PlaylistId(itemId) + if pageToken != "" { + req = req.PageToken(pageToken) + } + + resp, err := req.Do(yt.key) + if err != nil { + return nil, "", errors.Wrap(err, "failed to query playlist items") + } + + return resp.Items, resp.NextPageToken, nil +} + +func (yt *YouTubeBuilder) parseDate(s string) (time.Time, error) { + date, err := time.Parse(time.RFC3339, s) + if err != nil { + return time.Time{}, errors.Wrapf(err, "failed to parse date: %s", s) + } + + return date, nil +} + +func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, quality api.Quality) string { + // Use high resolution thumbnails for high quality mode + // https://github.com/mxpv/Podsync/issues/14 + if quality == api.HighQuality { + if snippet.Maxres != nil { + return snippet.Maxres.Url + } + + if snippet.High != nil { + return snippet.High.Url + } + + if snippet.Medium != nil { + return snippet.Medium.Url + } + } + + return snippet.Default.Url +} + +func (yt *YouTubeBuilder) queryFeed(feed *api.Feed) (*itunes.Podcast, string, error) { + now := time.Now() + + if feed.LinkType == api.Channel || feed.LinkType == api.User { + channel, err := yt.listChannels(feed.LinkType, feed.ItemId) + if err != nil { + return nil, "", err + } + + itemId := channel.ContentDetails.RelatedPlaylists.Uploads + + link := "" + if feed.LinkType == api.Channel { + link = fmt.Sprintf("https://youtube.com/channel/%s", itemId) + } else { + link = fmt.Sprintf("https://youtube.com/user/%s", itemId) + } + + pubDate, err := yt.parseDate(channel.Snippet.PublishedAt) + if err != nil { + return nil, "", err + } + + title := channel.Snippet.Title + + podcast := itunes.New(title, link, channel.Snippet.Description, &pubDate, &now) + podcast.Generator = podsyncGenerator + + podcast.AddSubTitle(title) + podcast.AddCategory(defaultCategory, nil) + podcast.AddImage(yt.selectThumbnail(channel.Snippet.Thumbnails, feed.Quality)) + + return &podcast, itemId, nil + } + + if feed.LinkType == api.Playlist { + playlist, err := yt.listPlaylists(feed.ItemId, "") + if err != nil { + return nil, "", err + } + + link := fmt.Sprintf("https://youtube.com/playlist?list=%s", playlist.Id) + + snippet := playlist.Snippet + + pubDate, err := yt.parseDate(snippet.PublishedAt) + if err != nil { + return nil, "", err + } + + title := fmt.Sprintf("%s: %s", snippet.ChannelTitle, snippet.Title) + + podcast := itunes.New(title, link, snippet.Description, &pubDate, &now) + podcast.Generator = podsyncGenerator + + podcast.AddSubTitle(title) + podcast.AddCategory(defaultCategory, nil) + podcast.AddImage(yt.selectThumbnail(snippet.Thumbnails, feed.Quality)) + + return &podcast, playlist.Id, nil + } + + return nil, "", errors.New("unsupported link format") +} + +func (yt *YouTubeBuilder) getVideoSize(definition string, duration int64, fmt api.Format) int64 { + // Video size information requires 1 additional call for each video (1 feed = 50 videos = 50 calls), + // which is too expensive, so get approximated size depending on duration and definition params + var size int64 = 0 + + if definition == "hd" { + size = duration * hdBytesPerSecond + } else { + size = duration * ldBytesPerSecond + } + + // Some podcasts are coming in with exactly double the actual runtime and with the second half just silence. + // https://github.com/mxpv/Podsync/issues/6 + if fmt == api.AudioFormat { + size /= 2 + } + + return size +} + +func (yt *YouTubeBuilder) queryVideoDescriptions(ids []string, feed *api.Feed, podcast *itunes.Podcast) error { + req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(strings.Join(ids, ",")).Do(yt.key) + if err != nil { + return errors.Wrap(err, "failed to query video descriptions") + } + + for _, video := range req.Items { + snippet := video.Snippet + + item := itunes.Item{} + + item.GUID = video.Id + item.Link = fmt.Sprintf("https://youtube.com/watch?v=%s", video.Id) + item.Title = snippet.Title + item.Description = snippet.Description + item.ISubtitle = snippet.Title + + // Select thumbnail + + item.AddImage(yt.selectThumbnail(snippet.Thumbnails, feed.Quality)) + + // Parse publication date + + pubDate, err := yt.parseDate(snippet.PublishedAt) + if err != nil { + return errors.Wrapf(err, "failed to parse video publish date: %s", snippet.PublishedAt) + } + + item.AddPubDate(&pubDate) + + // Parse duration + + d, err := duration.FromString(video.ContentDetails.Duration) + if err != nil { + return errors.Wrapf(err, "failed to parse duration %s", video.ContentDetails.Duration) + } + + seconds := int64(d.ToDuration().Seconds()) + item.AddDuration(seconds) + + // Add download links + + size := yt.getVideoSize(video.ContentDetails.Definition, seconds, feed.Format) + item.AddEnclosure(makeEnclosure(feed, video.Id, size)) + + // podcast.AddItem requires description to be not empty, use workaround + + if item.Description == "" { + item.Description = " " + } + + _, err = podcast.AddItem(item) + if err != nil { + return errors.Wrapf(err, "failed to add item to podcast (id '%s')", video.Id) + } + } + + return nil +} + +func (yt *YouTubeBuilder) queryItems(itemId string, feed *api.Feed, podcast *itunes.Podcast) error { + pageToken := "" + count := 0 + + for { + items, pageToken, err := yt.listPlaylistItems(itemId, pageToken) + if err != nil { + return err + } + + if len(items) == 0 { + return nil + } + + // Extract video ids + ids := make([]string, len(items)) + for index, item := range items { + ids[index] = item.Snippet.ResourceId.VideoId + count++ + } + + // Query video descriptions from the list of ids + if err := yt.queryVideoDescriptions(ids, feed, podcast); err != nil { + return err + } + + if count >= feed.PageSize || pageToken == "" { + return nil + } + } +} + +func (yt *YouTubeBuilder) Build(feed *api.Feed) (*itunes.Podcast, error) { + + // Query general information about feed (title, description, lang, etc) + + podcast, itemId, err := yt.queryFeed(feed) + if err != nil { + return nil, err + } + + // Get video descriptions + + if err := yt.queryItems(itemId, feed, podcast); err != nil { + return nil, err + } + + return podcast, nil +} + +func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) { + yt, err := youtube.New(&http.Client{}) + if err != nil { + return nil, errors.Wrap(err, "failed to create youtube client") + } + + return &YouTubeBuilder{client: yt, key: apiKey(key)}, nil +} diff --git a/pkg/builders/youtube_test.go b/pkg/builders/youtube_test.go new file mode 100644 index 0000000..36fec74 --- /dev/null +++ b/pkg/builders/youtube_test.go @@ -0,0 +1,56 @@ +package builders + +import ( + "os" + "testing" + + "github.com/mxpv/podsync/pkg/api" + "github.com/stretchr/testify/require" +) + +var ytKey = os.Getenv("YOUTUBE_TEST_API_KEY") + +func TestQueryYTChannel(t *testing.T) { + if testing.Short() { + t.Skip("skipping YT test in short mode") + } + + builder, err := NewYouTubeBuilder(ytKey) + require.NoError(t, err) + + channel, err := builder.listChannels(api.Channel, "UC2yTVSttx7lxAOAzx1opjoA") + require.NoError(t, err) + require.Equal(t, "UC2yTVSttx7lxAOAzx1opjoA", channel.Id) + + channel, err = builder.listChannels(api.User, "fxigr1") + require.NoError(t, err) + require.Equal(t, "UCr_fwF-n-2_olTYd-m3n32g", channel.Id) +} + +func TestBuildYTFeed(t *testing.T) { + if testing.Short() { + t.Skip("skipping YT test in short mode") + } + + builder, err := NewYouTubeBuilder(ytKey) + require.NoError(t, err) + + podcast, err := builder.Build(&api.Feed{ + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "UCupvZG-5ko_eiXAupbDfxWw", + PageSize: maxYoutubeResults, + }) + require.NoError(t, err) + + require.Equal(t, "CNN", podcast.Title) + require.NotEmpty(t, podcast.Description) + + require.Equal(t, 50, len(podcast.Items)) + + for _, item := range podcast.Items { + require.NotEmpty(t, item.Title) + require.NotEmpty(t, item.Link) + require.NotEmpty(t, item.IDuration) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..8719a55 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,66 @@ +package config + +import ( + "strings" + + "github.com/spf13/viper" +) + +const FileName = "podsync" + +type AppConfig struct { + YouTubeApiKey string `yaml:"youtubeApiKey"` + VimeoApiKey string `yaml:"vimeoApiKey"` + PatreonClientId string `yaml:"patreonClientId"` + PatreonSecret string `yaml:"patreonSecret"` + PatreonRedirectURL string `yaml:"patreonRedirectUrl"` + PostgresConnectionURL string `yaml:"postgresConnectionUrl"` + RedisURL string `yaml:"redisUrl"` + CookieSecret string `yaml:"cookieSecret"` + AssetsPath string `yaml:"assetsPath"` + TemplatesPath string `yaml:"templatesPath"` +} + +func ReadConfiguration() (cfg *AppConfig, err error) { + viper.SetConfigName(FileName) + + // Configuration file + viper.AddConfigPath(".") + viper.AddConfigPath("/app/config/") + + // Env variables + viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + envmap := map[string]string{ + "youtubeApiKey": "YOUTUBE_API_KEY", + "vimeoApiKey": "VIMEO_API_KEY", + "patreonClientId": "PATREON_CLIENT_ID", + "patreonSecret": "PATREON_SECRET", + "patreonRedirectUrl": "PATREON_REDIRECT_URL", + "postgresConnectionUrl": "POSTGRES_CONNECTION_URL", + "redisUrl": "REDIS_CONNECTION_URL", + "cookieSecret": "COOKIE_SECRET", + "assetsPath": "ASSETS_PATH", + "templatesPath": "TEMPLATES_PATH", + } + + for k, v := range envmap { + viper.BindEnv(k, v) + } + + err = viper.ReadInConfig() + if err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { + return + } + + // Ignore file not found error + err = nil + } + + cfg = &AppConfig{} + + viper.Unmarshal(cfg) + return +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 0000000..437a495 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,71 @@ +package config + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +const YamlConfig = ` +youtubeApiKey: "1" +vimeoApiKey: "2" +patreonClientId: "3" +patreonSecret: "4" +postgresConnectionUrl: "5" +cookieSecret: "6" +patreonRedirectUrl: "7" +assetsPath: "8" +templatesPath: "9" +` + +func TestReadYaml(t *testing.T) { + defer viper.Reset() + + err := ioutil.WriteFile("./podsync.yaml", []byte(YamlConfig), 0644) + defer os.Remove("./podsync.yaml") + require.NoError(t, err) + + cfg, err := ReadConfiguration() + require.NoError(t, err) + + require.Equal(t, "1", cfg.YouTubeApiKey) + require.Equal(t, "2", cfg.VimeoApiKey) + require.Equal(t, "3", cfg.PatreonClientId) + require.Equal(t, "4", cfg.PatreonSecret) + require.Equal(t, "5", cfg.PostgresConnectionURL) + require.Equal(t, "6", cfg.CookieSecret) + require.Equal(t, "7", cfg.PatreonRedirectURL) + require.Equal(t, "8", cfg.AssetsPath) + require.Equal(t, "9", cfg.TemplatesPath) +} + +func TestReadEnv(t *testing.T) { + defer viper.Reset() + defer os.Clearenv() + + os.Setenv("YOUTUBE_API_KEY", "11") + os.Setenv("VIMEO_API_KEY", "22") + os.Setenv("PATREON_CLIENT_ID", "33") + os.Setenv("PATREON_SECRET", "44") + os.Setenv("POSTGRES_CONNECTION_URL", "55") + os.Setenv("COOKIE_SECRET", "66") + os.Setenv("PATREON_REDIRECT_URL", "77") + os.Setenv("ASSETS_PATH", "88") + os.Setenv("TEMPLATES_PATH", "99") + + cfg, err := ReadConfiguration() + require.NoError(t, err) + + require.Equal(t, "11", cfg.YouTubeApiKey) + require.Equal(t, "22", cfg.VimeoApiKey) + require.Equal(t, "33", cfg.PatreonClientId) + require.Equal(t, "44", cfg.PatreonSecret) + require.Equal(t, "55", cfg.PostgresConnectionURL) + require.Equal(t, "66", cfg.CookieSecret) + require.Equal(t, "77", cfg.PatreonRedirectURL) + require.Equal(t, "88", cfg.AssetsPath) + require.Equal(t, "99", cfg.TemplatesPath) +} diff --git a/pkg/feeds/feeds.go b/pkg/feeds/feeds.go new file mode 100644 index 0000000..39877cd --- /dev/null +++ b/pkg/feeds/feeds.go @@ -0,0 +1,114 @@ +package feeds + +import ( + "fmt" + "time" + + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" + "github.com/pkg/errors" +) + +const ( + maxPageSize = 150 +) + +type service struct { + id id + storage storage + builders map[api.Provider]builder +} + +func (s *service) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) { + feed, err := parseURL(req.URL) + if err != nil { + return "", errors.Wrapf(err, "failed to create feed for URL: %s", req.URL) + } + + // Make sure builder exists for this provider + _, ok := s.builders[feed.Provider] + if !ok { + return "", fmt.Errorf("failed to get builder for URL: %s", req.URL) + } + + // Set default fields + feed.PageSize = api.DefaultPageSize + feed.Format = api.VideoFormat + feed.Quality = api.HighQuality + feed.FeatureLevel = api.DefaultFeatures + feed.LastAccess = time.Now().UTC() + + if identity.FeatureLevel > 0 { + feed.Quality = req.Quality + feed.Format = req.Format + feed.FeatureLevel = identity.FeatureLevel + feed.PageSize = req.PageSize + if feed.PageSize > maxPageSize { + feed.PageSize = maxPageSize + } + } + + // Generate short id + hashId, err := s.id.Generate(feed) + if err != nil { + return "", errors.Wrap(err, "failed to generate id for feed") + } + + feed.HashId = hashId + + // Save to database + if err := s.storage.CreateFeed(feed); err != nil { + return "", errors.Wrap(err, "failed to save feed to database") + } + + return hashId, nil +} + +func (s *service) GetFeed(hashId string) (*itunes.Podcast, error) { + feed, err := s.GetMetadata(hashId) + if err != nil { + return nil, err + } + + builder, ok := s.builders[feed.Provider] + if !ok { + return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashId) + } + + return builder.Build(feed) +} + +func (s *service) GetMetadata(hashId string) (*api.Feed, error) { + return s.storage.GetFeed(hashId) +} + +type feedOption func(*service) + +func WithStorage(storage storage) feedOption { + return func(service *service) { + service.storage = storage + } +} + +func WithIdGen(id id) feedOption { + return func(service *service) { + service.id = id + } +} + +func WithBuilder(provider api.Provider, builder builder) feedOption { + return func(service *service) { + service.builders[provider] = builder + } +} + +func NewFeedService(opts ...feedOption) *service { + svc := &service{} + svc.builders = make(map[api.Provider]builder) + + for _, fn := range opts { + fn(svc) + } + + return svc +} diff --git a/pkg/feeds/feeds_test.go b/pkg/feeds/feeds_test.go new file mode 100644 index 0000000..7d81baa --- /dev/null +++ b/pkg/feeds/feeds_test.go @@ -0,0 +1,72 @@ +//go:generate mockgen -source=interfaces.go -destination=interfaces_mock_test.go -package=feeds + +package feeds + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/mxpv/podsync/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestService_CreateFeed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + id := NewMockid(ctrl) + id.EXPECT().Generate(gomock.Any()).Times(1).Return("123", nil) + + storage := NewMockstorage(ctrl) + storage.EXPECT().CreateFeed(gomock.Any()).Times(1).Return(nil) + + s := service{ + id: id, + storage: storage, + builders: map[api.Provider]builder{api.Youtube: nil}, + } + + req := &api.CreateFeedRequest{ + URL: "youtube.com/channel/123", + PageSize: 50, + Quality: api.HighQuality, + Format: api.VideoFormat, + } + + hashId, err := s.CreateFeed(req, &api.Identity{}) + require.NoError(t, err) + require.Equal(t, "123", hashId) +} + +func TestService_GetFeed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + feed := &api.Feed{Provider: api.Youtube} + + storage := NewMockstorage(ctrl) + storage.EXPECT().GetFeed("123").Times(1).Return(feed, nil) + + bld := NewMockbuilder(ctrl) + bld.EXPECT().Build(feed).Return(nil, nil) + + s := service{ + storage: storage, + builders: map[api.Provider]builder{api.Youtube: bld}, + } + + _, err := s.GetFeed("123") + require.NoError(t, err) +} + +func TestService_GetMetadata(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + storage := NewMockstorage(ctrl) + storage.EXPECT().GetFeed("123").Times(1).Return(&api.Feed{}, nil) + + s := service{storage: storage} + _, err := s.GetMetadata("123") + require.NoError(t, err) +} diff --git a/pkg/feeds/interfaces.go b/pkg/feeds/interfaces.go new file mode 100644 index 0000000..f85355e --- /dev/null +++ b/pkg/feeds/interfaces.go @@ -0,0 +1,19 @@ +package feeds + +import ( + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" +) + +type id interface { + Generate(feed *api.Feed) (string, error) +} + +type storage interface { + CreateFeed(feed *api.Feed) error + GetFeed(hashId string) (*api.Feed, error) +} + +type builder interface { + Build(feed *api.Feed) (podcast *itunes.Podcast, err error) +} diff --git a/pkg/feeds/interfaces_mock_test.go b/pkg/feeds/interfaces_mock_test.go new file mode 100644 index 0000000..a76822c --- /dev/null +++ b/pkg/feeds/interfaces_mock_test.go @@ -0,0 +1,131 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: interfaces.go + +package feeds + +import ( + gomock "github.com/golang/mock/gomock" + podcast "github.com/mxpv/podcast" + api "github.com/mxpv/podsync/pkg/api" + reflect "reflect" +) + +// Mockid is a mock of id interface +type Mockid struct { + ctrl *gomock.Controller + recorder *MockidMockRecorder +} + +// MockidMockRecorder is the mock recorder for Mockid +type MockidMockRecorder struct { + mock *Mockid +} + +// NewMockid creates a new mock instance +func NewMockid(ctrl *gomock.Controller) *Mockid { + mock := &Mockid{ctrl: ctrl} + mock.recorder = &MockidMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (_m *Mockid) EXPECT() *MockidMockRecorder { + return _m.recorder +} + +// Generate mocks base method +func (_m *Mockid) Generate(feed *api.Feed) (string, error) { + ret := _m.ctrl.Call(_m, "Generate", feed) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Generate indicates an expected call of Generate +func (_mr *MockidMockRecorder) Generate(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Generate", reflect.TypeOf((*Mockid)(nil).Generate), arg0) +} + +// Mockstorage is a mock of storage interface +type Mockstorage struct { + ctrl *gomock.Controller + recorder *MockstorageMockRecorder +} + +// MockstorageMockRecorder is the mock recorder for Mockstorage +type MockstorageMockRecorder struct { + mock *Mockstorage +} + +// NewMockstorage creates a new mock instance +func NewMockstorage(ctrl *gomock.Controller) *Mockstorage { + mock := &Mockstorage{ctrl: ctrl} + mock.recorder = &MockstorageMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (_m *Mockstorage) EXPECT() *MockstorageMockRecorder { + return _m.recorder +} + +// CreateFeed mocks base method +func (_m *Mockstorage) CreateFeed(feed *api.Feed) error { + ret := _m.ctrl.Call(_m, "CreateFeed", feed) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateFeed indicates an expected call of CreateFeed +func (_mr *MockstorageMockRecorder) CreateFeed(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "CreateFeed", reflect.TypeOf((*Mockstorage)(nil).CreateFeed), arg0) +} + +// GetFeed mocks base method +func (_m *Mockstorage) GetFeed(hashId string) (*api.Feed, error) { + ret := _m.ctrl.Call(_m, "GetFeed", hashId) + ret0, _ := ret[0].(*api.Feed) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFeed indicates an expected call of GetFeed +func (_mr *MockstorageMockRecorder) GetFeed(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), arg0) +} + +// Mockbuilder is a mock of builder interface +type Mockbuilder struct { + ctrl *gomock.Controller + recorder *MockbuilderMockRecorder +} + +// MockbuilderMockRecorder is the mock recorder for Mockbuilder +type MockbuilderMockRecorder struct { + mock *Mockbuilder +} + +// NewMockbuilder creates a new mock instance +func NewMockbuilder(ctrl *gomock.Controller) *Mockbuilder { + mock := &Mockbuilder{ctrl: ctrl} + mock.recorder = &MockbuilderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (_m *Mockbuilder) EXPECT() *MockbuilderMockRecorder { + return _m.recorder +} + +// Build mocks base method +func (_m *Mockbuilder) Build(feed *api.Feed) (*podcast.Podcast, error) { + ret := _m.ctrl.Call(_m, "Build", feed) + ret0, _ := ret[0].(*podcast.Podcast) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Build indicates an expected call of Build +func (_mr *MockbuilderMockRecorder) Build(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Build", reflect.TypeOf((*Mockbuilder)(nil).Build), arg0) +} diff --git a/pkg/feeds/url.go b/pkg/feeds/url.go new file mode 100644 index 0000000..81b0bd4 --- /dev/null +++ b/pkg/feeds/url.go @@ -0,0 +1,150 @@ +package feeds + +import ( + "net/url" + "strings" + + "github.com/mxpv/podsync/pkg/api" + "github.com/pkg/errors" +) + +func parseURL(link string) (*api.Feed, error) { + if !strings.HasPrefix(link, "http") { + link = "https://" + link + } + + parsed, err := url.Parse(link) + if err != nil { + err = errors.Wrapf(err, "failed to parse url: %s", link) + return nil, err + } + + feed := &api.Feed{} + + if strings.HasSuffix(parsed.Host, "youtube.com") { + kind, id, err := parseYoutubeURL(parsed) + if err != nil { + return nil, err + } + + feed.Provider = api.Youtube + feed.LinkType = kind + feed.ItemId = id + + return feed, nil + } + + if strings.HasSuffix(parsed.Host, "vimeo.com") { + kind, id, err := parseVimeoURL(parsed) + if err != nil { + return nil, err + } + + feed.Provider = api.Vimeo + feed.LinkType = kind + feed.ItemId = id + + return feed, nil + } + + return nil, errors.New("unsupported URL host") +} + +func parseYoutubeURL(parsed *url.URL) (kind api.LinkType, id string, err error) { + path := parsed.EscapedPath() + + // https://www.youtube.com/playlist?list=PLCB9F975ECF01953C + if strings.HasPrefix(path, "/playlist") { + kind = api.Playlist + + id = parsed.Query().Get("list") + if id != "" { + return + } + + err = errors.New("invalid playlist link") + return + } + + // - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og + // - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos + if strings.HasPrefix(path, "/channel") { + kind = api.Channel + parts := strings.Split(parsed.EscapedPath(), "/") + if len(parts) <= 2 { + err = errors.New("invalid youtube channel link") + return + } + + id = parts[2] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + // - https://www.youtube.com/user/fxigr1 + if strings.HasPrefix(path, "/user") { + kind = api.User + + parts := strings.Split(parsed.EscapedPath(), "/") + if len(parts) <= 2 { + err = errors.New("invalid user link") + return + } + + id = parts[2] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + err = errors.New("unsupported link format") + return +} + +func parseVimeoURL(parsed *url.URL) (kind api.LinkType, id string, err error) { + parts := strings.Split(parsed.EscapedPath(), "/") + + if len(parts) <= 1 { + err = errors.New("invalid vimeo link path") + return + } + + if parts[1] == "groups" { + kind = api.Group + } else if parts[1] == "channels" { + kind = api.Channel + } else { + kind = api.User + } + + if kind == api.Group || kind == api.Channel { + if len(parts) <= 2 { + err = errors.New("invalid channel link") + return + } + + id = parts[2] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + if kind == api.User { + id = parts[1] + if id == "" { + err = errors.New("invalid id") + } + + return + } + + err = errors.New("unsupported link format") + return +} diff --git a/pkg/feeds/url_test.go b/pkg/feeds/url_test.go new file mode 100644 index 0000000..fb6fc6a --- /dev/null +++ b/pkg/feeds/url_test.go @@ -0,0 +1,107 @@ +package feeds + +import ( + "net/url" + "testing" + + "github.com/mxpv/podsync/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestParseYoutubeURL_Playlist(t *testing.T) { + link, _ := url.ParseRequestURI("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C") + kind, id, err := parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.Playlist, kind) + require.Equal(t, "PLCB9F975ECF01953C", id) +} + +func TestParseYoutubeURL_Channel(t *testing.T) { + link, _ := url.ParseRequestURI("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og") + kind, id, err := parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "UC5XPnUk8Vvv_pWslhwom6Og", id) + + link, _ = url.ParseRequestURI("https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos") + kind, id, err = parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id) +} + +func TestParseYoutubeURL_User(t *testing.T) { + link, _ := url.ParseRequestURI("https://youtube.com/user/fxigr1") + kind, id, err := parseYoutubeURL(link) + require.NoError(t, err) + require.Equal(t, api.User, kind) + require.Equal(t, "fxigr1", id) +} + +func TestParseYoutubeURL_InvalidLink(t *testing.T) { + link, _ := url.ParseRequestURI("https://www.youtube.com/user///") + _, _, err := parseYoutubeURL(link) + require.Error(t, err) + + link, _ = url.ParseRequestURI("https://www.youtube.com/channel//videos") + _, _, err = parseYoutubeURL(link) + require.Error(t, err) +} + +func TestParseVimeoURL_Group(t *testing.T) { + link, _ := url.ParseRequestURI("https://vimeo.com/groups/109") + kind, id, err := parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) + + link, _ = url.ParseRequestURI("http://vimeo.com/groups/109") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) + + link, _ = url.ParseRequestURI("http://www.vimeo.com/groups/109") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) + + link, _ = url.ParseRequestURI("https://vimeo.com/groups/109/videos/") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Group, kind) + require.Equal(t, "109", id) +} + +func TestParseVimeoURL_Channel(t *testing.T) { + link, _ := url.ParseRequestURI("https://vimeo.com/channels/staffpicks") + kind, id, err := parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "staffpicks", id) + + link, _ = url.ParseRequestURI("http://vimeo.com/channels/staffpicks/146224925") + kind, id, err = parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.Channel, kind) + require.Equal(t, "staffpicks", id) +} + +func TestParseVimeoURL_User(t *testing.T) { + link, _ := url.ParseRequestURI("https://vimeo.com/awhitelabelproduct") + kind, id, err := parseVimeoURL(link) + require.NoError(t, err) + require.Equal(t, api.User, kind) + require.Equal(t, "awhitelabelproduct", id) +} + +func TestParseVimeoURL_InvalidLink(t *testing.T) { + link, _ := url.ParseRequestURI("http://www.apple.com") + _, _, err := parseVimeoURL(link) + require.Error(t, err) + + link, _ = url.ParseRequestURI("http://www.vimeo.com") + _, _, err = parseVimeoURL(link) + require.Error(t, err) +} diff --git a/pkg/id/hashids.go b/pkg/id/hashids.go new file mode 100644 index 0000000..44c1314 --- /dev/null +++ b/pkg/id/hashids.go @@ -0,0 +1,32 @@ +package id + +import ( + "hash/fnv" + "time" + + "github.com/mxpv/podsync/pkg/api" + "github.com/ventu-io/go-shortid" +) + +type hashId struct { + sid *shortid.Shortid +} + +func hashString(s string) int { + h := fnv.New32a() + h.Write([]byte(s)) + return int(h.Sum32()) +} + +func (h *hashId) Generate(feed *api.Feed) (string, error) { + return h.sid.Generate() +} + +func NewIdGenerator() (*hashId, error) { + sid, err := shortid.New(1, shortid.DefaultABC, uint64(time.Now().UnixNano())) + if err != nil { + return nil, err + } + + return &hashId{sid}, nil +} diff --git a/pkg/id/hashids_test.go b/pkg/id/hashids_test.go new file mode 100644 index 0000000..e06e8fd --- /dev/null +++ b/pkg/id/hashids_test.go @@ -0,0 +1,19 @@ +package id + +import ( + "testing" + + "github.com/mxpv/podsync/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestEncode(t *testing.T) { + hid, err := NewIdGenerator() + require.NoError(t, err) + + feed := &api.Feed{} + + hash, err := hid.Generate(feed) + require.NoError(t, err) + require.NotEmpty(t, hash) +} diff --git a/pkg/server/server.go b/pkg/server/server.go new file mode 100644 index 0000000..bb043c7 --- /dev/null +++ b/pkg/server/server.go @@ -0,0 +1,263 @@ +package server + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "log" + "net/http" + "path" + + "github.com/gin-contrib/sessions" + "github.com/gin-gonic/gin" + "github.com/mxpv/patreon-go" + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" + "github.com/mxpv/podsync/pkg/config" + "golang.org/x/oauth2" +) + +const ( + campaignId = "278915" + identitySessionKey = "identity" +) + +type feed interface { + CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) + GetFeed(hashId string) (*itunes.Podcast, error) + GetMetadata(hashId string) (*api.Feed, error) +} + +func MakeHandlers(feed feed, cfg *config.AppConfig) http.Handler { + r := gin.New() + r.Use(gin.Recovery()) + + store := sessions.NewCookieStore([]byte(cfg.CookieSecret)) + r.Use(sessions.Sessions("podsync", store)) + + // Static files + HTML + + log.Printf("using assets path: %s", cfg.AssetsPath) + if cfg.AssetsPath != "" { + r.Static("/assets", cfg.AssetsPath) + } + + log.Printf("using templates path: %s", cfg.TemplatesPath) + if cfg.TemplatesPath != "" { + r.LoadHTMLGlob(path.Join(cfg.TemplatesPath, "*.html")) + } + + conf := &oauth2.Config{ + ClientID: cfg.PatreonClientId, + ClientSecret: cfg.PatreonSecret, + RedirectURL: cfg.PatreonRedirectURL, + Scopes: []string{"users", "pledges-to-me", "my-campaign"}, + Endpoint: oauth2.Endpoint{ + AuthURL: patreon.AuthorizationURL, + TokenURL: patreon.AccessTokenURL, + }, + } + + r.GET("/", func(c *gin.Context) { + s := sessions.Default(c) + + identity := &api.Identity{ + FeatureLevel: api.DefaultFeatures, + } + + buf, ok := s.Get(identitySessionKey).(string) + if ok { + // We are failed to deserialize Identity structure, do cleanup, force user to login again + if err := json.Unmarshal([]byte(buf), identity); err != nil { + s.Clear() + s.Save() + } + } + + c.HTML(http.StatusOK, "index.html", identity) + }) + + r.GET("/login", func(c *gin.Context) { + state := randToken() + + s := sessions.Default(c) + s.Set("state", state) + s.Save() + + authURL := conf.AuthCodeURL(state) + c.Redirect(http.StatusFound, authURL) + }) + + r.GET("/logout", func(c *gin.Context) { + s := sessions.Default(c) + s.Clear() + s.Save() + + c.Redirect(http.StatusFound, "/") + }) + + r.GET("/patreon", func(c *gin.Context) { + // Validate session state + s := sessions.Default(c) + state := s.Get("state") + if state != c.Query("state") { + c.String(http.StatusUnauthorized, "invalid state") + return + } + + // Exchange code with tokens + token, err := conf.Exchange(c.Request.Context(), c.Query("code")) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + // Create Patreon client + tc := conf.Client(c.Request.Context(), token) + client := patreon.NewClient(tc) + + // Query user info from Patreon + user, err := client.FetchUser() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + + // Determine feature level + level := api.DefaultFeatures + amount := 0 + for _, item := range user.Included.Items { + pledge, ok := item.(*patreon.Pledge) + if ok { + amount += pledge.Attributes.AmountCents + } + } + + if amount >= 100 { + level = api.ExtendedFeatures + } + + identity := &api.Identity{ + UserId: user.Data.Id, + FullName: user.Data.Attributes.FullName, + Email: user.Data.Attributes.Email, + ProfileURL: user.Data.Attributes.URL, + FeatureLevel: level, + } + + // Serialize identity and return cookies + buf, err := json.Marshal(identity) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + + s.Clear() + s.Set(identitySessionKey, string(buf)) + s.Save() + + c.Redirect(http.StatusFound, "/") + }) + + // GET /robots.txt + r.GET("/robots.txt", func(c *gin.Context) { + c.String(http.StatusOK, `User-agent: * +Allow: /$ +Disallow: / +Host: www.podsync.net`) + }) + + // REST API + + r.GET("/api/ping", func(c *gin.Context) { + c.String(http.StatusOK, "ok") + }) + + r.POST("/api/create", func(c *gin.Context) { + req := &api.CreateFeedRequest{} + + if err := c.BindJSON(req); err != nil { + c.JSON(badRequest(err)) + return + } + + s := sessions.Default(c) + + identity := &api.Identity{ + FeatureLevel: api.DefaultFeatures, + } + + buf, ok := s.Get(identitySessionKey).(string) + if ok { + // We are failed to deserialize Identity structure, do cleanup, force user to login again + if err := json.Unmarshal([]byte(buf), identity); err != nil { + s.Clear() + s.Save() + } + } + + hashId, err := feed.CreateFeed(req, identity) + if err != nil { + c.JSON(internalError(err)) + return + } + + c.JSON(http.StatusOK, gin.H{"id": hashId}) + }) + + r.NoRoute(func(c *gin.Context) { + hashId := c.Request.URL.Path[1:] + if hashId == "" || len(hashId) > 12 { + c.String(http.StatusBadRequest, "invalid feed id") + return + } + + podcast, err := feed.GetFeed(hashId) + if err != nil { + code := http.StatusInternalServerError + if err == api.ErrNotFound { + code = http.StatusNotFound + } else { + log.Printf("server error (hash id: %s): %v", hashId, err) + } + + c.String(code, err.Error()) + return + } + + c.Data(http.StatusOK, "application/rss+xml; charset=UTF-8", podcast.Bytes()) + }) + + r.GET("/api/metadata/:hashId", func(c *gin.Context) { + hashId := c.Param("hashId") + if hashId == "" || len(hashId) > 12 { + c.String(http.StatusBadRequest, "invalid feed id") + return + } + + feed, err := feed.GetMetadata(hashId) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + + c.JSON(http.StatusOK, feed) + }) + + return r +} + +func badRequest(err error) (int, interface{}) { + return http.StatusBadRequest, gin.H{"error": err.Error()} +} + +func internalError(err error) (int, interface{}) { + log.Printf("server error: %v", err) + return http.StatusInternalServerError, gin.H{"error": err.Error()} +} + +func randToken() string { + b := make([]byte, 32) + rand.Read(b) + return base64.StdEncoding.EncodeToString(b) +} diff --git a/pkg/server/server_mock_test.go b/pkg/server/server_mock_test.go new file mode 100644 index 0000000..a808718 --- /dev/null +++ b/pkg/server/server_mock_test.go @@ -0,0 +1,73 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: server.go + +package server + +import ( + gomock "github.com/golang/mock/gomock" + podcast "github.com/mxpv/podcast" + api "github.com/mxpv/podsync/pkg/api" + reflect "reflect" +) + +// Mockfeed is a mock of feed interface +type Mockfeed struct { + ctrl *gomock.Controller + recorder *MockfeedMockRecorder +} + +// MockfeedMockRecorder is the mock recorder for Mockfeed +type MockfeedMockRecorder struct { + mock *Mockfeed +} + +// NewMockfeed creates a new mock instance +func NewMockfeed(ctrl *gomock.Controller) *Mockfeed { + mock := &Mockfeed{ctrl: ctrl} + mock.recorder = &MockfeedMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (_m *Mockfeed) EXPECT() *MockfeedMockRecorder { + return _m.recorder +} + +// CreateFeed mocks base method +func (_m *Mockfeed) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) { + ret := _m.ctrl.Call(_m, "CreateFeed", req, identity) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFeed indicates an expected call of CreateFeed +func (_mr *MockfeedMockRecorder) CreateFeed(arg0, arg1 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "CreateFeed", reflect.TypeOf((*Mockfeed)(nil).CreateFeed), arg0, arg1) +} + +// GetFeed mocks base method +func (_m *Mockfeed) GetFeed(hashId string) (*podcast.Podcast, error) { + ret := _m.ctrl.Call(_m, "GetFeed", hashId) + ret0, _ := ret[0].(*podcast.Podcast) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFeed indicates an expected call of GetFeed +func (_mr *MockfeedMockRecorder) GetFeed(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetFeed", reflect.TypeOf((*Mockfeed)(nil).GetFeed), arg0) +} + +// GetMetadata mocks base method +func (_m *Mockfeed) GetMetadata(hashId string) (*api.Feed, error) { + ret := _m.ctrl.Call(_m, "GetMetadata", hashId) + ret0, _ := ret[0].(*api.Feed) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetadata indicates an expected call of GetMetadata +func (_mr *MockfeedMockRecorder) GetMetadata(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetMetadata", reflect.TypeOf((*Mockfeed)(nil).GetMetadata), arg0) +} diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go new file mode 100644 index 0000000..7a5864d --- /dev/null +++ b/pkg/server/server_test.go @@ -0,0 +1,133 @@ +//go:generate mockgen -source=server.go -destination=server_mock_test.go -package=server + +package server + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/golang/mock/gomock" + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" + "github.com/mxpv/podsync/pkg/config" + "github.com/stretchr/testify/require" +) + +var cfg = &config.AppConfig{} + +func TestCreateFeed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + req := &api.CreateFeedRequest{ + URL: "https://youtube.com/channel/123", + PageSize: 55, + Quality: api.LowQuality, + Format: api.AudioFormat, + } + + feed := NewMockfeed(ctrl) + feed.EXPECT().CreateFeed(gomock.Eq(req), gomock.Any()).Times(1).Return("456", nil) + + srv := httptest.NewServer(MakeHandlers(feed, cfg)) + defer srv.Close() + + query := `{"url": "https://youtube.com/channel/123", "page_size": 55, "quality": "low", "format": "audio"}` + resp, err := http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + require.JSONEq(t, `{"id": "456"}`, readBody(t, resp)) +} + +func TestCreateInvalidFeed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + srv := httptest.NewServer(MakeHandlers(NewMockfeed(ctrl), cfg)) + defer srv.Close() + + query := `{}` + resp, err := http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + query = `{"url": "not a url", "page_size": 55, "quality": "low", "format": "audio"}` + resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + query = `{"url": "https://youtube.com/channel/123", "page_size": 1, "quality": "low", "format": "audio"}` + resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + query = `{"url": "https://youtube.com/channel/123", "page_size": 151, "quality": "low", "format": "audio"}` + resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "xyz", "format": "audio"}` + resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "low", "format": "xyz"}` + resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "low", "format": ""}` + resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + query = `{"url": "https://youtube.com/channel/123", "page_size": 50, "quality": "", "format": "audio"}` + resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query)) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +func TestGetFeed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + podcast := itunes.New("", "", "", nil, nil) + + feed := NewMockfeed(ctrl) + feed.EXPECT().GetFeed("123").Return(&podcast, nil) + + srv := httptest.NewServer(MakeHandlers(feed, cfg)) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/123") + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) +} + +func TestGetMetadata(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + feed := NewMockfeed(ctrl) + feed.EXPECT().GetMetadata("123").Times(1).Return(&api.Feed{}, nil) + + srv := httptest.NewServer(MakeHandlers(feed, cfg)) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/api/metadata/123") + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) +} + +func readBody(t *testing.T, resp *http.Response) string { + buf, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + + require.NoError(t, err) + + return string(buf) +} diff --git a/pkg/storage/pg.go b/pkg/storage/pg.go new file mode 100644 index 0000000..a88cb6c --- /dev/null +++ b/pkg/storage/pg.go @@ -0,0 +1,76 @@ +package storage + +import ( + "log" + "net" + "strings" + "time" + + "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy" + "github.com/go-pg/pg" + "github.com/mxpv/podsync/pkg/api" + "github.com/pkg/errors" +) + +type PgConfig struct { + ConnectionUrl string `yaml:"connectionUrl"` +} + +type PgStorage struct { + db *pg.DB +} + +func (p *PgStorage) CreateFeed(feed *api.Feed) error { + feed.LastAccess = time.Now().UTC() + _, err := p.db.Model(feed).OnConflict("DO NOTHING").Insert() + if err != nil { + return errors.Wrap(err, "failed to create feed") + } + + return nil +} + +func (p *PgStorage) GetFeed(hashId string) (*api.Feed, error) { + lastAccess := time.Now().UTC() + + feed := &api.Feed{} + _, err := p.db.Model(feed). + Set("last_access = ?", lastAccess). + Where("hash_id = ?", hashId). + Returning("*"). + Update() + + return feed, err +} + +func NewPgStorage(config *PgConfig) (*PgStorage, error) { + opts, err := pg.ParseURL(config.ConnectionUrl) + if err != nil { + return nil, err + } + + // If host format is "projection:region:host", than use Google SQL Proxy + // See https://github.com/go-pg/pg/issues/576 + if strings.Count(opts.Addr, ":") == 2 { + log.Print("using GCP SQL proxy") + opts.Dialer = func(network, addr string) (net.Conn, error) { + return proxy.Dial(addr) + } + } + + db := pg.Connect(opts) + + // Check database connectivity + if _, err := db.ExecOne("SELECT 1"); err != nil { + db.Close() + return nil, errors.Wrap(err, "failed to check database connectivity") + } + + log.Print("running update script") + if _, err := db.Exec(installScript); err != nil { + return nil, errors.Wrap(err, "failed to upgrade database structure") + } + + storage := &PgStorage{db: db} + return storage, nil +} diff --git a/pkg/storage/pg_sql.go b/pkg/storage/pg_sql.go new file mode 100644 index 0000000..4a74a61 --- /dev/null +++ b/pkg/storage/pg_sql.go @@ -0,0 +1,38 @@ +package storage + +const installScript = ` +BEGIN; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'provider') THEN + CREATE TYPE provider AS ENUM ('youtube', 'vimeo'); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'link_type') THEN + CREATE TYPE link_type AS ENUM ('channel', 'playlist', 'user', 'group'); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'quality') THEN + CREATE TYPE quality AS ENUM ('high', 'low'); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'format') THEN + CREATE TYPE format AS ENUM ('audio', 'video'); + END IF; +END +$$; + +CREATE TABLE IF NOT EXISTS feeds ( + id BIGSERIAL PRIMARY KEY, + hash_id VARCHAR(12) NOT NULL CHECK (hash_id <> '') UNIQUE, + user_id VARCHAR(32) NULL, + item_id VARCHAR(32) NOT NULL CHECK (item_id <> ''), + provider provider NOT NULL, + link_type link_type NOT NULL, + page_size INT NOT NULL DEFAULT 50, + format format NOT NULL DEFAULT 'video', + quality quality NOT NULL DEFAULT 'high', + feature_level INT NOT NULL DEFAULT 0, + last_access timestamp WITHOUT TIME ZONE NOT NULL +); + +COMMIT; +` diff --git a/pkg/storage/pg_test.go b/pkg/storage/pg_test.go new file mode 100644 index 0000000..ce2629b --- /dev/null +++ b/pkg/storage/pg_test.go @@ -0,0 +1,106 @@ +package storage + +import ( + "testing" + + "github.com/mxpv/podsync/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestPgStorage_CreateFeed(t *testing.T) { + feed := &api.Feed{ + HashId: "xyz", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", + } + + client := createClient(t) + err := client.CreateFeed(feed) + require.NoError(t, err) + require.True(t, feed.Id > 0) +} + +func TestPgStorage_CreateFeedWithDuplicate(t *testing.T) { + feed := &api.Feed{ + HashId: "123", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", + } + + client := createClient(t) + err := client.CreateFeed(feed) + require.NoError(t, err) + + // Ensure 1 record + count, err := client.db.Model(&api.Feed{}).Count() + require.NoError(t, err) + require.Equal(t, 1, count) + + // Insert duplicated feed + err = client.CreateFeed(feed) + require.NoError(t, err) + + // Check no duplicates inserted + count, err = client.db.Model(&api.Feed{}).Count() + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestPgStorage_GetFeed(t *testing.T) { + feed := &api.Feed{ + HashId: "xyz", + UserId: "123", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", + } + + client := createClient(t) + client.CreateFeed(feed) + + out, err := client.GetFeed("xyz") + require.NoError(t, err) + require.Equal(t, feed.Id, out.Id) +} + +func TestPgStorage_UpdateLastAccess(t *testing.T) { + feed := &api.Feed{ + HashId: "xyz", + UserId: "123", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", + } + + client := createClient(t) + err := client.CreateFeed(feed) + require.NoError(t, err) + + lastAccess := feed.LastAccess + require.True(t, lastAccess.Unix() > 0) + + last, err := client.GetFeed("xyz") + require.NoError(t, err) + + require.NotEmpty(t, last.HashId) + require.NotEmpty(t, last.UserId) + require.NotEmpty(t, last.Provider) + require.NotEmpty(t, last.LinkType) + require.NotEmpty(t, last.ItemId) + + require.True(t, last.LastAccess.UnixNano() > lastAccess.UnixNano()) +} + +const TestDatabaseConnectionUrl = "postgres://postgres:@localhost/podsync?sslmode=disable" + +func createClient(t *testing.T) *PgStorage { + pg, err := NewPgStorage(&PgConfig{ConnectionUrl: TestDatabaseConnectionUrl}) + require.NoError(t, err) + + _, err = pg.db.Model(&api.Feed{}).Where("1=1").Delete() + require.NoError(t, err) + + return pg +} diff --git a/pkg/storage/redis.go b/pkg/storage/redis.go new file mode 100644 index 0000000..dcc0b88 --- /dev/null +++ b/pkg/storage/redis.go @@ -0,0 +1,206 @@ +package storage + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/go-redis/redis" + "github.com/mxpv/podsync/pkg/api" + "github.com/pkg/errors" +) + +const expiration = 24 * time.Hour * 90 + +// Backward compatible Redis storage for feeds +type RedisStorage struct { + client *redis.Client +} + +func (r *RedisStorage) parsePageSize(m map[string]string) (int, error) { + str, ok := m["pagesize"] + if !ok { + return 50, nil + } + + size, err := strconv.ParseInt(str, 10, 32) + if err != nil { + return 0, err + } + + if size > 150 { + return 0, errors.New("invalid page size") + } + + return int(size), nil +} + +func (r *RedisStorage) parseFormat(m map[string]string) (api.Format, api.Quality, error) { + quality, ok := m["quality"] + if !ok { + return api.VideoFormat, api.HighQuality, nil + } + + if quality == "VideoHigh" { + return api.VideoFormat, api.HighQuality, nil + } else if quality == "VideoLow" { + return api.VideoFormat, api.LowQuality, nil + } else if quality == "AudioHigh" { + return api.AudioFormat, api.HighQuality, nil + } else if quality == "AudioLow" { + return api.AudioFormat, api.LowQuality, nil + } + + return "", "", fmt.Errorf("unsupported formmat %s", quality) +} + +func (r *RedisStorage) GetFeed(hashId string) (*api.Feed, error) { + result, err := r.client.HGetAll(hashId).Result() + if err != nil { + return nil, errors.Wrapf(err, "failed to query feed with id %s", hashId) + } + + if len(result) == 0 { + return nil, api.ErrNotFound + } + + // Expire after 3 month if no use + if err := r.client.Expire(hashId, expiration).Err(); err != nil { + return nil, errors.Wrap(err, "failed query update feed") + } + + feed := &api.Feed{ + PageSize: api.DefaultPageSize, + Quality: api.DefaultQuality, + Format: api.DefaultFormat, + HashId: hashId, + LastAccess: time.Now().UTC(), + } + + m := make(map[string]string, len(result)) + for key, val := range result { + m[strings.ToLower(key)] = val + } + + // Unpack provider and link type + provider := m["provider"] + linkType := m["type"] + if strings.EqualFold(provider, "youtube") { + feed.Provider = api.Youtube + + if strings.EqualFold(linkType, "channel") { + feed.LinkType = api.Channel + } else if strings.EqualFold(linkType, "playlist") { + feed.LinkType = api.Playlist + } else if strings.EqualFold(linkType, "user") { + feed.LinkType = api.User + } else { + return nil, fmt.Errorf("unsupported yt link type %s", linkType) + } + + } else if strings.EqualFold(provider, "vimeo") { + feed.Provider = api.Vimeo + + if strings.EqualFold(linkType, "channel") { + feed.LinkType = api.Channel + } else if strings.EqualFold(linkType, "user") { + feed.LinkType = api.User + } else if strings.EqualFold(linkType, "group") { + feed.LinkType = api.Group + } else { + return nil, fmt.Errorf("unsupported vimeo link type %s", linkType) + } + + } else { + return nil, errors.New("unsupported provider") + } + + // Unpack item id + id, ok := m["id"] + if !ok || id == "" { + return nil, errors.New("failed to unpack item id") + } + + feed.ItemId = id + + // Fetch user id + patreonId, ok := m["patreonid"] + if ok { + feed.UserId = patreonId + } + + // Unpack page size + pageSize, err := r.parsePageSize(m) + if err != nil { + return nil, err + } + + if patreonId == "" && pageSize > 50 { + return nil, errors.New("wrong feed data") + } + + // Parse feed's format and quality + format, quality, err := r.parseFormat(m) + if err != nil { + return nil, err + } + + feed.PageSize = pageSize + feed.Format = format + feed.Quality = quality + + return feed, nil +} + +func (r *RedisStorage) CreateFeed(feed *api.Feed) error { + fields := map[string]interface{}{ + "provider": string(feed.Provider), + "type": string(feed.LinkType), + "id": feed.ItemId, + "patreonid": feed.UserId, + "pagesize": feed.PageSize, + } + + if feed.Format == api.VideoFormat { + + if feed.Quality == api.HighQuality { + fields["quality"] = "VideoHigh" + } else { + fields["quality"] = "VideoLow" + } + + } else { + + if feed.Quality == api.HighQuality { + fields["quality"] = "AudioHigh" + } else { + fields["quality"] = "AudioLow" + } + + } + + if err := r.client.HMSet(feed.HashId, fields).Err(); err != nil { + return errors.Wrap(err, "failed to save feed") + } + + return r.client.Expire(feed.HashId, expiration).Err() +} + +func (r *RedisStorage) keys() ([]string, error) { + return r.client.Keys("*").Result() +} + +func NewRedisStorage(redisUrl string) (*RedisStorage, error) { + opts, err := redis.ParseURL(redisUrl) + if err != nil { + return nil, err + } + + client := redis.NewClient(opts) + if err := client.Ping().Err(); err != nil { + return nil, err + } + + return &RedisStorage{client}, nil +} diff --git a/pkg/storage/redis_test.go b/pkg/storage/redis_test.go new file mode 100644 index 0000000..5075258 --- /dev/null +++ b/pkg/storage/redis_test.go @@ -0,0 +1,71 @@ +package storage + +import ( + "strconv" + "testing" + "time" + + "github.com/mxpv/podsync/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestRedisStorage_GetFeed(t *testing.T) { + t.Skip("run redis tests manually") + + client := createRedisClient(t) + + keys, err := client.keys() + require.NoError(t, err) + + require.True(t, len(keys) > 0) + + for idx, key := range keys { + if key == "keygen" { + continue + } + + feed, err := client.GetFeed(key) + require.NoError(t, err, "feed %s (id = %d) failed", key, idx) + require.NotNil(t, feed) + } +} + +func TestRedisStorage_CreateFeed(t *testing.T) { + t.Skip("run redis tests manually") + + client := createRedisClient(t) + + hashId := strconv.FormatInt(time.Now().UTC().UnixNano(), 10) + + err := client.CreateFeed(&api.Feed{ + HashId: hashId, + UserId: "321", + Provider: api.Youtube, + LinkType: api.Channel, + ItemId: "123", + PageSize: 45, + Quality: api.LowQuality, + Format: api.AudioFormat, + }) + + require.NoError(t, err) + + feed, err := client.GetFeed(hashId) + require.NoError(t, err) + + require.Equal(t, hashId, feed.HashId) + require.Equal(t, "321", feed.UserId) + require.Equal(t, api.Youtube, feed.Provider) + require.Equal(t, api.Channel, feed.LinkType) + require.Equal(t, "123", feed.ItemId) + require.Equal(t, 45, feed.PageSize) + require.Equal(t, api.LowQuality, feed.Quality) + require.Equal(t, api.AudioFormat, feed.Format) +} + +func createRedisClient(t *testing.T) *RedisStorage { + client, err := NewRedisStorage("redis://localhost") + require.NoError(t, err) + + return client +} diff --git a/src/Podsync/.bowerrc b/src/Podsync/.bowerrc deleted file mode 100644 index 6406626..0000000 --- a/src/Podsync/.bowerrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "directory": "wwwroot/lib" -} diff --git a/src/Podsync/Controllers/FeedController.cs b/src/Podsync/Controllers/FeedController.cs deleted file mode 100644 index f12e3c9..0000000 --- a/src/Podsync/Controllers/FeedController.cs +++ /dev/null @@ -1,104 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using System.Linq; -using System.Security.Claims; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; -using Podsync.Helpers; -using Podsync.Services; -using Podsync.Services.Links; -using Podsync.Services.Rss; -using Podsync.Services.Rss.Contracts; -using Podsync.Services.Storage; - -namespace Podsync.Controllers -{ - [Route("feed")] - [ServiceFilter(typeof(HandleExceptionAttribute), IsReusable = true)] - public class FeedController : Controller - { - private static readonly IDictionary Extensions = new Dictionary - { - ["video/mp4"] = "mp4", - ["audio/mp4"] = "m4a" - }; - - private readonly ILinkService _linkService; - private readonly IFeedService _feedService; - - public FeedController(IRssBuilder rssBuilder, ILinkService linkService, IFeedService feedService) - { - _linkService = linkService; - _feedService = feedService; - } - - [HttpPost] - [Route("create")] - [ValidateModelState] - public async Task Create([FromBody] CreateFeedRequest request) - { - var linkInfo = _linkService.Parse(new Uri(request.Url)); - - var feed = new FeedMetadata - { - Provider = linkInfo.Provider, - LinkType = linkInfo.LinkType, - Id = linkInfo.Id, - Quality = request.Quality ?? Constants.DefaultFormat, - PageSize = request.PageSize ?? Constants.DefaultPageSize - }; - - // Check if user eligible for Patreon features - var allowFeatures = User.EnablePatreonFeatures(); - if (allowFeatures) - { - feed.PatreonId = User.GetClaim(ClaimTypes.NameIdentifier); - } - else - { - feed.Quality = Constants.DefaultFormat; - feed.PageSize = Constants.DefaultPageSize; - } - - var feedId = await _feedService.Create(feed); - var url = _linkService.Feed(Request.GetBaseUrl(), feedId); - - return url; - } - - [HttpGet] - [Route("~/{feedId:length(4, 6)}")] - [ValidateModelState] - public async Task Feed([Required] string feedId) - { - try - { - var body = await _feedService.Get(feedId, FixLinks); - return Content(body, "application/rss+xml; charset=UTF-8"); - } - catch (KeyNotFoundException) - { - return NotFound($"ERROR: No feed with id {feedId}"); - } - } - - [NonAction] - private void FixLinks(string feedId, Feed feed) - { - var selfHost = Request.GetBaseUrl(); - - // Set atom link to this feed - // See https://validator.w3.org/feed/docs/warning/MissingAtomSelfLink.html - var selfLink = new Uri(selfHost, Request.Path); - feed.Channels.ForEach(x => x.AtomLink = selfLink); - - // No magic here, just make download links to DownloadController.Download - feed.Channels.SelectMany(x => x.Items).ForEach(item => - { - var ext = Extensions[item.ContentType]; - item.DownloadLink = new Uri(selfHost, $"download/{feedId}/{item.Id}.{ext}"); - }); - } - } -} \ No newline at end of file diff --git a/src/Podsync/Controllers/HomeController.cs b/src/Podsync/Controllers/HomeController.cs deleted file mode 100644 index 8281577..0000000 --- a/src/Podsync/Controllers/HomeController.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Microsoft.AspNetCore.Mvc; - -namespace Podsync.Controllers -{ - public class HomeController : Controller - { - public IActionResult Index() - { - return View(); - } - } -} diff --git a/src/Podsync/Controllers/StatusController.cs b/src/Podsync/Controllers/StatusController.cs deleted file mode 100644 index 31c0d6b..0000000 --- a/src/Podsync/Controllers/StatusController.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; -using Podsync.Services.Resolver; -using Podsync.Services.Storage; - -namespace Podsync.Controllers -{ - public class StatusController : Controller - { - private const string ErrorStatus = "ERROR"; - - private readonly IStorageService _storageService; - private readonly IResolverService _resolverService; - - public StatusController(IStorageService storageService, IResolverService resolverService) - { - _storageService = storageService; - _resolverService = resolverService; - } - - [HttpGet] - [Route("~/status")] - public async Task Index() - { - var storageStatus = ErrorStatus; - - try - { - var time = await _storageService.Ping(); - storageStatus = time.ToString(); - } - catch (Exception) - { - // Nothing to do - } - - var resolverStatus = _resolverService.Version ?? ErrorStatus; - - return $"Path: {Request.Path}\r\n" + - $"Redis: {storageStatus}\r\n" + - $"Resolve: {resolverStatus}"; - } - } -} \ No newline at end of file diff --git a/src/Podsync/Dockerfile b/src/Podsync/Dockerfile deleted file mode 100644 index 881213c..0000000 --- a/src/Podsync/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM microsoft/aspnetcore-build AS build -WORKDIR /workspace -COPY . . -RUN dotnet restore -RUN dotnet publish --configuration release --output ./bin/ - -FROM microsoft/aspnetcore:1.1 -ENV ASPNETCORE_URLS http://*:8080 -ENV ASPNETCORE_ENVIRONMENT Production -WORKDIR /app -COPY --from=build /workspace/bin/ . -EXPOSE 8080 -ENTRYPOINT ["dotnet", "Podsync.dll"] \ No newline at end of file diff --git a/src/Podsync/Helpers/EnumerableExtensions.cs b/src/Podsync/Helpers/EnumerableExtensions.cs deleted file mode 100644 index ac1233b..0000000 --- a/src/Podsync/Helpers/EnumerableExtensions.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Podsync.Helpers -{ - internal static class EnumerableExtensions - { - public static IEnumerable> Chunk(this IEnumerable source, int chunkSize) - { - if (chunkSize < 1) - { - throw new ArgumentException("Chunk size must be positive", nameof(chunkSize)); - } - - while (source.Any()) - { - yield return source.Take(chunkSize); - source = source.Skip(chunkSize); - } - } - - public static bool SafeAny(this IEnumerable source) - { - if (source == null) - { - return false; - } - - if (source.Any()) - { - return true; - } - - return false; - } - - public static void ForEach(this IEnumerable source, Action action) - { - if (source == null) - { - return; - } - - foreach (var item in source) - { - action(item); - } - } - - public static void AddTo(this IEnumerable collection, List target) - { - target.AddRange(collection); - } - - public static void SafeForEach(this IEnumerable source, Action action) - { - if (source == null) - { - return; - } - - foreach (var item in source) - { - try - { - action(item); - } - catch - { - // Nothing to do - } - } - } - } -} \ No newline at end of file diff --git a/src/Podsync/Helpers/Extensions.cs b/src/Podsync/Helpers/Extensions.cs deleted file mode 100644 index 46a17f6..0000000 --- a/src/Podsync/Helpers/Extensions.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System; -using System.Linq; -using System.Security.Claims; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Podsync.Services; -using Podsync.Services.Resolver; - -namespace Podsync.Helpers -{ - public static class Extensions - { - /// - /// Generates a fully qualified URL to the specified content by using the specified content path. - /// Converts a virtual (relative) path to an application absolute path. - /// See http://stackoverflow.com/questions/30755827/getting-absolute-urls-using-asp-net-core-mvc-6 - /// - /// - /// - /// - public static string AbsoluteContent(this IUrlHelper url, string contentPath) - { - var request = url.ActionContext.HttpContext.Request; - var baseUri = new Uri($"{request.Scheme}://{request.Host.Value}"); - var fullUri = new Uri(baseUri, url.Content(contentPath)); - - return fullUri.ToString(); - } - - public static Uri GetBaseUrl(this HttpRequest request) - { - return new Uri($"{request.Scheme}://{request.Host}"); - } - - private const string OwnerId = "2822191"; - - /// - /// Check if user eligible for advanced features - /// - /// - /// - public static bool EnablePatreonFeatures(this ClaimsPrincipal user) - { - if (!user.Identity.IsAuthenticated) - { - return false; - } - - if (user.GetClaim(ClaimTypes.NameIdentifier) == OwnerId) - { - return true; - } - - const int MinAmountCents = 100; - - int amount; - if (int.TryParse(user.GetClaim(Constants.Patreon.AmountDonated), out amount)) - { - return amount >= MinAmountCents; - } - - return false; - } - - public static string GetName(this ClaimsPrincipal claimsPrincipal) - { - return claimsPrincipal.GetClaim(ClaimTypes.Name) - ?? claimsPrincipal.GetClaim(ClaimTypes.Email) - ?? "noname :("; - } - - public static string GetClaim(this ClaimsPrincipal claimsPrincipal, string type) - { - return claimsPrincipal.Claims.FirstOrDefault(x => x.Type == type)?.Value; - } - - public static bool IsAudio(this ResolveFormat format) - { - return format == ResolveFormat.AudioHigh || format == ResolveFormat.AudioLow; - } - } -} \ No newline at end of file diff --git a/src/Podsync/Helpers/HandleExceptionAttribute.cs b/src/Podsync/Helpers/HandleExceptionAttribute.cs deleted file mode 100644 index 81b04fe..0000000 --- a/src/Podsync/Helpers/HandleExceptionAttribute.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using System.Net; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.Extensions.Logging; -using Podsync.Services; - -namespace Podsync.Helpers -{ - public class HandleExceptionAttribute : ExceptionFilterAttribute - { - private readonly ILogger _logger; - - public HandleExceptionAttribute(ILogger logger) - { - _logger = logger; - } - - public override void OnException(ExceptionContext context) - { - var exception = context.Exception; - if (exception is ArgumentNullException || exception is ArgumentException) - { - context.Result = new BadRequestObjectResult(exception.Message); - } - else - { - context.Result = new StatusCodeResult((int)HttpStatusCode.InternalServerError); - - _logger.LogCritical(Constants.Events.UnhandledError, context.Exception, "Unhandled exception"); - } - } - } -} \ No newline at end of file diff --git a/src/Podsync/Helpers/ServiceProviderExtensions.cs b/src/Podsync/Helpers/ServiceProviderExtensions.cs deleted file mode 100644 index 82eb002..0000000 --- a/src/Podsync/Helpers/ServiceProviderExtensions.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using Microsoft.Extensions.DependencyInjection; - -namespace Podsync.Helpers -{ - internal static class ServiceProviderExtensions - { - public static T CreateInstance(this IServiceProvider serviceProvider) - { - return serviceProvider.CreateInstance(typeof(T)); - } - - public static T CreateInstance(this IServiceProvider serviceProvider, Type implementation) - { - return (T)serviceProvider.CreateInstance(implementation); - } - - public static object CreateInstance(this IServiceProvider serviceProvider, Type type) - { - return ActivatorUtilities.CreateInstance(serviceProvider, type); - } - - public static IEnumerable FindAllImplementationsOf(this IServiceProvider serviceProvider, Type interfaceType, Assembly assembly) - { - if (!interfaceType.GetTypeInfo().IsInterface) - { - throw new ArgumentException("T should be an interface"); - } - - return GetLoadableTypes(assembly).Where(type => IsAssignableFrom(interfaceType, type)); - } - - public static IEnumerable FindAllImplementationsOf(this IServiceProvider serviceProvider, Assembly assembly) - { - return serviceProvider.FindAllImplementationsOf(typeof(T), assembly); - } - - private static bool IsAssignableFrom(Type interfaceType, Type serviceType) - { - var serviceTypeInfo = serviceType.GetTypeInfo(); - if (serviceTypeInfo.IsInterface || serviceTypeInfo.IsAbstract) - { - return false; - } - - var interfaceTypeInfo = interfaceType.GetTypeInfo(); - if (!interfaceTypeInfo.IsGenericType) - { - return interfaceType.IsAssignableFrom(serviceType); - } - - return serviceType - .GetInterfaces() - .Where(type => type.GetTypeInfo().IsGenericType) - .Any(type => type.GetGenericTypeDefinition() == interfaceType); - } - - private static IEnumerable GetLoadableTypes(Assembly assembly) - { - try - { - return assembly.GetTypes(); - } - catch (ReflectionTypeLoadException e) - { - return e.Types.Where(x => x != null); - } - } - } -} \ No newline at end of file diff --git a/src/Podsync/Helpers/ValidateModelStateAttribute.cs b/src/Podsync/Helpers/ValidateModelStateAttribute.cs deleted file mode 100644 index f755e40..0000000 --- a/src/Podsync/Helpers/ValidateModelStateAttribute.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Linq; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Filters; - -namespace Podsync.Helpers -{ - /// - /// Validate the model state prior to action execution. - /// See http://www.strathweb.com/2015/06/action-filters-service-filters-type-filters-asp-net-5-mvc-6/ - /// - public class ValidateModelStateAttribute : ActionFilterAttribute - { - private readonly bool _validateNotNull; - - public ValidateModelStateAttribute(bool validateNotNull = true) - { - _validateNotNull = validateNotNull; - } - - public override void OnActionExecuting(ActionExecutingContext context) - { - if (!context.ModelState.IsValid || _validateNotNull && context.ActionArguments.Any(p => p.Value == null)) - { - context.Result = new BadRequestObjectResult(context.ModelState); - } - } - } -} \ No newline at end of file diff --git a/src/Podsync/Podsync.csproj b/src/Podsync/Podsync.csproj deleted file mode 100644 index 1ea8dde..0000000 --- a/src/Podsync/Podsync.csproj +++ /dev/null @@ -1,90 +0,0 @@ - - - netcoreapp1.1 - true - Podsync - Exe - Podsync - aspnet-Podsync-20161004104901 - - - - - PreserveNewest - - - - - 1.25.0.760 - - - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.3 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.2 - - - 1.1.1 - - - - 1.2.3 - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/Podsync/Program.cs b/src/Podsync/Program.cs deleted file mode 100644 index d9b3cb5..0000000 --- a/src/Podsync/Program.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.IO; -using Microsoft.AspNetCore.Hosting; - -namespace Podsync -{ - public class Program - { - public static void Main(string[] args) - { - var host = new WebHostBuilder() - .UseKestrel() - .UseContentRoot(Directory.GetCurrentDirectory()) - .UseIISIntegration() - .UseStartup() - .Build(); - - host.Run(); - } - } -} diff --git a/src/Podsync/Properties/launchSettings.json b/src/Podsync/Properties/launchSettings.json deleted file mode 100644 index 73787a2..0000000 --- a/src/Podsync/Properties/launchSettings.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "iisSettings": { - "windowsAuthentication": false, - "anonymousAuthentication": true, - "iisExpress": { - "applicationUrl": "http://localhost:56247/", - "sslPort": 0 - } - }, - "profiles": { - "IIS Express": { - "commandName": "IISExpress", - "launchBrowser": true, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Podsync": { - "commandName": "Project", - "launchBrowser": true, - "launchUrl": "http://localhost:5000", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Constants.cs b/src/Podsync/Services/Constants.cs deleted file mode 100644 index 6467b09..0000000 --- a/src/Podsync/Services/Constants.cs +++ /dev/null @@ -1,38 +0,0 @@ -using Podsync.Services.Resolver; - -namespace Podsync.Services -{ - public static class Constants - { - public const int DefaultPageSize = 50; - - public const ResolveFormat DefaultFormat = ResolveFormat.VideoHigh; - - public static class Patreon - { - public const string AuthenticationScheme = "Patreon"; - - public const string AuthorizationEndpoint = "https://www.patreon.com/oauth2/authorize"; - - public const string TokenEndpoint = "https://api.patreon.com/oauth2/token"; - - public const string AmountDonated = "Patreon/" + nameof(AmountDonated); - } - - public static class Events - { - public const int RssError = 1; - - public const int YtdlError = 2; - - public const int UnhandledError = 3; - } - - public static class Cache - { - public const string VideosPrefix = "video_urls"; - - public const string FeedsPrefix = "feeds"; - } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/CreateFeedRequest.cs b/src/Podsync/Services/CreateFeedRequest.cs deleted file mode 100644 index 058b2dc..0000000 --- a/src/Podsync/Services/CreateFeedRequest.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.ComponentModel.DataAnnotations; -using Podsync.Services.Resolver; - -namespace Podsync.Services -{ - public class CreateFeedRequest - { - [Required] - [Url] - public string Url { get; set; } - - public ResolveFormat? Quality { get; set; } - - [Range(50, 150)] - public int? PageSize { get; set; } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Links/ILinkService.cs b/src/Podsync/Services/Links/ILinkService.cs deleted file mode 100644 index 06f9e7a..0000000 --- a/src/Podsync/Services/Links/ILinkService.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; - -namespace Podsync.Services.Links -{ - public interface ILinkService - { - LinkInfo Parse(Uri link); - - Uri Make(LinkInfo info); - - Uri Feed(Uri baseUrl, string feedId); - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Links/LinkInfo.cs b/src/Podsync/Services/Links/LinkInfo.cs deleted file mode 100644 index 8f5e5d3..0000000 --- a/src/Podsync/Services/Links/LinkInfo.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Podsync.Services.Links -{ - public struct LinkInfo - { - public string Id { get; set; } - - public LinkType LinkType { get; set; } - - public Provider Provider { get; set; } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Links/LinkService.cs b/src/Podsync/Services/Links/LinkService.cs deleted file mode 100644 index 45f7493..0000000 --- a/src/Podsync/Services/Links/LinkService.cs +++ /dev/null @@ -1,169 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text.RegularExpressions; -using Microsoft.AspNetCore.WebUtilities; -using Microsoft.Extensions.Primitives; - -namespace Podsync.Services.Links -{ - public class LinkService : ILinkService - { - private static readonly IDictionary> LinkFormats = new Dictionary> - { - [Provider.YouTube] = new Dictionary - { - [LinkType.Video] = "https://youtube.com/watch?v={0}", - [LinkType.Channel] = "https://youtube.com/channel/{0}", - [LinkType.Playlist] = "https://youtube.com/playlist?list={0}" - }, - - [Provider.Vimeo] = new Dictionary - { - [LinkType.Channel] = "https://vimeo.com/channels/{0}", - [LinkType.Group] = "https://vimeo.com/groups/{0}", - [LinkType.User] = "https://vimeo.com/user{0}", - [LinkType.Video] = "https://vimeo.com/{0}" - } - }; - - /* - YouTube users, channels and playlists - Test input: - https://www.youtube.com/playlist?list=PLCB9F975ECF01953C - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og - https://www.youtube.com/user/fxigr1 - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos - */ - - private static readonly Regex YouTubeRegex = new Regex(@"^(?:https?://)?(?:www\.|m\.)?(?:youtube.com/)(?user|channel|playlist|watch)/?(?[-\w]+)?", RegexOptions.Compiled); - - /* - Vimeo groups, channels and users - Test input: - https://vimeo.com/groups/109 - http://vimeo.com/groups/109 - http://www.vimeo.com/groups/109 - https://vimeo.com/groups/109/videos/ - https://vimeo.com/channels/staffpicks - https://vimeo.com/channels/staffpicks/146224925 - https://vimeo.com/awhitelabelproduct - */ - private static readonly Regex VimeoRegex = new Regex(@"^(?:https?://)?(?:www\.)?(?:vimeo.com/)(?groups|channels)?/?(?\w+)", RegexOptions.Compiled); - - public LinkInfo Parse(Uri link) - { - if (link == null) - { - throw new ArgumentNullException(nameof(link), "Link can't be null"); - } - - var provider = Provider.Unknown; - var linkType = LinkType.Unknown; - - var id = string.Empty; - - // YouTube - var match = YouTubeRegex.Match(link.AbsoluteUri); - if (match.Success) - { - provider = Provider.YouTube; - - var type = match.Groups["type"]?.ToString(); - if (type == "user") - { - // https://www.youtube.com/user/fxigr1 - - id = match.Groups["id"]?.ToString(); - linkType = LinkType.User; - } - else if (type == "channel") - { - // https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og - - id = match.Groups["id"]?.ToString(); - linkType = LinkType.Channel; - } - else if (type == "playlist" || type == "watch") - { - // https://www.youtube.com/playlist?list=PLCB9F975ECF01953C - // https://www.youtube.com/watch?v=otm9NaT9OWU&list=PLCB9F975ECF01953C - - var qs = QueryHelpers.ParseQuery(link.Query); - - StringValues list; - if (qs.TryGetValue("list", out list)) - { - id = list; - } - - linkType = LinkType.Playlist; - } - } - else - { - // Vimeo - match = VimeoRegex.Match(link.AbsoluteUri); - if (match.Success) - { - provider = Provider.Vimeo; - id = match.Groups["id"]?.ToString(); - - var type = match.Groups["type"]?.ToString(); - if (type == "groups") - { - // https://vimeo.com/groups/109 - - linkType = LinkType.Group; - } - else if (type == "channels") - { - // https://vimeo.com/channels/staffpicks - - linkType = LinkType.Channel; - } - else - { - // https://vimeo.com/awhitelabelproduct - - linkType = LinkType.User; - } - } - } - - if (string.IsNullOrWhiteSpace(id) || linkType == LinkType.Unknown || provider == Provider.Unknown) - { - throw new ArgumentException("Not supported provider or link format"); - } - - return new LinkInfo - { - Id = id, - LinkType = linkType, - Provider = provider - }; - } - - public Uri Make(LinkInfo info) - { - if (info.Id == null) - { - throw new ArgumentNullException(nameof(info.Id), "Id can't be empty"); - } - - try - { - var format = LinkFormats[info.Provider][info.LinkType]; - return new Uri(string.Format(format, info.Id)); - } - catch (KeyNotFoundException ex) - { - throw new ArgumentException("Unsupported provider or link type", nameof(info), ex); - } - } - - public Uri Feed(Uri baseUrl, string feedId) - { - return new Uri(baseUrl, feedId); - } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Links/LinkType.cs b/src/Podsync/Services/Links/LinkType.cs deleted file mode 100644 index c6e14a4..0000000 --- a/src/Podsync/Services/Links/LinkType.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Podsync.Services.Links -{ - public enum LinkType - { - Unknown = 0, - Video, - Playlist, - User, - Channel, - Group - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Links/Provider.cs b/src/Podsync/Services/Links/Provider.cs deleted file mode 100644 index 4d4f3b6..0000000 --- a/src/Podsync/Services/Links/Provider.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Podsync.Services.Links -{ - public enum Provider - { - Unknown = 0, - YouTube, - Vimeo, - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Patreon/Contracts/Pledge.cs b/src/Podsync/Services/Patreon/Contracts/Pledge.cs deleted file mode 100644 index 79433d8..0000000 --- a/src/Podsync/Services/Patreon/Contracts/Pledge.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; - -namespace Podsync.Services.Patreon.Contracts -{ - public class Pledge - { - public string Id { get; set; } - - public DateTime CreatedAt { get; set; } - - public DateTime DeclinedSince { get; set; } - - public int AmountCents { get; set; } - - public int PledgeCapCents { get; set; } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Patreon/Contracts/User.cs b/src/Podsync/Services/Patreon/Contracts/User.cs deleted file mode 100644 index 0ffdc9c..0000000 --- a/src/Podsync/Services/Patreon/Contracts/User.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace Podsync.Services.Patreon.Contracts -{ - public class User - { - public User() - { - Pledges = Enumerable.Empty(); - } - - public string Id { get; set; } - - public string Email { get; set; } - - public string Name { get; set; } - - public string Url { get; set; } - - public IEnumerable Pledges { get; set; } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Patreon/IPatreonApi.cs b/src/Podsync/Services/Patreon/IPatreonApi.cs deleted file mode 100644 index bbdd221..0000000 --- a/src/Podsync/Services/Patreon/IPatreonApi.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using System.Threading.Tasks; - -namespace Podsync.Services.Patreon -{ - public interface IPatreonApi : IDisposable - { - Task FetchProfile(Tokens tokens); - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Patreon/PatreonApi.cs b/src/Podsync/Services/Patreon/PatreonApi.cs deleted file mode 100644 index a297d42..0000000 --- a/src/Podsync/Services/Patreon/PatreonApi.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; - -namespace Podsync.Services.Patreon -{ - public sealed class PatreonApi : IPatreonApi - { - private readonly HttpClient _client; - - public PatreonApi() - { - _client = new HttpClient - { - BaseAddress = new Uri("https://api.patreon.com/oauth2/api/") - }; - - _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer"); - } - - public Task FetchProfile(Tokens tokens) - { - return Query("current_user", tokens); - } - - public void Dispose() - { - _client.Dispose(); - } - - private async Task Query(string path, Tokens tokens) - { - var request = new HttpRequestMessage(HttpMethod.Get, path); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", tokens.AccessToken); - - var response = await _client.SendAsync(request); - response.EnsureSuccessStatusCode(); - - var json = await response.Content.ReadAsStringAsync(); - - return JObject.Parse(json); - } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Patreon/PatreonExtensions.cs b/src/Podsync/Services/Patreon/PatreonExtensions.cs deleted file mode 100644 index 12011e3..0000000 --- a/src/Podsync/Services/Patreon/PatreonExtensions.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Podsync.Services.Patreon.Contracts; - -namespace Podsync.Services.Patreon -{ - public static class PatreonExtensions - { - public static async Task FetchUserAndPledges(this IPatreonApi api, Tokens tokens) - { - var resp = await api.FetchProfile(tokens); - - dynamic userAttrs = resp.data.attributes; - - var user = new User - { - Id = resp.data.id, - Email = userAttrs.email, - Name = userAttrs.first_name ?? userAttrs.full_name, - Url = userAttrs.url, - Pledges = ParsePledges(resp) - }; - - return user; - } - - private static IEnumerable ParsePledges(dynamic resp) - { - dynamic pledges = resp.data.relationships.pledges.data; - - foreach (var pledge in pledges) - { - var id = pledge.id; - var type = pledge.type; - - foreach (var include in resp.included) - { - if (include.id == id && include.type == type) - { - dynamic attrs = include.attributes; - - yield return new Pledge - { - Id = include.id, - CreatedAt = ParseDate(attrs.created_at), - DeclinedSince = ParseDate(attrs.declined_since), - AmountCents = attrs.amount_cents, - PledgeCapCents = attrs.pledge_cap_cents - }; - - break; - } - } - } - } - - private static DateTime ParseDate(object obj) - { - var date = obj?.ToString(); - - if (string.IsNullOrWhiteSpace(date)) - { - return DateTime.MinValue; - } - - var dateTime = DateTime.Parse(date); - - return DateTime.SpecifyKind(dateTime, DateTimeKind.Utc); - } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Patreon/Tokens.cs b/src/Podsync/Services/Patreon/Tokens.cs deleted file mode 100644 index 20c60c5..0000000 --- a/src/Podsync/Services/Patreon/Tokens.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Podsync.Services.Patreon -{ - public struct Tokens - { - public string AccessToken { get; set; } - - public string RefreshToken { get; set; } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/PodsyncConfiguration.cs b/src/Podsync/Services/PodsyncConfiguration.cs deleted file mode 100644 index cd81bdd..0000000 --- a/src/Podsync/Services/PodsyncConfiguration.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Podsync.Services.Patreon; - -namespace Podsync.Services -{ - public class PodsyncConfiguration - { - public string YouTubeApiKey { get; set; } - - public string VimeoApiKey { get; set; } - - public string RedisConnectionString { get; set; } - - public string PatreonClientId { get; set; } - - public string PatreonSecret { get; set; } - - public Tokens CreatorTokens { get; set; } - - public string RemoteResolverUrl { get; set; } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Resolver/CachedResolver.cs b/src/Podsync/Services/Resolver/CachedResolver.cs deleted file mode 100644 index bc38dc5..0000000 --- a/src/Podsync/Services/Resolver/CachedResolver.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Threading.Tasks; -using Podsync.Services.Storage; - -namespace Podsync.Services.Resolver -{ - public abstract class CachedResolver : IResolverService - { - private readonly TimeSpan UrlExpiration = TimeSpan.FromHours(3); - private readonly IStorageService _storageService; - - protected CachedResolver(IStorageService storageService) - { - _storageService = storageService; - } - - public abstract string Version { get; } - - public async Task Resolve(Uri videoUrl, ResolveFormat format) - { - var id = videoUrl.GetHashCode().ToString(); - - // Check if this video URL was resolved within last 3 hours - var value = await _storageService.GetCached(Constants.Cache.VideosPrefix, id); - if (!string.IsNullOrWhiteSpace(value)) - { - return new Uri(value); - } - - // Resolve and save to cache - var uri = await ResolveInternal(videoUrl, format); - await _storageService.Cache(Constants.Cache.VideosPrefix, id, uri.ToString(), UrlExpiration); - - return uri; - } - - protected abstract Task ResolveInternal(Uri videoUrl, ResolveFormat format); - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Resolver/IResolverService.cs b/src/Podsync/Services/Resolver/IResolverService.cs deleted file mode 100644 index c45094c..0000000 --- a/src/Podsync/Services/Resolver/IResolverService.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Threading.Tasks; - -namespace Podsync.Services.Resolver -{ - public interface IResolverService - { - string Version { get; } - - Task Resolve(Uri videoUrl, ResolveFormat format = ResolveFormat.VideoHigh); - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Resolver/RemoteResolver.cs b/src/Podsync/Services/Resolver/RemoteResolver.cs deleted file mode 100644 index 56280a2..0000000 --- a/src/Podsync/Services/Resolver/RemoteResolver.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Podsync.Services.Storage; - -namespace Podsync.Services.Resolver -{ - public class RemoteResolver : CachedResolver - { - private readonly ILogger _logger; - private readonly HttpClient _client = new HttpClient(); - - public RemoteResolver(IStorageService storageService, IOptions options, ILogger logger) : base(storageService) - { - _logger = logger; - _client.BaseAddress = new Uri(options.Value.RemoteResolverUrl); - - _logger.LogInformation($"Remote resolver URL: {_client.BaseAddress}"); - } - - public override string Version { get; } = "Remote"; - - protected override async Task ResolveInternal(Uri videoUrl, ResolveFormat format) - { - using(var response = await _client.GetAsync($"/resolve?url={videoUrl}&quality={format}")) - { - using(response.Content) - { - var body = await response.Content.ReadAsStringAsync(); - if (!response.IsSuccessStatusCode) - { - throw new HttpRequestException(body); - } - - return new Uri(body); - } - } - } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Resolver/ResolveFormat.cs b/src/Podsync/Services/Resolver/ResolveFormat.cs deleted file mode 100644 index cfaf76c..0000000 --- a/src/Podsync/Services/Resolver/ResolveFormat.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Podsync.Services.Resolver -{ - public enum ResolveFormat - { - VideoHigh = 0, - VideoLow, - AudioHigh, - AudioLow - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Rss/Builders/CompositeRssBuilder.cs b/src/Podsync/Services/Rss/Builders/CompositeRssBuilder.cs deleted file mode 100644 index 49a473b..0000000 --- a/src/Podsync/Services/Rss/Builders/CompositeRssBuilder.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Podsync.Helpers; -using Podsync.Services.Links; -using Podsync.Services.Storage; - -namespace Podsync.Services.Rss.Builders -{ - // ReSharper disable once ClassNeverInstantiated.Global - public class CompositeRssBuilder : RssBuilderBase - { - private readonly IDictionary _builders; - private readonly ILogger _logger; - - public CompositeRssBuilder(IServiceProvider serviceProvider, IStorageService storageService, ILogger logger) : base(storageService) - { - _logger = logger; - - // Find all RSS builders (all implementations of IRssBuilder), create instances and make dictionary for fast search by Provider type - var buildTypes = serviceProvider.FindAllImplementationsOf(Assembly.GetEntryAssembly()).Where(x => x != typeof(CompositeRssBuilder)); - var builders = buildTypes.Select(builderType => (IRssBuilder)serviceProvider.CreateInstance(builderType)).ToDictionary(builder => builder.Provider); - - _logger.LogInformation($"Found {builders.Count} RSS builders"); - _builders = new ReadOnlyDictionary(builders); - } - - public override Provider Provider => throw new NotSupportedException(); - - public override Task Query(FeedMetadata feed) - { - try - { - IRssBuilder builder; - if (_builders.TryGetValue(feed.Provider, out builder)) - { - return builder.Query(feed); - } - - throw new NotSupportedException("Not supported provider"); - } - catch (Exception ex) - { - _logger.LogError(Constants.Events.RssError, ex, "Failed to query RSS feed (id: {ID})", feed.Id); - - throw; - } - } - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Rss/Builders/RssBuilderBase.cs b/src/Podsync/Services/Rss/Builders/RssBuilderBase.cs deleted file mode 100644 index b6289f7..0000000 --- a/src/Podsync/Services/Rss/Builders/RssBuilderBase.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Threading.Tasks; -using Podsync.Services.Links; -using Podsync.Services.Rss.Contracts; -using Podsync.Services.Storage; - -namespace Podsync.Services.Rss.Builders -{ - public abstract class RssBuilderBase : IRssBuilder - { - private readonly IStorageService _storageService; - - protected RssBuilderBase(IStorageService storageService) - { - _storageService = storageService; - } - - public abstract Provider Provider { get; } - - public async Task Query(string feedId) - { - var metadata = await _storageService.Load(feedId); - - return await Query(metadata); - } - - public abstract Task Query(FeedMetadata metadata); - } -} \ No newline at end of file diff --git a/src/Podsync/Services/Rss/Builders/VimeoRssBuilder.cs b/src/Podsync/Services/Rss/Builders/VimeoRssBuilder.cs deleted file mode 100644 index 6e33548..0000000 --- a/src/Podsync/Services/Rss/Builders/VimeoRssBuilder.cs +++ /dev/null @@ -1,114 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Podsync.Services.Links; -using Podsync.Services.Rss.Contracts; -using Podsync.Services.Storage; -using Podsync.Services.Videos.Vimeo; - -namespace Podsync.Services.Rss.Builders -{ - // ReSharper disable once ClassNeverInstantiated.Global - public class VimeoRssBuilder : RssBuilderBase - { - private readonly IVimeoClient _client; - - public VimeoRssBuilder(IStorageService storageService, IVimeoClient client) : base(storageService) - { - _client = client; - } - - public override Provider Provider { get; } = Provider.Vimeo; - - public override async Task Query(FeedMetadata metadata) - { - var linkType = metadata.LinkType; - - var id = metadata.Id; - - var pageSize = metadata.PageSize; - if (pageSize == 0) - { - pageSize = Constants.DefaultPageSize; - } - - Channel channel; - if (linkType == LinkType.Channel) - { - channel = CreateChannel(await _client.Channel(id)); - channel.Items = CreateItems(await _client.ChannelVideos(id, pageSize)); - } - else if (linkType == LinkType.Group) - { - channel = CreateChannel(await _client.Group(id)); - channel.Items = CreateItems(await _client.GroupVideos(id, pageSize)); - } - else if (linkType == LinkType.User) - { - channel = CreateChannel(await _client.User(id)); - channel.Items = CreateItems(await _client.UserVideos(id, pageSize)); - } - else - { - throw new NotSupportedException("URL type is not supported"); - } - - var rss = new Feed - { - Channels = new[] { channel } - }; - - return rss; - } - - private static Channel CreateChannel(Group group) - { - return new Channel - { - Title = group.Name, - Description = group.Description, - Link = group.Link, - PubDate = group.CreatedAt, - Image = group.Thumbnail, - Thumbnail = group.Thumbnail, - Guid = group.Link.ToString() - }; - } - - private static Channel CreateChannel(User user) - { - return new Channel - { - Title = user.Name, - Description = user.Bio, - Link = user.Link, - PubDate = user.CreatedAt, - Image = user.Thumbnail, - Thumbnail = user.Thumbnail, - Guid = user.Link.ToString() - }; - } - - private static Item CreateItem(Video video) - { - return new Item - { - Id = video.Id, - Title = video.Title, - Description = video.Description, - PubDate = video.CreatedAt, - Link = video.Link, - Duration = video.Duration, - FileSize = video.Size, - ContentType = "video/mp4", - Author = video.Author - }; - } - - private static Item[] CreateItems(IEnumerable