Implement Vimeo builder

web/pkg/builders/common.go

@@ -0,0 +1,35 @@
+package builders
+
+import (
+	"fmt"
+
+	itunes "github.com/mxpv/podcast"
+	"github.com/mxpv/podsync/web/pkg/database"
+)
+
+const (
+	podsyncGenerator = "Podsync generator"
+	defaultCategory  = "TV & Film"
+)
+
+type linkType int
+
+const (
+	_                        = iota
+	linkTypeChannel linkType = iota
+	linkTypePlaylist
+	linkTypeUser
+	linkTypeGroup
+)
+
+func makeEnclosure(feed *database.Feed, id string, lengthInBytes int64) (string, itunes.EnclosureType, int64) {
+	ext := "mp4"
+	contentType := itunes.MP4
+	if feed.Format == database.AudioFormat {
+		ext = "mp3"
+		contentType = itunes.MP3
+	}
+
+	url := fmt.Sprintf("http://podsync.net/download/%s/%s.%s", feed.HashId, id, ext)
+	return url, contentType, lengthInBytes
+}

web/pkg/builders/vimeo.go

@@ -0,0 +1,241 @@
+package builders
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	itunes "github.com/mxpv/podcast"
+	"github.com/mxpv/podsync/web/pkg/database"
+	"github.com/pkg/errors"
+	"github.com/silentsokolov/go-vimeo"
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
+)
+
+const (
+	vimeoDefaultPageSize = 50
+)
+
+type VimeoBuilder struct {
+	client *vimeo.Client
+}
+
+func (v *VimeoBuilder) parseUrl(link string) (kind linkType, id string, err error) {
+	parsed, err := url.Parse(link)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to parse url: %s", link)
+		return
+	}
+
+	if !strings.HasSuffix(parsed.Host, "vimeo.com") {
+		err = errors.New("invalid vimeo host")
+		return
+	}
+
+	parts := strings.Split(parsed.EscapedPath(), "/")
+
+	if len(parts) <= 1 {
+		err = errors.New("invalid vimeo link path")
+		return
+	}
+
+	if parts[1] == "groups" {
+		kind = linkTypeGroup
+	} else if parts[1] == "channels" {
+		kind = linkTypeChannel
+	} else {
+		kind = linkTypeUser
+	}
+
+	if kind == linkTypeGroup || kind == linkTypeChannel {
+		if len(parts) <= 2 {
+			err = errors.New("invalid channel link")
+			return
+		}
+
+		id = parts[2]
+		if id == "" {
+			err = errors.New("invalid id")
+		}
+
+		return
+	}
+
+	if kind == linkTypeUser {
+		id = parts[1]
+		if id == "" {
+			err = errors.New("invalid id")
+		}
+
+		return
+	}
+
+	err = errors.New("unsupported link format")
+	return
+}
+
+func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q database.Quality) string {
+	if p == nil || len(p.Sizes) < 1 {
+		return ""
+	}
+
+	if q == database.LowQuality {
+		return p.Sizes[0].Link
+	} else {
+		return p.Sizes[len(p.Sizes)-1].Link
+	}
+}
+
+func (v *VimeoBuilder) queryChannel(channelId string, feed *database.Feed) (*itunes.Podcast, error) {
+	ch, resp, err := v.client.Channels.Get(channelId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to query channel with channelId %s", channelId)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("invalid http response from video server")
+	}
+
+	podcast := itunes.New(ch.Name, ch.Link, ch.Description, &ch.CreatedTime, nil)
+	podcast.Generator = podsyncGenerator
+	podcast.AddSubTitle(ch.Name)
+	podcast.AddImage(v.selectImage(ch.Pictures, feed.Quality))
+	podcast.AddCategory(defaultCategory, nil)
+	podcast.IAuthor = ch.User.Name
+
+	return &podcast, nil
+}
+
+func (v *VimeoBuilder) queryGroup(groupId string, feed *database.Feed) (*itunes.Podcast, error) {
+	gr, resp, err := v.client.Groups.Get(groupId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to query group with id %s", groupId)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("invalid http response from video server")
+	}
+
+	podcast := itunes.New(gr.Name, gr.Link, gr.Description, &gr.CreatedTime, nil)
+	podcast.Generator = podsyncGenerator
+	podcast.AddSubTitle(gr.Name)
+	podcast.AddImage(v.selectImage(gr.Pictures, feed.Quality))
+	podcast.AddCategory(defaultCategory, nil)
+	podcast.IAuthor = gr.User.Name
+
+	return &podcast, nil
+}
+
+func (v *VimeoBuilder) queryUser(userId string, feed *database.Feed) (*itunes.Podcast, error) {
+	user, resp, err := v.client.Users.Get(userId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to query user with id %s", userId)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("invalid http response from video server")
+	}
+
+	podcast := itunes.New(user.Name, user.Link, user.Bio, &user.CreatedTime, nil)
+	podcast.Generator = podsyncGenerator
+	podcast.AddSubTitle(user.Name)
+	podcast.AddImage(v.selectImage(user.Pictures, feed.Quality))
+	podcast.AddCategory(defaultCategory, nil)
+	podcast.IAuthor = user.Name
+
+	return &podcast, nil
+}
+
+func (v *VimeoBuilder) getVideoSize(video *vimeo.Video) int64 {
+	// Very approximate video file size
+	return int64(float64(video.Duration*video.Width*video.Height) * 0.38848958333)
+}
+
+type queryVideosFunc func(id string, opt *vimeo.ListVideoOptions) ([]*vimeo.Video, *vimeo.Response, error)
+
+func (v *VimeoBuilder) queryVideos(queryVideos queryVideosFunc, id string, podcast *itunes.Podcast, feed *database.Feed) error {
+	opt := vimeo.ListVideoOptions{}
+	opt.Page = 1
+	opt.PerPage = vimeoDefaultPageSize
+
+	added := 0
+
+	for {
+		videos, response, err := queryVideos(id, &opt)
+		if err != nil {
+			return errors.Wrap(err, "failed to query videos")
+		}
+
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("invalid http response %d from vimeo: %s", response.StatusCode, response.Status)
+		}
+
+		for _, video := range videos {
+			item := itunes.Item{}
+
+			item.GUID = strconv.Itoa(video.GetID())
+			item.Link = video.Link
+			item.Title = video.Name
+			item.Description = video.Description
+			if item.Description == "" {
+				item.Description = " " // Videos can be without description, workaround for AddItem
+			}
+
+			item.AddDuration(int64(video.Duration))
+			item.AddPubDate(&video.CreatedTime)
+			item.AddImage(v.selectImage(video.Pictures, feed.Quality))
+
+			size := v.getVideoSize(video)
+			item.AddEnclosure(makeEnclosure(feed, item.GUID, size))
+
+			_, err = podcast.AddItem(item)
+			if err != nil {
+				return errors.Wrapf(err, "failed to add episode")
+			}
+
+			added++
+		}
+
+		if added >= feed.PageSize || response.NextPage == "" {
+			return nil
+		}
+
+		opt.Page++
+	}
+}
+
+func (v *VimeoBuilder) Build(feed *database.Feed) (podcast *itunes.Podcast, err error) {
+	kind, id, err := v.parseUrl(feed.URL)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse link: %s", feed.URL)
+	}
+
+	if kind == linkTypeChannel {
+		if podcast, err = v.queryChannel(id, feed); err == nil {
+			err = v.queryVideos(v.client.Channels.ListVideo, id, podcast, feed)
+		}
+	} else if kind == linkTypeGroup {
+		if podcast, err = v.queryGroup(id, feed); err == nil {
+			err = v.queryVideos(v.client.Groups.ListVideo, id, podcast, feed)
+		}
+	} else if kind == linkTypeUser {
+		if podcast, err = v.queryUser(id, feed); err == nil {
+			err = v.queryVideos(v.client.Users.ListVideo, id, podcast, feed)
+		}
+	} else {
+		err = errors.New("unsupported feed type")
+	}
+
+	return
+}
+
+func NewVimeoBuilder(ctx context.Context, token string) (*VimeoBuilder, error) {
+	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
+	tc := oauth2.NewClient(ctx, ts)
+
+	client := vimeo.NewClient(tc)
+	return &VimeoBuilder{client}, nil
+}

web/pkg/builders/vimeo_test.go

@@ -0,0 +1,141 @@
+package builders
+
+import (
+	"os"
+	"testing"
+
+	"context"
+
+	itunes "github.com/mxpv/podcast"
+	"github.com/mxpv/podsync/web/pkg/database"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	vimeoKey    = os.Getenv("VIMEO_TEST_API_KEY")
+	defaultFeed = &database.Feed{Quality: database.HighQuality}
+)
+
+func TestParseVimeoGroupLink(t *testing.T) {
+	builder := &VimeoBuilder{}
+
+	kind, id, err := builder.parseUrl("https://vimeo.com/groups/109")
+	require.NoError(t, err)
+	require.Equal(t, linkTypeGroup, kind)
+	require.Equal(t, "109", id)
+
+	kind, id, err = builder.parseUrl("http://vimeo.com/groups/109")
+	require.NoError(t, err)
+	require.Equal(t, linkTypeGroup, kind)
+	require.Equal(t, "109", id)
+
+	kind, id, err = builder.parseUrl("http://www.vimeo.com/groups/109")
+	require.NoError(t, err)
+	require.Equal(t, linkTypeGroup, kind)
+	require.Equal(t, "109", id)
+
+	kind, id, err = builder.parseUrl("https://vimeo.com/groups/109/videos/")
+	require.NoError(t, err)
+	require.Equal(t, linkTypeGroup, kind)
+	require.Equal(t, "109", id)
+}
+
+func TestParseVimeoChannelLink(t *testing.T) {
+	builder := &VimeoBuilder{}
+
+	kind, id, err := builder.parseUrl("https://vimeo.com/channels/staffpicks")
+	require.NoError(t, err)
+	require.Equal(t, linkTypeChannel, kind)
+	require.Equal(t, "staffpicks", id)
+
+	kind, id, err = builder.parseUrl("http://vimeo.com/channels/staffpicks/146224925")
+	require.NoError(t, err)
+	require.Equal(t, linkTypeChannel, kind)
+	require.Equal(t, "staffpicks", id)
+}
+
+func TestParseVimeoUserLink(t *testing.T) {
+	builder := &VimeoBuilder{}
+
+	kind, id, err := builder.parseUrl("https://vimeo.com/awhitelabelproduct")
+	require.NoError(t, err)
+	require.Equal(t, linkTypeUser, kind)
+	require.Equal(t, "awhitelabelproduct", id)
+}
+
+func TestParseInvalidVimeoLink(t *testing.T) {
+	builder := &VimeoBuilder{}
+
+	_, _, err := builder.parseUrl("")
+	require.Error(t, err)
+
+	_, _, err = builder.parseUrl("http://www.apple.com")
+	require.Error(t, err)
+
+	_, _, err = builder.parseUrl("http://www.vimeo.com")
+	require.Error(t, err)
+}
+
+func TestQueryVimeoChannel(t *testing.T) {
+	builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
+	require.NoError(t, err)
+
+	podcast, err := builder.queryChannel("staffpicks", defaultFeed)
+	require.NoError(t, err)
+
+	require.Equal(t, "https://vimeo.com/channels/staffpicks", podcast.Link)
+	require.Equal(t, "Vimeo Staff Picks", podcast.Title)
+	require.Equal(t, "Vimeo Curation", podcast.IAuthor)
+	require.NotEmpty(t, podcast.Description)
+	require.NotEmpty(t, podcast.Image)
+	require.NotEmpty(t, podcast.IImage)
+}
+
+func TestQueryVimeoGroup(t *testing.T) {
+	builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
+	require.NoError(t, err)
+
+	podcast, err := builder.queryGroup("motion", defaultFeed)
+	require.NoError(t, err)
+
+	require.Equal(t, "https://vimeo.com/groups/motion", podcast.Link)
+	require.Equal(t, "Motion Graphic Artists", podcast.Title)
+	require.Equal(t, "Danny Garcia", podcast.IAuthor)
+	require.NotEmpty(t, podcast.Description)
+	require.NotEmpty(t, podcast.Image)
+	require.NotEmpty(t, podcast.IImage)
+}
+
+func TestQueryVimeoUser(t *testing.T) {
+	builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
+	require.NoError(t, err)
+
+	podcast, err := builder.queryUser("motionarray", defaultFeed)
+	require.NoError(t, err)
+
+	require.Equal(t, "https://vimeo.com/motionarray", podcast.Link)
+	require.Equal(t, "Motion Array", podcast.Title)
+	require.Equal(t, "Motion Array", podcast.IAuthor)
+	require.NotEmpty(t, podcast.Description)
+}
+
+func TestQueryVimeoVideos(t *testing.T) {
+	builder, err := NewVimeoBuilder(context.Background(), vimeoKey)
+	require.NoError(t, err)
+
+	feed := &itunes.Podcast{}
+
+	err = builder.queryVideos(builder.client.Channels.ListVideo, "staffpicks", feed, &database.Feed{})
+	require.NoError(t, err)
+
+	require.Equal(t, vimeoDefaultPageSize, len(feed.Items))
+
+	for _, item := range feed.Items {
+		require.NotEmpty(t, item.Title)
+		require.NotEmpty(t, item.Link)
+		require.NotEmpty(t, item.GUID)
+		require.NotNil(t, item.Enclosure)
+		require.NotEmpty(t, item.Enclosure.URL)
+		require.True(t, item.Enclosure.Length > 0)
+	}
+}

web/pkg/builders/youtube.go

@@ -15,20 +15,11 @@ import (
 )
 
 const (
-	linkTypeChannel  = linkType(1)
+	maxYoutubeResults = 50
-	linkTypePlaylist = linkType(2)
+	hdBytesPerSecond  = 350000
-	linkTypeUser     = linkType(3)
+	ldBytesPerSecond  = 100000
 )
 
-const (
-	maxResults       = 50
-	podsyncGenerator = "Podsync YouTube generator"
-	defaultCategory  = "TV & Film"
-	hdBytesPerSecond = 350000
-	ldBytesPerSecond = 100000
-)
-
-type linkType int
 type apiKey string
 
 func (key apiKey) Get() (string, string) {
@@ -48,7 +39,7 @@ func (yt *YouTubeBuilder) parseUrl(link string) (kind linkType, id string, err e
 	}
 
 	if !strings.HasSuffix(parsed.Host, "youtube.com") {
-		err = errors.New("invalid youtube link")
+		err = errors.New("invalid youtube host")
 		return
 	}
 
@@ -75,7 +66,7 @@ func (yt *YouTubeBuilder) parseUrl(link string) (kind linkType, id string, err e
 		kind = linkTypeChannel
 		parts := strings.Split(parsed.EscapedPath(), "/")
 		if len(parts) <= 2 {
-			err = errors.New("invalid channel link")
+			err = errors.New("invalid youtube channel link")
 			return
 		}
 
@@ -157,7 +148,7 @@ func (yt *YouTubeBuilder) listPlaylists(id, channelId string) (*youtube.Playlist
 }
 
 func (yt *YouTubeBuilder) listPlaylistItems(itemId string, pageToken string) ([]*youtube.PlaylistItem, string, error) {
-	req := yt.client.PlaylistItems.List("id,snippet").MaxResults(maxResults).PlaylistId(itemId)
+	req := yt.client.PlaylistItems.List("id,snippet").MaxResults(maxYoutubeResults).PlaylistId(itemId)
 	if pageToken != "" {
 		req = req.PageToken(pageToken)
 	}
@@ -327,14 +318,7 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ids []string, feed *database.Fe
 		// Add download links
 
 		size := yt.getVideoSize(video.ContentDetails.Definition, seconds, feed.Format)
-
+		item.AddEnclosure(makeEnclosure(feed, video.Id, size))
-		if feed.Format == database.VideoFormat {
-			downloadLink := fmt.Sprintf("http://podsync.net/download/%s/%s.mp4", feed.HashId, video.Id)
-			item.AddEnclosure(downloadLink, itunes.MP4, size)
-		} else {
-			downloadLink := fmt.Sprintf("http://podsync.net/download/%s/%s.m4a", feed.HashId, video.Id)
-			item.AddEnclosure(downloadLink, itunes.M4A, size)
-		}
 
 		_, err = podcast.AddItem(item)
 		if err != nil {

web/pkg/builders/youtube_test.go

@@ -3,13 +3,15 @@ package builders
 import (
 	"testing"
 
+	"os"
+
 	"github.com/mxpv/podsync/web/pkg/database"
 	"github.com/stretchr/testify/require"
 )
 
-var ytKey = "AIzaSyAp0mB03BFY3fm0Oxaxk96-mnE0D3MeUp4"
+var ytKey = os.Getenv("YOUTUBE_TEST_API_KEY")
 
-func TestParsePlaylist(t *testing.T) {
+func TestParseYTPlaylist(t *testing.T) {
 	builder := &YouTubeBuilder{}
 
 	kind, id, err := builder.parseUrl("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C")
@@ -18,7 +20,7 @@ func TestParsePlaylist(t *testing.T) {
 	require.Equal(t, "PLCB9F975ECF01953C", id)
 }
 
-func TestParseChannel(t *testing.T) {
+func TestParseYTChannel(t *testing.T) {
 	builder := &YouTubeBuilder{}
 
 	kind, id, err := builder.parseUrl("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og")
@@ -32,7 +34,7 @@ func TestParseChannel(t *testing.T) {
 	require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id)
 }
 
-func TestParseUser(t *testing.T) {
+func TestParseYTUser(t *testing.T) {
 	builder := &YouTubeBuilder{}
 
 	kind, id, err := builder.parseUrl("https://youtube.com/user/fxigr1")
@@ -41,7 +43,7 @@ func TestParseUser(t *testing.T) {
 	require.Equal(t, "fxigr1", id)
 }
 
-func TestHandleInvalidLink(t *testing.T) {
+func TestHandleInvalidYTLink(t *testing.T) {
 	builder := &YouTubeBuilder{}
 
 	_, _, err := builder.parseUrl("https://www.youtube.com/user///")
@@ -51,7 +53,7 @@ func TestHandleInvalidLink(t *testing.T) {
 	require.Error(t, err)
 }
 
-func TestQueryChannel(t *testing.T) {
+func TestQueryYTChannel(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping YT test in short mode")
 	}
@@ -68,7 +70,7 @@ func TestQueryChannel(t *testing.T) {
 	require.Equal(t, "UCr_fwF-n-2_olTYd-m3n32g", channel.Id)
 }
 
-func TestBuild(t *testing.T) {
+func TestBuildYTFeed(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping YT test in short mode")
 	}
@@ -78,7 +80,7 @@ func TestBuild(t *testing.T) {
 
 	podcast, err := builder.Build(&database.Feed{
 		URL:      "https://youtube.com/channel/UCupvZG-5ko_eiXAupbDfxWw",
-		PageSize: maxResults,
+		PageSize: maxYoutubeResults,
 	})
 	require.NoError(t, err)