From 616fac57fd37a53d79f050562f4c9748de36c374 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 29 Oct 2019 14:38:29 -0700 Subject: [PATCH] Implement basic feed updater --- cmd/podsync/main.go | 3 +- {pkg/web => cmd/podsync}/server.go | 4 +- cmd/podsync/updater.go | 158 +++++++++++++++++ pkg/api/api.go | 93 ---------- pkg/builder/common.go | 17 ++ pkg/builder/vimeo.go | 60 ++++--- pkg/builder/vimeo_test.go | 18 +- pkg/builder/youtube.go | 83 ++++----- pkg/config/config.go | 30 ++-- pkg/feeds/feeds.go | 270 ----------------------------- pkg/feeds/feeds_mock_test.go | 144 --------------- pkg/feeds/feeds_test.go | 144 --------------- pkg/feeds/id_gen.go | 24 --- pkg/link/url.go | 20 ++- pkg/{builder => model}/feed.go | 27 ++- pkg/model/model.go | 61 ------- pkg/model/timestamp.go | 50 ------ pkg/model/timestamp_test.go | 22 --- 18 files changed, 314 insertions(+), 914 deletions(-) rename {pkg/web => cmd/podsync}/server.go (87%) create mode 100644 cmd/podsync/updater.go delete mode 100644 pkg/api/api.go create mode 100644 pkg/builder/common.go delete mode 100644 pkg/feeds/feeds.go delete mode 100644 pkg/feeds/feeds_mock_test.go delete mode 100644 pkg/feeds/feeds_test.go delete mode 100644 pkg/feeds/id_gen.go rename pkg/{builder => model}/feed.go (57%) delete mode 100644 pkg/model/model.go delete mode 100644 pkg/model/timestamp.go delete mode 100644 pkg/model/timestamp_test.go diff --git a/cmd/podsync/main.go b/cmd/podsync/main.go index 1fccb2d..b3b6419 100644 --- a/cmd/podsync/main.go +++ b/cmd/podsync/main.go @@ -11,7 +11,6 @@ import ( "golang.org/x/sync/errgroup" "github.com/mxpv/podsync/pkg/config" - "github.com/mxpv/podsync/pkg/web" ) type Opts struct { @@ -48,7 +47,7 @@ func main() { log.WithError(err).Fatal("failed to load configuration file") } - srv := web.New(cfg) + srv := NewServer(cfg) group.Go(func() error { log.Infof("running listener at %s", srv.Addr) diff --git a/pkg/web/server.go b/cmd/podsync/server.go similarity index 87% rename from pkg/web/server.go rename to cmd/podsync/server.go index 9d4a2df..8d20bd0 100644 --- a/pkg/web/server.go +++ b/cmd/podsync/server.go @@ -1,4 +1,4 @@ -package web +package main import ( "fmt" @@ -13,7 +13,7 @@ type Server struct { http.Server } -func New(cfg *config.Config) *Server { +func NewServer(cfg *config.Config) *Server { port := cfg.Server.Port if port == 0 { port = 8080 diff --git a/cmd/podsync/updater.go b/cmd/podsync/updater.go new file mode 100644 index 0000000..30cc6f9 --- /dev/null +++ b/cmd/podsync/updater.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + "time" + + itunes "github.com/mxpv/podcast" + "github.com/pkg/errors" + + "github.com/mxpv/podsync/pkg/builder" + "github.com/mxpv/podsync/pkg/config" + "github.com/mxpv/podsync/pkg/link" + "github.com/mxpv/podsync/pkg/model" +) + +type Updater struct { + config *config.Config +} + +func NewUpdater(config *config.Config) (*Updater, error) { + return &Updater{config: config}, nil +} + +func (u *Updater) Update(ctx context.Context, feed *config.Feed) error { + // Create an updater for this feed type + provider, err := u.makeBuilder(ctx, feed) + if err != nil { + return err + } + + // Query API to get episodes + result, err := provider.Build(feed) + if err != nil { + return err + } + + // Build iTunes XML feed with data received from builder + podcast, err := u.buildPodcast(result) + if err != nil { + return err + } + + // Save XML to disk + xmlName := fmt.Sprintf("%s.xml", result.ItemID) + xmlPath := filepath.Join(u.config.Server.DataDir, xmlName) + if err := ioutil.WriteFile(xmlPath, []byte(podcast.String()), 600); err != nil { + return errors.Wrapf(err, "failed to write XML feed to disk") + } + + return nil +} + +func (u *Updater) buildPodcast(feed *model.Feed) (*itunes.Podcast, error) { + const ( + podsyncGenerator = "Podsync generator" + defaultCategory = "TV & Film" + ) + + now := time.Now().UTC() + + p := itunes.New(feed.Title, feed.ItemURL, feed.Description, &feed.PubDate, &now) + p.Generator = podsyncGenerator + p.AddSubTitle(feed.Title) + p.AddCategory(defaultCategory, nil) + p.AddImage(feed.CoverArt) + p.IAuthor = feed.Title + p.AddSummary(feed.Description) + + if feed.Explicit { + p.IExplicit = "yes" + } else { + p.IExplicit = "no" + } + + if feed.Language != "" { + p.Language = feed.Language + } + + for i, episode := range feed.Episodes { + item := itunes.Item{ + GUID: episode.ID, + Link: episode.VideoURL, + Title: episode.Title, + Description: episode.Description, + ISubtitle: episode.Title, + IOrder: strconv.Itoa(i), + } + + pubDate := episode.PubDate + if pubDate.IsZero() { + pubDate = now + } + + item.AddPubDate(&pubDate) + + item.AddSummary(episode.Description) + item.AddImage(episode.Thumbnail) + item.AddDuration(episode.Duration) + item.AddEnclosure(u.makeEnclosure(feed, episode)) + + // p.AddItem requires description to be not empty, use workaround + if item.Description == "" { + item.Description = " " + } + + if feed.Explicit { + item.IExplicit = "yes" + } else { + item.IExplicit = "no" + } + + _, err := p.AddItem(item) + if err != nil { + return nil, errors.Wrapf(err, "failed to add item to podcast (id %q)", episode.ID) + } + } + + return &p, nil +} + +func (u *Updater) makeEnclosure(feed *model.Feed, episode *model.Episode) (string, itunes.EnclosureType, int64) { + ext := "mp4" + contentType := itunes.MP4 + if feed.Format == model.FormatAudio { + ext = "m4a" + contentType = itunes.M4A + } + + url := fmt.Sprintf("%s/%s/%s.%s", u.config.Server.Hostname, feed.ItemID, episode.ID, ext) + return url, contentType, episode.Size +} + +func (u *Updater) makeBuilder(ctx context.Context, feed *config.Feed) (builder.Builder, error) { + var ( + provider builder.Builder + err error + ) + + info, err := link.Parse(feed.URL) + if err != nil { + return nil, err + } + + switch info.Provider { + case link.ProviderYoutube: + provider, err = builder.NewYouTubeBuilder(u.config.Tokens.YouTube) + case link.ProviderVimeo: + provider, err = builder.NewVimeoBuilder(ctx, u.config.Tokens.Vimeo) + default: + return nil, errors.Errorf("unsupported provider %q", info.Provider) + } + + return provider, err +} diff --git a/pkg/api/api.go b/pkg/api/api.go deleted file mode 100644 index d1686e1..0000000 --- a/pkg/api/api.go +++ /dev/null @@ -1,93 +0,0 @@ -package api - -import ( - "github.com/pkg/errors" -) - -var ( - ErrNotFound = errors.New("resource not found") - ErrQuotaExceeded = errors.New("query limit is exceeded") -) - -type Provider string - -const ( - ProviderYoutube = Provider("youtube") - ProviderVimeo = Provider("vimeo") - ProviderGeneric = Provider("generic") -) - -type LinkType string - -const ( - LinkTypeChannel = LinkType("channel") - LinkTypePlaylist = LinkType("playlist") - LinkTypeUser = LinkType("user") - LinkTypeGroup = LinkType("group") -) - -type Quality string - -const ( - QualityHigh = Quality("high") - QualityLow = Quality("low") -) - -type Format string - -const ( - FormatAudio = Format("audio") - FormatVideo = Format("video") -) - -const ( - DefaultPageSize = 50 - DefaultFormat = FormatVideo - DefaultQuality = QualityHigh - ExtendedPaginationQueryLimit = 5000 -) - -type Metadata struct { - Provider Provider `json:"provider"` - Format Format `json:"format"` - Quality Quality `json:"quality"` - Downloads int64 `json:"downloads"` -} - -const ( - // DefaultFeatures represent features for Anonymous user - // Page size: 50 - // Format: video - // Quality: high - DefaultFeatures = iota - - // ExtendedFeatures represent features for 1$ pledges - // Max page size: 150 - // Format: any - // Quality: any - ExtendedFeatures - - // ExtendedPagination represent extended pagination feature set - // Max page size: 600 - // Format: any - // Quality: any - ExtendedPagination - - // PodcasterFeatures reserved for future - PodcasterFeatures -) - -type CreateFeedRequest struct { - URL string `json:"url" binding:"url,required"` - PageSize int `json:"page_size" binding:"min=10,max=600,required"` - Quality Quality `json:"quality" binding:"eq=high|eq=low"` - Format Format `json:"format" binding:"eq=video|eq=audio"` -} - -type Identity struct { - UserID string `json:"user_id"` - FullName string `json:"full_name"` - Email string `json:"email"` - ProfileURL string `json:"profile_url"` - FeatureLevel int `json:"feature_level"` -} diff --git a/pkg/builder/common.go b/pkg/builder/common.go new file mode 100644 index 0000000..af66f7e --- /dev/null +++ b/pkg/builder/common.go @@ -0,0 +1,17 @@ +package builder + +import ( + "errors" + + "github.com/mxpv/podsync/pkg/config" + "github.com/mxpv/podsync/pkg/model" +) + +var ( + ErrNotFound = errors.New("resource not found") + ErrQuotaExceeded = errors.New("query limit is exceeded") +) + +type Builder interface { + Build(cfg *config.Feed) (*model.Feed, error) +} diff --git a/pkg/builder/vimeo.go b/pkg/builder/vimeo.go index 8c0917d..eaf637c 100644 --- a/pkg/builder/vimeo.go +++ b/pkg/builder/vimeo.go @@ -10,9 +10,9 @@ import ( "golang.org/x/net/context" "golang.org/x/oauth2" - "github.com/mxpv/podsync/pkg/api" "github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/link" + "github.com/mxpv/podsync/pkg/model" ) const ( @@ -23,25 +23,25 @@ type VimeoBuilder struct { client *vimeo.Client } -func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q config.Quality) string { +func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q model.Quality) string { if p == nil || len(p.Sizes) == 0 { return "" } - if q == config.QualityLow { + if q == model.QualityLow { return p.Sizes[0].Link } return p.Sizes[len(p.Sizes)-1].Link } -func (v *VimeoBuilder) queryChannel(feed *Feed, cfg *config.Feed) error { +func (v *VimeoBuilder) queryChannel(feed *model.Feed) error { channelID := feed.ItemID ch, resp, err := v.client.Channels.Get(channelID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return api.ErrNotFound + return ErrNotFound } return errors.Wrapf(err, "failed to query channel with id %q", channelID) @@ -50,7 +50,7 @@ func (v *VimeoBuilder) queryChannel(feed *Feed, cfg *config.Feed) error { feed.Title = ch.Name feed.ItemURL = ch.Link feed.Description = ch.Description - feed.CoverArt = v.selectImage(ch.Pictures, cfg.Quality) + feed.CoverArt = v.selectImage(ch.Pictures, feed.Quality) feed.Author = ch.User.Name feed.PubDate = ch.CreatedTime feed.UpdatedAt = time.Now().UTC() @@ -58,13 +58,13 @@ func (v *VimeoBuilder) queryChannel(feed *Feed, cfg *config.Feed) error { return nil } -func (v *VimeoBuilder) queryGroup(feed *Feed, cfg *config.Feed) error { +func (v *VimeoBuilder) queryGroup(feed *model.Feed) error { groupID := feed.ItemID gr, resp, err := v.client.Groups.Get(groupID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return api.ErrNotFound + return ErrNotFound } return errors.Wrapf(err, "failed to query group with id %q", groupID) @@ -73,7 +73,7 @@ func (v *VimeoBuilder) queryGroup(feed *Feed, cfg *config.Feed) error { feed.Title = gr.Name feed.ItemURL = gr.Link feed.Description = gr.Description - feed.CoverArt = v.selectImage(gr.Pictures, cfg.Quality) + feed.CoverArt = v.selectImage(gr.Pictures, feed.Quality) feed.Author = gr.User.Name feed.PubDate = gr.CreatedTime feed.UpdatedAt = time.Now().UTC() @@ -81,13 +81,13 @@ func (v *VimeoBuilder) queryGroup(feed *Feed, cfg *config.Feed) error { return nil } -func (v *VimeoBuilder) queryUser(feed *Feed, cfg *config.Feed) error { +func (v *VimeoBuilder) queryUser(feed *model.Feed) error { userID := feed.ItemID user, resp, err := v.client.Users.Get(userID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return api.ErrNotFound + return ErrNotFound } return errors.Wrapf(err, "failed to query user with id %q", userID) @@ -96,7 +96,7 @@ func (v *VimeoBuilder) queryUser(feed *Feed, cfg *config.Feed) error { feed.Title = user.Name feed.ItemURL = user.Link feed.Description = user.Bio - feed.CoverArt = v.selectImage(user.Pictures, cfg.Quality) + feed.CoverArt = v.selectImage(user.Pictures, feed.Quality) feed.Author = user.Name feed.PubDate = user.CreatedTime feed.UpdatedAt = time.Now().UTC() @@ -111,7 +111,7 @@ func (v *VimeoBuilder) getVideoSize(video *vimeo.Video) int64 { type getVideosFunc func(string, ...vimeo.CallOption) ([]*vimeo.Video, *vimeo.Response, error) -func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, feed *Feed, cfg *config.Feed) error { +func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, feed *model.Feed) error { var ( page = 1 added = 0 @@ -133,10 +133,10 @@ func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, feed *Feed, cfg *con videoURL = video.Link duration = int64(video.Duration) size = v.getVideoSize(video) - image = v.selectImage(video.Pictures, cfg.Quality) + image = v.selectImage(video.Pictures, feed.Quality) ) - feed.Episodes = append(feed.Episodes, &Item{ + feed.Episodes = append(feed.Episodes, &model.Episode{ ID: videoID, Title: video.Name, Description: video.Description, @@ -150,7 +150,7 @@ func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, feed *Feed, cfg *con added++ } - if added >= cfg.PageSize || response.NextPage == "" { + if added >= feed.PageSize || response.NextPage == "" { return nil } @@ -158,20 +158,28 @@ func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, feed *Feed, cfg *con } } -func (v *VimeoBuilder) Build(cfg *config.Feed) (*Feed, error) { +func (v *VimeoBuilder) Build(cfg *config.Feed) (*model.Feed, error) { info, err := link.Parse(cfg.URL) if err != nil { return nil, err } - feed := &Feed{} + feed := &model.Feed{ + ItemID: info.ItemID, + Provider: info.Provider, + LinkType: info.LinkType, + Format: cfg.Format, + Quality: cfg.Quality, + PageSize: cfg.PageSize, + UpdatedAt: time.Now().UTC(), + } if info.LinkType == link.TypeChannel { - if err := v.queryChannel(feed, cfg); err != nil { + if err := v.queryChannel(feed); err != nil { return nil, err } - if err := v.queryVideos(v.client.Channels.ListVideo, feed, cfg); err != nil { + if err := v.queryVideos(v.client.Channels.ListVideo, feed); err != nil { return nil, err } @@ -179,11 +187,11 @@ func (v *VimeoBuilder) Build(cfg *config.Feed) (*Feed, error) { } if info.LinkType == link.TypeGroup { - if err := v.queryGroup(feed, cfg); err != nil { + if err := v.queryGroup(feed); err != nil { return nil, err } - if err := v.queryVideos(v.client.Groups.ListVideo, feed, cfg); err != nil { + if err := v.queryVideos(v.client.Groups.ListVideo, feed); err != nil { return nil, err } @@ -191,11 +199,11 @@ func (v *VimeoBuilder) Build(cfg *config.Feed) (*Feed, error) { } if info.LinkType == link.TypeUser { - if err := v.queryUser(feed, cfg); err != nil { + if err := v.queryUser(feed); err != nil { return nil, err } - if err := v.queryVideos(v.client.Users.ListVideo, feed, cfg); err != nil { + if err := v.queryVideos(v.client.Users.ListVideo, feed); err != nil { return nil, err } @@ -206,6 +214,10 @@ func (v *VimeoBuilder) Build(cfg *config.Feed) (*Feed, error) { } func NewVimeoBuilder(ctx context.Context, token string) (*VimeoBuilder, error) { + if token == "" { + return nil, errors.New("empty Vimeo access token") + } + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) tc := oauth2.NewClient(ctx, ts) diff --git a/pkg/builder/vimeo_test.go b/pkg/builder/vimeo_test.go index 5f4b896..dba71e3 100644 --- a/pkg/builder/vimeo_test.go +++ b/pkg/builder/vimeo_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/mxpv/podsync/pkg/config" + "github.com/mxpv/podsync/pkg/model" ) var ( @@ -22,8 +22,8 @@ func TestQueryVimeoChannel(t *testing.T) { builder, err := NewVimeoBuilder(context.Background(), vimeoKey) require.NoError(t, err) - podcast := &Feed{ItemID: "staffpicks"} - err = builder.queryChannel(podcast, &config.Feed{Quality: config.QualityHigh}) + podcast := &model.Feed{ItemID: "staffpicks", Quality: model.QualityHigh} + err = builder.queryChannel(podcast) require.NoError(t, err) require.Equal(t, "https://vimeo.com/channels/staffpicks", podcast.ItemURL) @@ -41,8 +41,8 @@ func TestQueryVimeoGroup(t *testing.T) { builder, err := NewVimeoBuilder(context.Background(), vimeoKey) require.NoError(t, err) - podcast := &Feed{ItemID: "motion"} - err = builder.queryGroup(podcast, &config.Feed{Quality: config.QualityHigh}) + podcast := &model.Feed{ItemID: "motion", Quality: model.QualityHigh} + err = builder.queryGroup(podcast) require.NoError(t, err) require.Equal(t, "https://vimeo.com/groups/motion", podcast.ItemURL) @@ -60,8 +60,8 @@ func TestQueryVimeoUser(t *testing.T) { builder, err := NewVimeoBuilder(context.Background(), vimeoKey) require.NoError(t, err) - podcast := &Feed{ItemID: "motionarray"} - err = builder.queryUser(podcast, &config.Feed{Quality: config.QualityHigh}) + podcast := &model.Feed{ItemID: "motionarray", Quality: model.QualityHigh} + err = builder.queryUser(podcast) require.NoError(t, err) require.Equal(t, "https://vimeo.com/motionarray", podcast.ItemURL) @@ -78,9 +78,9 @@ func TestQueryVimeoVideos(t *testing.T) { builder, err := NewVimeoBuilder(context.Background(), vimeoKey) require.NoError(t, err) - feed := &Feed{ItemID: "staffpicks"} + feed := &model.Feed{ItemID: "staffpicks", Quality: model.QualityHigh} - err = builder.queryVideos(builder.client.Channels.ListVideo, feed, &config.Feed{Quality: config.QualityHigh}) + err = builder.queryVideos(builder.client.Channels.ListVideo, feed) require.NoError(t, err) require.Equal(t, vimeoDefaultPageSize, len(feed.Episodes)) diff --git a/pkg/builder/youtube.go b/pkg/builder/youtube.go index 23baa64..2a66f25 100644 --- a/pkg/builder/youtube.go +++ b/pkg/builder/youtube.go @@ -12,9 +12,9 @@ import ( "github.com/pkg/errors" "google.golang.org/api/youtube/v3" - "github.com/mxpv/podsync/pkg/api" "github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/link" + "github.com/mxpv/podsync/pkg/model" ) const ( @@ -56,7 +56,7 @@ func (yt *YouTubeBuilder) listChannels(linkType link.Type, id string, parts stri } if len(resp.Items) == 0 { - return nil, api.ErrNotFound + return nil, ErrNotFound } item := resp.Items[0] @@ -80,7 +80,7 @@ func (yt *YouTubeBuilder) listPlaylists(id, channelID string, parts string) (*yo } if len(resp.Items) == 0 { - return nil, api.ErrNotFound + return nil, ErrNotFound } item := resp.Items[0] @@ -112,7 +112,7 @@ func (yt *YouTubeBuilder) parseDate(s string) (time.Time, error) { return date, nil } -func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, quality config.Quality, videoID string) string { +func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, quality model.Quality, videoID string) string { if snippet == nil { if videoID != "" { return fmt.Sprintf("https://img.youtube.com/vi/%s/default.jpg", videoID) @@ -124,7 +124,7 @@ func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, qua // Use high resolution thumbnails for high quality mode // https://github.com/mxpv/Podsync/issues/14 - if quality == config.QualityHigh { + if quality == model.QualityHigh { if snippet.Maxres != nil { return snippet.Maxres.Url } @@ -164,24 +164,17 @@ func (yt *YouTubeBuilder) GetVideoCount(info *link.Info) (uint64, error) { } } -func (yt *YouTubeBuilder) queryFeed(info *link.Info, config *config.Feed) (*Feed, error) { +func (yt *YouTubeBuilder) queryFeed(feed *model.Feed, info *link.Info) error { var ( thumbnails *youtube.ThumbnailDetails ) - feed := Feed{ - ItemID: info.ItemID, - Provider: info.Provider, - LinkType: info.LinkType, - UpdatedAt: time.Now().UTC(), - } - switch info.LinkType { case link.TypeChannel, link.TypeUser: // Cost: 5 units for channel or user channel, err := yt.listChannels(info.LinkType, info.ItemID, "id,snippet,contentDetails") if err != nil { - return nil, err + return err } feed.Title = channel.Snippet.Title @@ -198,7 +191,7 @@ func (yt *YouTubeBuilder) queryFeed(info *link.Info, config *config.Feed) (*Feed feed.ItemID = channel.ContentDetails.RelatedPlaylists.Uploads if date, err := yt.parseDate(channel.Snippet.PublishedAt); err != nil { - return nil, err + return err } else { // nolint:golint feed.PubDate = date } @@ -209,7 +202,7 @@ func (yt *YouTubeBuilder) queryFeed(info *link.Info, config *config.Feed) (*Feed // Cost: 3 units for playlist playlist, err := yt.listPlaylists(info.ItemID, "", "id,snippet") if err != nil { - return nil, err + return err } feed.Title = fmt.Sprintf("%s: %s", playlist.Snippet.ChannelTitle, playlist.Snippet.Title) @@ -221,7 +214,7 @@ func (yt *YouTubeBuilder) queryFeed(info *link.Info, config *config.Feed) (*Feed feed.Author = "" if date, err := yt.parseDate(playlist.Snippet.PublishedAt); err != nil { - return nil, err + return err } else { // nolint:golint feed.PubDate = date } @@ -229,29 +222,25 @@ func (yt *YouTubeBuilder) queryFeed(info *link.Info, config *config.Feed) (*Feed thumbnails = playlist.Snippet.Thumbnails default: - return nil, errors.New("unsupported link format") + return errors.New("unsupported link format") } - // Apply customizations and default values - if feed.Description == "" { feed.Description = fmt.Sprintf("%s (%s)", feed.Title, feed.PubDate) } - if config.CoverArt != "" { - feed.CoverArt = config.CoverArt - } else { - feed.CoverArt = yt.selectThumbnail(thumbnails, config.Quality, "") + if feed.CoverArt == "" { + feed.CoverArt = yt.selectThumbnail(thumbnails, feed.Quality, "") } - return &feed, nil + return nil } // Video size information requires 1 additional call for each video (1 feed = 50 videos = 50 calls), // which is too expensive, so get approximated size depending on duration and definition params -func (yt *YouTubeBuilder) getSize(duration int64, cfg *config.Feed) int64 { - if cfg.Format == config.FormatAudio { - if cfg.Quality == config.QualityHigh { +func (yt *YouTubeBuilder) getSize(duration int64, feed *model.Feed) int64 { + if feed.Format == model.FormatAudio { + if feed.Quality == model.QualityHigh { return highAudioBytesPerSecond * duration } @@ -260,7 +249,7 @@ func (yt *YouTubeBuilder) getSize(duration int64, cfg *config.Feed) int64 { // Video format - if cfg.Quality == config.QualityHigh { + if feed.Quality == model.QualityHigh { return duration * hdBytesPerSecond } @@ -269,7 +258,7 @@ func (yt *YouTubeBuilder) getSize(duration int64, cfg *config.Feed) int64 { // Cost: 5 units (call: 1, snippet: 2, contentDetails: 2) // See https://developers.google.com/youtube/v3/docs/videos/list#part -func (yt *YouTubeBuilder) queryVideoDescriptions(playlist map[string]*youtube.PlaylistItemSnippet, feed *Feed, cfg *config.Feed) error { +func (yt *YouTubeBuilder) queryVideoDescriptions(playlist map[string]*youtube.PlaylistItemSnippet, feed *model.Feed) error { // Make the list of video ids ids := make([]string, 0, len(playlist)) for _, s := range playlist { @@ -286,7 +275,7 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(playlist map[string]*youtube.Pl snippet = video.Snippet videoID = video.Id videoURL = fmt.Sprintf("https://youtube.com/watch?v=%s", video.Id) - image = yt.selectThumbnail(snippet.Thumbnails, cfg.Quality, videoID) + image = yt.selectThumbnail(snippet.Thumbnails, feed.Quality, videoID) ) // Parse date added to playlist / publication date @@ -317,10 +306,10 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(playlist map[string]*youtube.Pl var ( order = strconv.FormatInt(playlistItem.Position, 10) - size = yt.getSize(seconds, cfg) + size = yt.getSize(seconds, feed) ) - feed.Episodes = append(feed.Episodes, &Item{ + feed.Episodes = append(feed.Episodes, &model.Episode{ ID: video.Id, Title: snippet.Title, Description: snippet.Description, @@ -337,7 +326,7 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(playlist map[string]*youtube.Pl } // Cost: (3 units + 5 units) * X pages = 8 units per page -func (yt *YouTubeBuilder) queryItems(feed *Feed, cfg *config.Feed) error { +func (yt *YouTubeBuilder) queryItems(feed *model.Feed) error { var ( token string count int @@ -363,29 +352,39 @@ func (yt *YouTubeBuilder) queryItems(feed *Feed, cfg *config.Feed) error { } // Query video descriptions from the list of ids - if err := yt.queryVideoDescriptions(snippets, feed, cfg); err != nil { + if err := yt.queryVideoDescriptions(snippets, feed); err != nil { return err } - if count >= cfg.PageSize || token == "" { + if count >= feed.PageSize || token == "" { return nil } } } -func (yt *YouTubeBuilder) Build(cfg *config.Feed) (*Feed, error) { +func (yt *YouTubeBuilder) Build(cfg *config.Feed) (*model.Feed, error) { info, err := link.Parse(cfg.URL) if err != nil { return nil, err } + feed := &model.Feed{ + ItemID: info.ItemID, + Provider: info.Provider, + LinkType: info.LinkType, + Format: cfg.Format, + Quality: cfg.Quality, + PageSize: cfg.PageSize, + CoverArt: cfg.CoverArt, + UpdatedAt: time.Now().UTC(), + } + // Query general information about feed (title, description, lang, etc) - feed, err := yt.queryFeed(&info, cfg) - if err != nil { + if err := yt.queryFeed(feed, &info); err != nil { return nil, err } - if err := yt.queryItems(feed, cfg); err != nil { + if err := yt.queryItems(feed); err != nil { return nil, err } @@ -399,6 +398,10 @@ func (yt *YouTubeBuilder) Build(cfg *config.Feed) (*Feed, error) { } func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) { + if key == "" { + return nil, errors.New("empty YouTube API key") + } + yt, err := youtube.New(&http.Client{}) if err != nil { return nil, errors.Wrap(err, "failed to create youtube client") diff --git a/pkg/config/config.go b/pkg/config/config.go index 664aee8..00cc80b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -5,27 +5,13 @@ import ( "github.com/BurntSushi/toml" "github.com/pkg/errors" -) -// Quality to use when downloading episodes -type Quality string - -const ( - QualityHigh = Quality("high") - QualityLow = Quality("low") -) - -// Format to convert episode when downloading episodes -type Format string - -const ( - FormatAudio = Format("audio") - FormatVideo = Format("video") + "github.com/mxpv/podsync/pkg/model" ) const ( - DefaultFormat = FormatVideo - DefaultQuality = QualityHigh + DefaultFormat = model.FormatVideo + DefaultQuality = model.QualityHigh DefaultPageSize = 50 DefaultUpdatePeriod = 24 * time.Hour ) @@ -43,9 +29,9 @@ type Feed struct { // NOTE: too often update check might drain your API token. UpdatePeriod Duration `toml:"update_period"` // Quality to use for this feed - Quality Quality `toml:"quality"` + Quality model.Quality `toml:"quality"` // Format to use for this feed - Format Format `toml:"format"` + Format model.Format `toml:"format"` // Custom image to use CoverArt string `toml:"cover_art"` } @@ -60,6 +46,8 @@ type Tokens struct { } type Server struct { + // Hostname to use for download links + Hostname string `toml:"name"` // Port is a server port to listen to Port int `toml:"port"` // DataDir is a path to a directory to keep XML feeds and downloaded episodes, @@ -86,6 +74,10 @@ func LoadConfig(path string) (*Config, error) { } // Apply defaults + if config.Server.Hostname == "" { + config.Server.Hostname = "http://localhost" + } + for _, feed := range config.Feeds { if feed.UpdatePeriod.Duration == 0 { feed.UpdatePeriod.Duration = DefaultUpdatePeriod diff --git a/pkg/feeds/feeds.go b/pkg/feeds/feeds.go deleted file mode 100644 index d857ea8..0000000 --- a/pkg/feeds/feeds.go +++ /dev/null @@ -1,270 +0,0 @@ -package feeds - -import ( - "fmt" - "strconv" - "time" - - itunes "github.com/mxpv/podcast" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - - "github.com/mxpv/podsync/pkg/api" - "github.com/mxpv/podsync/pkg/link" - "github.com/mxpv/podsync/pkg/model" -) - -type Builder interface { - Build(feed *model.Feed) error -} - -type storage interface { - SaveFeed(feed *model.Feed) error - GetFeed(hashID string) (*model.Feed, error) - UpdateFeed(feed *model.Feed) error - GetMetadata(hashID string) (*model.Feed, error) - Downgrade(userID string, featureLevel int) ([]string, error) -} - -type Service struct { - generator IDGen - storage storage - builders map[api.Provider]Builder -} - -func NewFeedService(db storage, builders map[api.Provider]Builder) (*Service, error) { - idGen, err := NewIDGen() - if err != nil { - return nil, err - } - - svc := &Service{ - generator: idGen, - storage: db, - builders: builders, - } - - return svc, nil -} - -func (s *Service) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) { - feed, err := s.makeFeed(req, identity) - if err != nil { - return "", err - } - - logger := log.WithField("feed_id", feed.HashID) - - // Make sure builder exists for this provider - builder, ok := s.builders[feed.Provider] - if !ok { - return "", fmt.Errorf("failed to get builder for URL: %s", req.URL) - } - - logger.Infof("creating new feed for %q", feed.ItemURL) - - if err := builder.Build(feed); err != nil { - logger.WithError(err).Error("failed to build feed") - - return "", err - } - - logger.Infof("saving new feed to database") - - if err := s.storage.SaveFeed(feed); err != nil { - return "", err - } - - return feed.HashID, nil -} - -func (s *Service) makeFeed(req *api.CreateFeedRequest, identity *api.Identity) (*model.Feed, error) { - _, err := link.Parse(req.URL) - if err != nil { - return nil, errors.Wrapf(err, "failed to create feed for URL: %s", req.URL) - } - - feed := &model.Feed{} - - now := time.Now().UTC() - - feed.UserID = identity.UserID - feed.FeatureLevel = identity.FeatureLevel - feed.Quality = req.Quality - feed.Format = req.Format - feed.PageSize = req.PageSize - feed.CreatedAt = now - feed.LastAccess = now - - switch { - case identity.FeatureLevel >= api.ExtendedPagination: - if feed.PageSize > 600 { - feed.PageSize = 600 - } - case identity.FeatureLevel == api.ExtendedFeatures: - if feed.PageSize > 150 { - feed.PageSize = 150 - } - default: - feed.Quality = api.QualityHigh - feed.Format = api.FormatVideo - feed.PageSize = 50 - } - - // Generate short id - hashID, err := s.generator.Generate() - if err != nil { - return nil, errors.Wrap(err, "failed to generate id for feed") - } - - feed.HashID = hashID - - return feed, nil -} - -func (s Service) QueryFeed(hashID string) (*model.Feed, error) { - return s.storage.GetFeed(hashID) -} - -func makeEnclosure(feed *model.Feed, id string, lengthInBytes int64) (string, itunes.EnclosureType, int64) { - ext := "mp4" - contentType := itunes.MP4 - if feed.Format == api.FormatAudio { - ext = "m4a" - contentType = itunes.M4A - } - - url := fmt.Sprintf("https://dl.podsync.net/download/%s/%s.%s", feed.HashID, id, ext) - return url, contentType, lengthInBytes -} - -func (s *Service) BuildFeed(hashID string) ([]byte, error) { - logger := log.WithField("hash_id", hashID) - - feed, err := s.QueryFeed(hashID) - if err != nil { - logger.WithError(err).Error("failed to query feed from dynamodb") - return nil, err - } - - // Output the feed - - if feed.PageSize < len(feed.Episodes) { - feed.Episodes = feed.Episodes[:feed.PageSize] - } - - podcast, err := s.buildPodcast(feed) - if err != nil { - return nil, err - } - - return []byte(podcast.String()), nil -} - -func (s *Service) buildPodcast(feed *model.Feed) (*itunes.Podcast, error) { - const ( - podsyncGenerator = "Podsync generator" - defaultCategory = "TV & Film" - ) - - now := time.Now().UTC() - - p := itunes.New(feed.Title, feed.ItemURL, feed.Description, &feed.PubDate, &now) - p.Generator = podsyncGenerator - p.AddSubTitle(feed.Title) - p.AddCategory(defaultCategory, nil) - p.AddImage(feed.CoverArt) - p.IAuthor = feed.Title - p.AddSummary(feed.Description) - - if feed.Explicit { - p.IExplicit = "yes" - } else { - p.IExplicit = "no" - } - - if feed.Language != "" { - p.Language = feed.Language - } - - for i, episode := range feed.Episodes { - item := itunes.Item{ - GUID: episode.ID, - Link: episode.VideoURL, - Title: episode.Title, - Description: episode.Description, - ISubtitle: episode.Title, - IOrder: strconv.Itoa(i), - } - - pubDate := time.Time(episode.PubDate) - if pubDate.IsZero() { - ts := now - if i > 0 { - if prev := time.Time(feed.Episodes[i-1].PubDate); !prev.IsZero() { - ts = prev - } - } - - // HACK: some dates are cached incorrectly resulting incorrect behavior of some podcast clients. - // Use this hack to have sequence ordered by date. - pubDate = ts.Add(-time.Duration(i) * time.Hour) - } - - item.AddPubDate(&pubDate) - - item.AddSummary(episode.Description) - item.AddImage(episode.Thumbnail) - item.AddDuration(episode.Duration) - item.AddEnclosure(makeEnclosure(feed, episode.ID, episode.Size)) - - // p.AddItem requires description to be not empty, use workaround - if item.Description == "" { - item.Description = " " - } - - if feed.Explicit { - item.IExplicit = "yes" - } else { - item.IExplicit = "no" - } - - _, err := p.AddItem(item) - if err != nil { - return nil, errors.Wrapf(err, "failed to add item to podcast (id %q)", episode.ID) - } - } - - return &p, nil -} - -func (s Service) GetMetadata(hashID string) (*api.Metadata, error) { - feed, err := s.storage.GetMetadata(hashID) - if err != nil { - return nil, err - } - - return &api.Metadata{ - Provider: feed.Provider, - Format: feed.Format, - Quality: feed.Quality, - }, nil -} - -func (s Service) Downgrade(patronID string, featureLevel int) error { - logger := log.WithFields(log.Fields{ - "user_id": patronID, - "level": featureLevel, - }) - - logger.Info("downgrading patron") - - _, err := s.storage.Downgrade(patronID, featureLevel) - if err != nil { - logger.WithError(err).Error("database error while downgrading patron") - return err - } - - logger.Info("successfully updated user") - return nil -} diff --git a/pkg/feeds/feeds_mock_test.go b/pkg/feeds/feeds_mock_test.go deleted file mode 100644 index 70cf226..0000000 --- a/pkg/feeds/feeds_mock_test.go +++ /dev/null @@ -1,144 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: feeds.go - -// Package feeds is a generated GoMock package. -package feeds - -import ( - gomock "github.com/golang/mock/gomock" - model "github.com/mxpv/podsync/pkg/model" - reflect "reflect" -) - -// MockBuilder is a mock of Builder interface -type MockBuilder struct { - ctrl *gomock.Controller - recorder *MockBuilderMockRecorder -} - -// MockBuilderMockRecorder is the mock recorder for MockBuilder -type MockBuilderMockRecorder struct { - mock *MockBuilder -} - -// NewMockBuilder creates a new mock instance -func NewMockBuilder(ctrl *gomock.Controller) *MockBuilder { - mock := &MockBuilder{ctrl: ctrl} - mock.recorder = &MockBuilderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockBuilder) EXPECT() *MockBuilderMockRecorder { - return m.recorder -} - -// Build mocks base method -func (m *MockBuilder) Build(feed *model.Feed) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Build", feed) - ret0, _ := ret[0].(error) - return ret0 -} - -// Build indicates an expected call of Build -func (mr *MockBuilderMockRecorder) Build(feed interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockBuilder)(nil).Build), feed) -} - -// Mockstorage is a mock of storage interface -type Mockstorage struct { - ctrl *gomock.Controller - recorder *MockstorageMockRecorder -} - -// MockstorageMockRecorder is the mock recorder for Mockstorage -type MockstorageMockRecorder struct { - mock *Mockstorage -} - -// NewMockstorage creates a new mock instance -func NewMockstorage(ctrl *gomock.Controller) *Mockstorage { - mock := &Mockstorage{ctrl: ctrl} - mock.recorder = &MockstorageMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *Mockstorage) EXPECT() *MockstorageMockRecorder { - return m.recorder -} - -// SaveFeed mocks base method -func (m *Mockstorage) SaveFeed(feed *model.Feed) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SaveFeed", feed) - ret0, _ := ret[0].(error) - return ret0 -} - -// SaveFeed indicates an expected call of SaveFeed -func (mr *MockstorageMockRecorder) SaveFeed(feed interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveFeed", reflect.TypeOf((*Mockstorage)(nil).SaveFeed), feed) -} - -// GetFeed mocks base method -func (m *Mockstorage) GetFeed(hashID string) (*model.Feed, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetFeed", hashID) - ret0, _ := ret[0].(*model.Feed) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetFeed indicates an expected call of GetFeed -func (mr *MockstorageMockRecorder) GetFeed(hashID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), hashID) -} - -// UpdateFeed mocks base method -func (m *Mockstorage) UpdateFeed(feed *model.Feed) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateFeed", feed) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateFeed indicates an expected call of UpdateFeed -func (mr *MockstorageMockRecorder) UpdateFeed(feed interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateFeed", reflect.TypeOf((*Mockstorage)(nil).UpdateFeed), feed) -} - -// GetMetadata mocks base method -func (m *Mockstorage) GetMetadata(hashID string) (*model.Feed, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetMetadata", hashID) - ret0, _ := ret[0].(*model.Feed) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetMetadata indicates an expected call of GetMetadata -func (mr *MockstorageMockRecorder) GetMetadata(hashID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetadata", reflect.TypeOf((*Mockstorage)(nil).GetMetadata), hashID) -} - -// Downgrade mocks base method -func (m *Mockstorage) Downgrade(userID string, featureLevel int) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Downgrade", userID, featureLevel) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Downgrade indicates an expected call of Downgrade -func (mr *MockstorageMockRecorder) Downgrade(userID, featureLevel interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Downgrade", reflect.TypeOf((*Mockstorage)(nil).Downgrade), userID, featureLevel) -} diff --git a/pkg/feeds/feeds_test.go b/pkg/feeds/feeds_test.go deleted file mode 100644 index c5e2d88..0000000 --- a/pkg/feeds/feeds_test.go +++ /dev/null @@ -1,144 +0,0 @@ -//go:generate mockgen -source=feeds.go -destination=feeds_mock_test.go -package=feeds - -package feeds - -import ( - "testing" - - "github.com/golang/mock/gomock" - "github.com/pkg/errors" - "github.com/stretchr/testify/require" - - "github.com/mxpv/podsync/pkg/api" - "github.com/mxpv/podsync/pkg/model" -) - -var feed = &model.Feed{ - HashID: "123", - ItemID: "xyz", - Provider: api.ProviderVimeo, - LinkType: api.LinkTypeChannel, - PageSize: 50, - Quality: api.QualityHigh, - Format: api.FormatVideo, - Episodes: []*model.Item{ - {ID: "1", Title: "Title", Description: "Description"}, - }, -} - -func TestService_CreateFeed(t *testing.T) { - t.Skip() - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - db := NewMockstorage(ctrl) - db.EXPECT().SaveFeed(gomock.Any()).Times(1).Return(nil) - - gen, _ := NewIDGen() - - builder := NewMockBuilder(ctrl) - builder.EXPECT().Build(gomock.Any()).Times(1).Return(nil) - - s := Service{ - generator: gen, - storage: db, - builders: map[api.Provider]Builder{api.ProviderYoutube: builder}, - } - - req := &api.CreateFeedRequest{ - URL: "youtube.com/channel/123", - PageSize: 50, - Quality: api.QualityHigh, - Format: api.FormatVideo, - } - - hashID, err := s.CreateFeed(req, &api.Identity{}) - require.NoError(t, err) - require.NotEmpty(t, hashID) -} - -func TestService_makeFeed(t *testing.T) { - req := &api.CreateFeedRequest{ - URL: "youtube.com/channel/123", - PageSize: 1000, - Quality: api.QualityLow, - Format: api.FormatAudio, - } - - gen, _ := NewIDGen() - - s := Service{ - generator: gen, - } - - feed, err := s.makeFeed(req, &api.Identity{}) - require.NoError(t, err) - require.Equal(t, 50, feed.PageSize) - require.Equal(t, api.QualityHigh, feed.Quality) - require.Equal(t, api.FormatVideo, feed.Format) - - feed, err = s.makeFeed(req, &api.Identity{FeatureLevel: api.ExtendedFeatures}) - require.NoError(t, err) - require.Equal(t, 150, feed.PageSize) - require.Equal(t, api.QualityLow, feed.Quality) - require.Equal(t, api.FormatAudio, feed.Format) - - feed, err = s.makeFeed(req, &api.Identity{FeatureLevel: api.ExtendedPagination}) - require.NoError(t, err) - require.Equal(t, 600, feed.PageSize) - require.Equal(t, api.QualityLow, feed.Quality) - require.Equal(t, api.FormatAudio, feed.Format) -} - -func TestService_QueryFeed(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - db := NewMockstorage(ctrl) - db.EXPECT().GetFeed("123").Times(1).Return(nil, nil) - - s := Service{storage: db} - _, err := s.QueryFeed("123") - require.NoError(t, err) -} - -func TestService_BuildFeed(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - stor := NewMockstorage(ctrl) - stor.EXPECT().GetFeed(feed.HashID).Times(1).Return(feed, nil) - - s := Service{storage: stor} - - _, err := s.BuildFeed(feed.HashID) - require.NoError(t, err) -} - -func TestService_WrongID(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - stor := NewMockstorage(ctrl) - stor.EXPECT().GetFeed(gomock.Any()).Times(1).Return(nil, errors.New("not found")) - - s := &Service{storage: stor} - - _, err := s.BuildFeed("invalid_feed_id") - require.Error(t, err) -} - -func TestService_GetMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - stor := NewMockstorage(ctrl) - stor.EXPECT().GetMetadata(feed.HashID).Times(1).Return(feed, nil) - - s := &Service{storage: stor} - - m, err := s.GetMetadata(feed.HashID) - require.NoError(t, err) - require.EqualValues(t, 0, m.Downloads) -} diff --git a/pkg/feeds/id_gen.go b/pkg/feeds/id_gen.go deleted file mode 100644 index e643231..0000000 --- a/pkg/feeds/id_gen.go +++ /dev/null @@ -1,24 +0,0 @@ -package feeds - -import ( - "time" - - shortid "github.com/ventu-io/go-shortid" -) - -type IDGen struct { - sid *shortid.Shortid -} - -func NewIDGen() (IDGen, error) { - sid, err := shortid.New(1, shortid.DefaultABC, uint64(time.Now().UnixNano())) - if err != nil { - return IDGen{}, err - } - - return IDGen{sid}, nil -} - -func (id IDGen) Generate() (string, error) { - return id.sid.Generate() -} diff --git a/pkg/link/url.go b/pkg/link/url.go index c7adf10..9a2c3ac 100644 --- a/pkg/link/url.go +++ b/pkg/link/url.go @@ -8,13 +8,8 @@ import ( ) func Parse(link string) (Info, error) { - if !strings.HasPrefix(link, "http") { - link = "https://" + link - } - - parsed, err := url.Parse(link) + parsed, err := parseURL(link) if err != nil { - err = errors.Wrapf(err, "failed to parse url: %s", link) return Info{}, err } @@ -49,6 +44,19 @@ func Parse(link string) (Info, error) { return Info{}, errors.New("unsupported URL host") } +func parseURL(link string) (*url.URL, error) { + if !strings.HasPrefix(link, "http") { + link = "https://" + link + } + + parsed, err := url.Parse(link) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse url: %s", link) + } + + return parsed, nil +} + func parseYoutubeURL(parsed *url.URL) (Type, string, error) { path := parsed.EscapedPath() diff --git a/pkg/builder/feed.go b/pkg/model/feed.go similarity index 57% rename from pkg/builder/feed.go rename to pkg/model/feed.go index 8534629..756d944 100644 --- a/pkg/builder/feed.go +++ b/pkg/model/feed.go @@ -1,4 +1,4 @@ -package builder +package model import ( "time" @@ -6,7 +6,23 @@ import ( "github.com/mxpv/podsync/pkg/link" ) -type Item struct { +// Quality to use when downloading episodes +type Quality string + +const ( + QualityHigh = Quality("high") + QualityLow = Quality("low") +) + +// Format to convert episode when downloading episodes +type Format string + +const ( + FormatAudio = Format("audio") + FormatVideo = Format("video") +) + +type Episode struct { // ID of episode ID string Title string @@ -27,6 +43,9 @@ type Feed struct { CreatedAt time.Time LastAccess time.Time ExpirationTime time.Time + Format Format + Quality Quality + PageSize int CoverArt string Explicit bool Language string // ISO 639 @@ -34,7 +53,7 @@ type Feed struct { Description string PubDate time.Time Author string - ItemURL string // Platform specific URL - Episodes []*Item // Array of episodes, serialized as gziped EpisodesData in DynamoDB + ItemURL string // Platform specific URL + Episodes []*Episode // Array of episodes, serialized as gziped EpisodesData in DynamoDB UpdatedAt time.Time } diff --git a/pkg/model/model.go b/pkg/model/model.go deleted file mode 100644 index 59e0676..0000000 --- a/pkg/model/model.go +++ /dev/null @@ -1,61 +0,0 @@ -package model - -import ( - "time" - - "github.com/mxpv/podsync/pkg/api" -) - -//noinspection SpellCheckingInspection -type Pledge struct { - PledgeID int64 `sql:",pk"` - PatronID int64 - CreatedAt time.Time `dynamodbav:",unixtime"` - DeclinedSince time.Time `dynamodbav:",unixtime"` - AmountCents int - TotalHistoricalAmountCents int - OutstandingPaymentAmountCents int - IsPaused bool -} - -type Item struct { - ID string - Title string - Description string - Thumbnail string - Duration int64 - VideoURL string - PubDate Timestamp `dynamodbav:",unixtime"` - Size int64 - - Order string `dynamodbav:"-"` -} - -//noinspection SpellCheckingInspection -type Feed struct { - FeedID int64 `sql:",pk" dynamodbav:"-"` - HashID string // Short human readable feed id for users - UserID string // Patreon user id - ItemID string - LinkType api.LinkType // Either group, channel or user - Provider api.Provider // Youtube or Vimeo - PageSize int // The number of episodes to return - Format api.Format - Quality api.Quality - FeatureLevel int - CreatedAt time.Time `dynamodbav:",unixtime"` - LastAccess time.Time `dynamodbav:",unixtime"` - ExpirationTime time.Time `sql:"-" dynamodbav:",unixtime"` - CoverArt string `dynamodbav:",omitempty"` - Explicit bool - Language string `dynamodbav:",omitempty"` // ISO 639 - Title string - Description string - PubDate time.Time `dynamodbav:",unixtime"` - Author string - ItemURL string // Platform specific URL - LastID string // Last seen video URL ID (for incremental updater) - UpdatedAt time.Time `dynamodbav:",unixtime"` - Episodes []*Item `dynamodbav:"-"` // Array of episodes, serialized as gziped EpisodesData in DynamoDB - EpisodesData []byte -} diff --git a/pkg/model/timestamp.go b/pkg/model/timestamp.go deleted file mode 100644 index 9ab76e9..0000000 --- a/pkg/model/timestamp.go +++ /dev/null @@ -1,50 +0,0 @@ -package model - -import ( - "fmt" - "strconv" - "time" - - "github.com/vmihailenco/msgpack" -) - -type Timestamp time.Time - -func (t Timestamp) MarshalJSON() ([]byte, error) { - ts := time.Time(t).Unix() - stamp := fmt.Sprint(ts) - return []byte(stamp), nil -} - -func (t *Timestamp) UnmarshalJSON(b []byte) error { - ts, err := strconv.Atoi(string(b)) - if err != nil { - return err - } - - *t = Timestamp(time.Unix(int64(ts), 0)) - return nil -} - -func (t Timestamp) EncodeMsgpack(enc *msgpack.Encoder) error { - ts := time.Time(t).Unix() - stamp := fmt.Sprint(ts) - return enc.EncodeString(stamp) -} - -func (t *Timestamp) DecodeMsgpack(dec *msgpack.Decoder) error { - str, err := dec.DecodeString() - if err != nil { - // TODO: old cache will fail here :( - *t = Timestamp{} - return nil - } - - ts, err := strconv.Atoi(str) - if err != nil { - return err - } - - *t = Timestamp(time.Unix(int64(ts), 0)) - return nil -} diff --git a/pkg/model/timestamp_test.go b/pkg/model/timestamp_test.go deleted file mode 100644 index c6f16b9..0000000 --- a/pkg/model/timestamp_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package model - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestTimestamp_MarshalJSON(t *testing.T) { - time1 := Timestamp(time.Now()) - - data, err := time1.MarshalJSON() - assert.NoError(t, err) - - time2 := Timestamp{} - - err = time2.UnmarshalJSON(data) - assert.NoError(t, err) - - assert.EqualValues(t, time.Time(time1).Unix(), time.Time(time2).Unix()) -}