mirror/mxpv-podsync
1
0
Fork 0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00
Code Issues Releases Activity

Implement 600 episodes feeds

Browse Source
This commit is contained in:
Maksym Pavlenko
2017-11-10 17:13:01 -08:00
parent 30214a65c3
commit 05a1b0f8d5
9 changed files with 230 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
assets/js/site.js
View File

@@ -84,6 +84,10 @@
} catch (e) {
return false;
}
},
allow600pages: function() {
return !this.locked && this.featureLevel >= 2;
}
},

13
pkg/api/api.go
View File

@@ -5,7 +5,8 @@ import (
)
var (
ErrNotFound = errors.New("resource not found")
ErrNotFound = errors.New("resource not found")
ErrQuotaExceeded = errors.New("query limit is exceeded")
)
type Provider string
@@ -39,9 +40,10 @@ const (
)
const (
DefaultPageSize = 50
DefaultFormat = FormatVideo
DefaultQuality = QualityHigh
DefaultPageSize = 50
DefaultFormat = FormatVideo
DefaultQuality = QualityHigh
ExtendedPaginationQueryLimit = 5000
)
type Metadata struct {
@@ -54,12 +56,13 @@ type Metadata struct {
const (
DefaultFeatures = iota
ExtendedFeatures
ExtendedPagination
PodcasterFeature
)
type CreateFeedRequest struct {
URL string `json:"url" binding:"url,required"`
PageSize int `json:"page_size" binding:"min=10,max=150,required"`
PageSize int `json:"page_size" binding:"min=10,max=600,required"`
Quality Quality `json:"quality" binding:"eq=high|eq=low"`
Format Format `json:"format" binding:"eq=video|eq=audio"`
}

130
pkg/feeds/feeds.go
View File

@@ -13,10 +13,6 @@ import (
"github.com/ventu-io/go-shortid"
)
const (
maxPageSize = 150
)
const (
MetricQueries = "queries"
MetricDownloads = "downloads"
@@ -38,10 +34,51 @@ type Service struct {
builders map[api.Provider]builder
}
func (s Service) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) {
func (s Service) makeFeed(req *api.CreateFeedRequest, identity *api.Identity) (*model.Feed, error) {
feed, err := parseURL(req.URL)
if err != nil {
return "", errors.Wrapf(err, "failed to create feed for URL: %s", req.URL)
return nil, errors.Wrapf(err, "failed to create feed for URL: %s", req.URL)
}
now := time.Now().UTC()
feed.UserID = identity.UserId
feed.FeatureLevel = identity.FeatureLevel
feed.Quality = req.Quality
feed.Format = req.Format
feed.PageSize = req.PageSize
feed.CreatedAt = now
feed.LastAccess = now
if identity.FeatureLevel == api.ExtendedPagination {
if feed.PageSize > 600 {
feed.PageSize = 600
}
} else if identity.FeatureLevel == api.ExtendedFeatures {
if feed.PageSize > 150 {
feed.PageSize = 150
}
} else {
feed.Quality = api.QualityHigh
feed.Format = api.FormatVideo
feed.PageSize = 50
}
// Generate short id
hashId, err := s.sid.Generate()
if err != nil {
return nil, errors.Wrap(err, "failed to generate id for feed")
}
feed.HashID = hashId
return feed, nil
}
func (s Service) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) {
feed, err := s.makeFeed(req, identity)
if err != nil {
return "", err
}
// Make sure builder exists for this provider
@@ -50,42 +87,13 @@ func (s Service) CreateFeed(req *api.CreateFeedRequest, identity *api.Identity)
return "", fmt.Errorf("failed to get builder for URL: %s", req.URL)
}
now := time.Now().UTC()
// Set default fields
feed.PageSize = api.DefaultPageSize
feed.Format = api.FormatVideo
feed.Quality = api.QualityHigh
feed.FeatureLevel = api.DefaultFeatures
feed.CreatedAt = now
feed.LastAccess = now
if identity.FeatureLevel > 0 {
feed.UserID = identity.UserId
feed.Quality = req.Quality
feed.Format = req.Format
feed.FeatureLevel = identity.FeatureLevel
feed.PageSize = req.PageSize
if feed.PageSize > maxPageSize {
feed.PageSize = maxPageSize
}
}
// Generate short id
hashId, err := s.sid.Generate()
if err != nil {
return "", errors.Wrap(err, "failed to generate id for feed")
}
feed.HashID = hashId
// Save to database
_, err = s.db.Model(feed).Insert()
if err != nil {
return "", errors.Wrap(err, "failed to save feed to database")
}
return hashId, nil
return feed.HashID, nil
}
func (s Service) QueryFeed(hashID string) (*model.Feed, error) {
@@ -115,6 +123,15 @@ func (s Service) BuildFeed(hashID string) (*itunes.Podcast, error) {
return nil, err
}
count, err := s.stats.Inc(MetricQueries, feed.HashID)
if err != nil {
return nil, errors.Wrapf(err, "failed to update metrics for feed: %s", hashID)
}
if feed.PageSize > 150 && count > api.ExtendedPaginationQueryLimit {
return nil, api.ErrQuotaExceeded
}
builder, ok := s.builders[feed.Provider]
if !ok {
return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashID)
@@ -125,11 +142,6 @@ func (s Service) BuildFeed(hashID string) (*itunes.Podcast, error) {
return nil, err
}
_, err = s.stats.Inc(MetricQueries, feed.HashID)
if err != nil {
return nil, errors.Wrapf(err, "failed to update metrics for feed: %s", hashID)
}
return podcast, nil
}
@@ -161,18 +173,50 @@ func (s Service) GetMetadata(hashID string) (*api.Metadata, error) {
func (s Service) Downgrade(patronID string, featureLevel int) error {
log.Printf("Downgrading patron '%s' to feature level %d", patronID, featureLevel)
if featureLevel > api.ExtendedFeatures {
return nil
}
if featureLevel == api.ExtendedFeatures {
const maxPages = 150
res, err := s.db.
Model(&model.Feed{}).
Set("page_size = ?", maxPages).
Where("user_id = ? AND page_size > ?", patronID, maxPages).
Update()
if err != nil {
log.Printf("! failed to reduce page sizes for patron '%s': %v", patronID, err)
return err
}
res, err = s.db.
Model(&model.Feed{}).
Set("feature_level = ?", api.ExtendedFeatures).
Where("user_id = ?", patronID, maxPages).
Update()
if err != nil {
log.Printf("! failed to downgrade patron '%s' to feature level %d: %v", patronID, featureLevel, err)
return err
}
log.Printf("Updated %d feed(s) of user '%s' to feature level %d", res.RowsAffected(), patronID, featureLevel)
return nil
}
if featureLevel == api.DefaultFeatures {
res, err := s.db.
Model(&model.Feed{}).
Set("page_size = ?", 50).
Set("feature_level = ?", 0).
Set("feature_level = ?", api.DefaultFeatures).
Set("format = ?", api.FormatVideo).
Set("quality = ?", api.QualityHigh).
Where("user_id = ?", patronID).
Update()
if err != nil {
log.Printf("failed to downgrade patron '%s' to feature level %d: %v", patronID, featureLevel, err)
log.Printf("! failed to downgrade patron '%s' to feature level %d: %v", patronID, featureLevel, err)
return err
}

96
pkg/feeds/feeds_test.go
View File

@@ -45,13 +45,76 @@ func TestService_CreateFeed(t *testing.T) {
require.NotEmpty(t, hashId)
}
func TestService_makeFeed(t *testing.T) {
req := &api.CreateFeedRequest{
URL: "youtube.com/channel/123",
PageSize: 1000,
Quality: api.QualityLow,
Format: api.FormatAudio,
}
s := Service{
sid: shortid.GetDefault(),
}
feed, err := s.makeFeed(req, &api.Identity{})
require.NoError(t, err)
require.Equal(t, 50, feed.PageSize)
require.Equal(t, api.QualityHigh, feed.Quality)
require.Equal(t, api.FormatVideo, feed.Format)
feed, err = s.makeFeed(req, &api.Identity{FeatureLevel: api.ExtendedFeatures})
require.NoError(t, err)
require.Equal(t, 150, feed.PageSize)
require.Equal(t, api.QualityLow, feed.Quality)
require.Equal(t, api.FormatAudio, feed.Format)
feed, err = s.makeFeed(req, &api.Identity{FeatureLevel: api.ExtendedPagination})
require.NoError(t, err)
require.Equal(t, 600, feed.PageSize)
require.Equal(t, api.QualityLow, feed.Quality)
require.Equal(t, api.FormatAudio, feed.Format)
}
func TestService_GetFeed(t *testing.T) {
s := Service{db: createDatabase(t)}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
stats := NewMockstats(ctrl)
stats.EXPECT().Inc(MetricQueries, feed.HashID).Return(int64(10), nil)
s := Service{db: createDatabase(t), stats: stats}
_, err := s.BuildFeed(feed.HashID)
require.NoError(t, err)
}
func TestService_BuildFeedQuotaCheck(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
f := &model.Feed{
HashID: "321",
ItemID: "xyz",
Provider: api.ProviderVimeo,
LinkType: api.LinkTypeChannel,
PageSize: 600,
Quality: api.QualityHigh,
Format: api.FormatVideo,
}
stats := NewMockstats(ctrl)
stats.EXPECT().Inc(MetricQueries, f.HashID).Return(int64(api.ExtendedPaginationQueryLimit)+1, nil)
s := Service{db: createDatabase(t), stats: stats}
err := s.db.Insert(f)
require.NoError(t, err)
_, err = s.BuildFeed(f.HashID)
require.Equal(t, api.ErrQuotaExceeded, err)
}
func TestService_WrongID(t *testing.T) {
s := Service{db: createDatabase(t)}
@@ -119,6 +182,37 @@ func TestService_DowngradeToAnonymous(t *testing.T) {
require.Equal(t, api.DefaultFeatures, downgraded.FeatureLevel)
}
func TestService_DowngradeToExtendedFeatures(t *testing.T) {
s := Service{db: createDatabase(t)}
feed := &model.Feed{
HashID: "123456",
UserID: "123456",
ItemID: "123456",
Provider: api.ProviderVimeo,
LinkType: api.LinkTypeGroup,
PageSize: 500,
Quality: api.QualityLow,
Format: api.FormatAudio,
FeatureLevel: api.ExtendedFeatures,
}
err := s.db.Insert(feed)
require.NoError(t, err)
err = s.Downgrade(feed.UserID, api.ExtendedFeatures)
require.NoError(t, err)
downgraded := &model.Feed{FeedID: feed.FeedID}
err = s.db.Select(downgraded)
require.NoError(t, err)
require.Equal(t, 150, downgraded.PageSize)
require.Equal(t, feed.Quality, downgraded.Quality)
require.Equal(t, feed.Format, downgraded.Format)
require.Equal(t, api.ExtendedFeatures, downgraded.FeatureLevel)
}
func createDatabase(t *testing.T) *pg.DB {
opts, err := pg.ParseURL("postgres://postgres:@localhost/podsync?sslmode=disable")
if err != nil {

21
pkg/handler/handler.go
View File

@@ -31,7 +31,8 @@ type feedService interface {
type patreonService interface {
Hook(pledge *patreon.Pledge, event string) error
GetFeatureLevel(patronID string) int
GetFeatureLevelByID(patronID string) int
GetFeatureLevelFromAmount(amount int) int
}
type handler struct {
@@ -93,7 +94,7 @@ func (h handler) patreonCallback(c *gin.Context) {
}
// Determine feature level
level := h.patreon.GetFeatureLevel(user.Data.ID)
level := h.patreon.GetFeatureLevelByID(user.Data.ID)
identity := &api.Identity{
UserId: user.Data.ID,
@@ -133,7 +134,7 @@ func (h handler) create(c *gin.Context) {
}
// Check feature level again if user deleted pledge by still logged in
identity.FeatureLevel = h.patreon.GetFeatureLevel(identity.UserId)
identity.FeatureLevel = h.patreon.GetFeatureLevelByID(identity.UserId)
hashId, err := h.feed.CreateFeed(req, identity)
if err != nil {
@@ -160,6 +161,8 @@ func (h handler) getFeed(c *gin.Context) {
code := http.StatusInternalServerError
if err == api.ErrNotFound {
code = http.StatusNotFound
} else if err == api.ErrQuotaExceeded {
code = http.StatusTooManyRequests
} else {
log.Printf("server error (hash id: %s): %v", hashId, err)
}
@@ -237,8 +240,16 @@ func (h handler) webhook(c *gin.Context) {
return
}
if eventName == patreon.EventDeletePledge {
if err := h.feed.Downgrade(pledge.Data.Relationships.Patron.Data.ID, api.DefaultFeatures); err != nil {
patronID := pledge.Data.Relationships.Patron.Data.ID
if eventName == patreon.EventUpdatePledge {
newLevel := h.patreon.GetFeatureLevelFromAmount(pledge.Data.Attributes.AmountCents)
if err := h.feed.Downgrade(patronID, newLevel); err != nil {
log.Printf("downgrade failed: %v", err)
return
}
} else if eventName == patreon.EventDeletePledge {
if err := h.feed.Downgrade(patronID, api.DefaultFeatures); err != nil {
log.Printf("downgrade failed: %v", err)
return
}

2
pkg/handler/handler_test.go
View File

@@ -68,7 +68,7 @@ func TestCreateInvalidFeed(t *testing.T) {
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
query = `{"url": "https://youtube.com/channel/123", "page_size": 151, "quality": "low", "format": "audio"}`
query = `{"url": "https://youtube.com/channel/123", "page_size": 1001, "quality": "low", "format": "audio"}`
resp, err = http.Post(srv.URL+"/api/create", "application/json", strings.NewReader(query))
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)

22
pkg/support/patreon.go
View File

@@ -100,7 +100,7 @@ func (h Patreon) FindPledge(patronID string) (*model.Pledge, error) {
return p, h.db.Model(p).Where("patron_id = ?", patronID).Limit(1).Select()
}
func (h Patreon) GetFeatureLevel(patronID string) (level int) {
func (h Patreon) GetFeatureLevelByID(patronID string) (level int) {
level = api.DefaultFeatures
if patronID == "" {
@@ -120,16 +120,26 @@ func (h Patreon) GetFeatureLevel(patronID string) (level int) {
// Check pledge is valid
if pledge.DeclinedSince.IsZero() && !pledge.IsPaused {
// Check the amount of pledge
if pledge.AmountCents >= 100 {
level = api.ExtendedFeatures
return
}
level = h.GetFeatureLevelFromAmount(pledge.AmountCents)
return
}
return
}
func (h Patreon) GetFeatureLevelFromAmount(amount int) int {
// Check the amount of pledge
if amount >= 300 {
return api.ExtendedPagination
}
if amount >= 100 {
return api.ExtendedFeatures
}
return api.DefaultFeatures
}
func NewPatreon(db *pg.DB) *Patreon {
return &Patreon{db: db}
}

2
pkg/support/patreon_test.go
View File

@@ -74,7 +74,7 @@ func TestGetFeatureLevel(t *testing.T) {
require.Equal(t, api.PodcasterFeature, hook.GetFeatureLevel(creatorID))
require.Equal(t, api.DefaultFeatures, hook.GetFeatureLevel("xyz"))
require.Equal(t, api.ExtendedFeatures, hook.GetFeatureLevel(pledge.Relationships.Patron.Data.ID))
require.Equal(t, api.ExtendedPagination, hook.GetFeatureLevel(pledge.Relationships.Patron.Data.ID))
}
func createHandler(t *testing.T) *Patreon {

2
templates/index.html
View File

@@ -107,6 +107,8 @@
<label for="page_100">100</label>
<input type="radio" id="page_150" value="150" name="page_count" v-model.number="count" :disabled="locked" />
<label for="page_150">150</label>
<input type="radio" id="page_600" value="600" name="page_count" v-model.number="count" :disabled="locked || !allow600pages" />
<label for="page_600" :class="{ locked: !allow600pages }">600</label>
</div>
</div>