mirror/mxpv-podsync
1
0
Fork 0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00
Code Issues Releases Activity

Implement feed cache verification

Browse Source
This commit is contained in:
Maksym Pavlenko
2019-03-31 20:59:24 -07:00
parent 16010dcd63
commit 344981f903
10 changed files with 284 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -17,7 +17,6 @@ require (
github.com/go-pg/pg v6.14.2+incompatible
github.com/go-redis/redis v6.15.2+incompatible
github.com/golang/mock v1.2.0
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
github.com/gorilla/sessions v1.1.1 // indirect
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce // indirect
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
@ -43,11 +42,11 @@ require (
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf // indirect
github.com/ugorji/go v1.1.1 // indirect
github.com/ventu-io/go-shortid v0.0.0-20171029131806-771a37caa5cf
github.com/vmihailenco/msgpack v4.0.4+incompatible
golang.org/x/net v0.0.0-20190311183353-d8887717615a
golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect
golang.org/x/sys v0.0.0-20190311152110-c8c8c57fd1e1 // indirect
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5 // indirect
google.golang.org/api v0.0.0-20180718221112-efcb5f25ac56
google.golang.org/appengine v1.1.0 // indirect
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect

10
go.sum
View File

@ -32,14 +32,10 @@ github.com/go-pg/pg v6.14.2+incompatible h1:FrOgsHDUhC3V3wkBGAIN5LVj4nJczFPyy1YN
github.com/go-pg/pg v6.14.2+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA=
github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
@ -83,8 +79,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c h1:KhHx/Ta3c9C1gcSo5UhDeo/D4JnhnxJTrlcOEOFiMfY=
github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c/go.mod h1:10FeaKUMy5t3KLsYfy54dFrq0rpwcfyKkKcF7vRGIRY=
github.com/silentsokolov/go-vimeo v1.2.0 h1:Xp3Vn8ekcE+b5ExJOrwEytkDd5h6AdfH31NPERAuTGI=
github.com/silentsokolov/go-vimeo v1.2.0/go.mod h1:10FeaKUMy5t3KLsYfy54dFrq0rpwcfyKkKcF7vRGIRY=
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I=
@ -106,6 +100,8 @@ github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w=
github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/ventu-io/go-shortid v0.0.0-20171029131806-771a37caa5cf h1:cgAKVljim9RJRcJNGjnBUajXj1FupBSdWwW4JaQG7vk=
github.com/ventu-io/go-shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:6rZqAOk/eYX5FJyjQJ6Z3RBSN389IXX2ijwW4FcggaM=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@ -127,8 +123,6 @@ golang.org/x/sys v0.0.0-20190311152110-c8c8c57fd1e1 h1:FQNj2xvjQ1lgFyzbSybGZr792
golang.org/x/sys v0.0.0-20190311152110-c8c8c57fd1e1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5 h1:ZcPpqKMdoZeNQ/4GHlyY4COf8n8SmpPv6mcqF1+VPSM=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
google.golang.org/api v0.0.0-20180718221112-efcb5f25ac56 h1:iDRbkenn0VZEo05mHiCtN6/EfbZj7x1Rg+tPjB5HiQc=
google.golang.org/api v0.0.0-20180718221112-efcb5f25ac56/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=

4
pkg/builders/common.go
View File

@ -24,7 +24,3 @@ func makeEnclosure(feed *model.Feed, id string, lengthInBytes int64) (string, it
url := fmt.Sprintf("http://podsync.net/download/%s/%s.%s", feed.HashID, id, ext)
return url, contentType, lengthInBytes
}
type VideoCounter interface {
GetVideoCount(feed *model.Feed) (uint64, error)
}

4
pkg/builders/vimeo.go
View File

@ -182,6 +182,10 @@ func (v *VimeoBuilder) Build(feed *model.Feed) (podcast *itunes.Podcast, err err
return
}
func (v *VimeoBuilder) GetVideoCount(feed *model.Feed) (uint64, error) {
return 0, errors.New("not supported")
}
func NewVimeoBuilder(ctx context.Context, token string) (*VimeoBuilder, error) {
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
tc := oauth2.NewClient(ctx, ts)

2
pkg/builders/youtube.go
View File

@ -25,8 +25,6 @@ const (
highAudioBytesPerSecond = 128000 / 8
)
var _ VideoCounter = (*YouTubeBuilder)(nil)
type apiKey string
func (key apiKey) Get() (string, string) {

29
pkg/cache/redis.go vendored
View File

@ -5,6 +5,7 @@ import (
"github.com/go-redis/redis"
"github.com/pkg/errors"
"github.com/vmihailenco/msgpack"
)
var ErrNotFound = errors.New("not found")
@ -41,6 +42,34 @@ func (c RedisCache) Get(key string) (string, error) {
}
}
func (c RedisCache) SaveItem(key string, item interface{}, exp time.Duration) error {
data, err := msgpack.Marshal(item)
if err != nil {
return err
}
return c.client.Set(key, data, exp).Err()
}
func (c RedisCache) GetItem(key string, item interface{}) error {
data, err := c.client.Get(key).Bytes()
if err == redis.Nil {
return ErrNotFound
} else if err != nil {
return err
}
if err := msgpack.Unmarshal(data, item); err != nil {
return err
}
return nil
}
func (c RedisCache) Invalidate(key... string) error {
return c.client.Del(key...).Err()
}
func (c RedisCache) Close() error {
return c.client.Close()
}

108
pkg/feeds/feeds.go
View File

@ -2,32 +2,38 @@ package feeds
import (
"fmt"
"log"
"time"
itunes "github.com/mxpv/podcast"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/mxpv/podsync/pkg/api"
"github.com/mxpv/podsync/pkg/model"
)
const feedCacheTTL = 15 * time.Minute
type CacheItem struct {
Feed []byte `msgpack:"feed"`
UpdatedAt time.Time `msgpack:"updated_at"`
ItemCount uint64 `msgpack:"item_count"`
}
type Builder interface {
Build(feed *model.Feed) (podcast *itunes.Podcast, err error)
GetVideoCount(feed *model.Feed) (uint64, error)
}
type storage interface {
SaveFeed(feed *model.Feed) error
GetFeed(hashID string) (*model.Feed, error)
GetMetadata(hashID string) (*model.Feed, error)
Downgrade(userID string, featureLevel int) error
Downgrade(userID string, featureLevel int) ([]string, error)
}
type cacheService interface {
Set(key, value string, ttl time.Duration) error
Get(key string) (string, error)
SaveItem(key string, item interface{}, exp time.Duration) error
GetItem(key string, item interface{}) error
Invalidate(key ...string) error
}
type Service struct {
@ -102,11 +108,39 @@ func (s Service) QueryFeed(hashID string) (*model.Feed, error) {
return s.db.GetFeed(hashID)
}
func (s Service) getVideoCount(feed *model.Feed, builder Builder) (uint64, bool) {
videoCount, err := builder.GetVideoCount(feed)
if err != nil {
return 0, false
}
return videoCount, true
}
func (s Service) BuildFeed(hashID string) ([]byte, error) {
const (
feedRecordTTL = 15 * 24 * time.Hour
cacheRecheckTTL = 10 * time.Minute
)
var (
cached CacheItem
now = time.Now().UTC()
verifyCache bool
)
// Check cached version first
cached, err := s.cache.Get(hashID)
err := s.cache.GetItem(hashID, &cached)
if err == nil {
return []byte(cached), nil
// We've succeded to retrieve data from Redis, check if it's up to date
// 1. If cached less than 15 minutes ago, just return data
if now.Sub(cached.UpdatedAt) < cacheRecheckTTL {
return cached.Feed, nil
}
// 2. Verify cache integrity by querying the number of episodes from YouTube
verifyCache = true
}
// Query feed metadata
@ -121,22 +155,55 @@ func (s Service) BuildFeed(hashID string) ([]byte, error) {
return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashID)
}
// Check if cached version is still valid
if verifyCache {
log.Debugf("pulling the number of videos from %q", feed.Provider)
// Query YouTube and check the number of videos.
// Most likely it'll remain the same, so we can return previously cached feed.
count, ok := s.getVideoCount(feed, builder)
if ok {
if count == cached.ItemCount {
// Cache is up to date, renew and save
cached.UpdatedAt = now
if s.cache.SaveItem(hashID, &cached, feedRecordTTL) != nil {
return nil, errors.Wrap(err, "failed to cache item")
}
return cached.Feed, nil
}
log.Debugf("the number of episodes is different (%d != %d)", cached.ItemCount, count)
cached.ItemCount = count
}
}
// Rebuild feed using YouTube API
podcast, err := builder.Build(feed)
if err != nil {
log.WithError(err).WithField("feed_id", hashID).Error("failed to build cache")
return nil, err
}
data := podcast.String()
data := []byte(podcast.String())
// Save to cache
if err := s.cache.Set(hashID, data, feedCacheTTL); err != nil {
log.Printf("failed to cache feed %q: %+v", hashID, err)
cached.Feed = data
cached.UpdatedAt = now
if !verifyCache {
cached.ItemCount, _ = s.getVideoCount(feed, builder)
}
return []byte(data), nil
if err := s.cache.SaveItem(hashID, cached, feedRecordTTL); err != nil {
log.WithError(err).Warnf("failed to save new feed %q to cache", hashID)
}
return data, nil
}
func (s Service) GetMetadata(hashID string) (*api.Metadata, error) {
@ -153,14 +220,25 @@ func (s Service) GetMetadata(hashID string) (*api.Metadata, error) {
}
func (s Service) Downgrade(patronID string, featureLevel int) error {
log.Printf("Downgrading patron '%s' to feature level %d", patronID, featureLevel)
logger := log.WithFields(log.Fields{
"user_id": patronID,
"level": featureLevel,
})
if err := s.db.Downgrade(patronID, featureLevel); err != nil {
log.Printf("! downgrade failed")
logger.Info("downgrading patron")
ids, err := s.db.Downgrade(patronID, featureLevel)
if err != nil {
logger.WithError(err).Error("database error while downgrading patron")
return err
}
log.Printf("updated user '%s' to feature level %d", patronID, featureLevel)
if s.cache.Invalidate(ids...) != nil {
logger.WithError(err).Error("failed to invalidate cached feeds")
return err
}
logger.Info("successfully updated user")
return nil
}

69
pkg/feeds/feeds_mock_test.go
View File

@ -50,6 +50,21 @@ func (mr *MockBuilderMockRecorder) Build(feed interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockBuilder)(nil).Build), feed)
}
// GetVideoCount mocks base method
func (m *MockBuilder) GetVideoCount(feed *model.Feed) (uint64, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetVideoCount", feed)
ret0, _ := ret[0].(uint64)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetVideoCount indicates an expected call of GetVideoCount
func (mr *MockBuilderMockRecorder) GetVideoCount(feed interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVideoCount", reflect.TypeOf((*MockBuilder)(nil).GetVideoCount), feed)
}
// Mockstorage is a mock of storage interface
type Mockstorage struct {
ctrl *gomock.Controller
@ -118,11 +133,12 @@ func (mr *MockstorageMockRecorder) GetMetadata(hashID interface{}) *gomock.Call
}
// Downgrade mocks base method
func (m *Mockstorage) Downgrade(userID string, featureLevel int) error {
func (m *Mockstorage) Downgrade(userID string, featureLevel int) ([]string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Downgrade", userID, featureLevel)
ret0, _ := ret[0].(error)
return ret0
ret0, _ := ret[0].([]string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Downgrade indicates an expected call of Downgrade
@ -154,31 +170,48 @@ func (m *MockcacheService) EXPECT() *MockcacheServiceMockRecorder {
return m.recorder
}
// Set mocks base method
func (m *MockcacheService) Set(key, value string, ttl time.Duration) error {
// SaveItem mocks base method
func (m *MockcacheService) SaveItem(key string, item interface{}, exp time.Duration) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Set", key, value, ttl)
ret := m.ctrl.Call(m, "SaveItem", key, item, exp)
ret0, _ := ret[0].(error)
return ret0
}
// Set indicates an expected call of Set
func (mr *MockcacheServiceMockRecorder) Set(key, value, ttl interface{}) *gomock.Call {
// SaveItem indicates an expected call of SaveItem
func (mr *MockcacheServiceMockRecorder) SaveItem(key, item, exp interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockcacheService)(nil).Set), key, value, ttl)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveItem", reflect.TypeOf((*MockcacheService)(nil).SaveItem), key, item, exp)
}
// Get mocks base method
func (m *MockcacheService) Get(key string) (string, error) {
// GetItem mocks base method
func (m *MockcacheService) GetItem(key string, item interface{}) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Get", key)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
ret := m.ctrl.Call(m, "GetItem", key, item)
ret0, _ := ret[0].(error)
return ret0
}
// Get indicates an expected call of Get
func (mr *MockcacheServiceMockRecorder) Get(key interface{}) *gomock.Call {
// GetItem indicates an expected call of GetItem
func (mr *MockcacheServiceMockRecorder) GetItem(key, item interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockcacheService)(nil).Get), key)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetItem", reflect.TypeOf((*MockcacheService)(nil).GetItem), key, item)
}
// Invalidate mocks base method
func (m *MockcacheService) Invalidate(key ...string) error {
m.ctrl.T.Helper()
varargs := []interface{}{}
for _, a := range key {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "Invalidate", varargs...)
ret0, _ := ret[0].(error)
return ret0
}
// Invalidate indicates an expected call of Invalidate
func (mr *MockcacheServiceMockRecorder) Invalidate(key ...interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Invalidate", reflect.TypeOf((*MockcacheService)(nil).Invalidate), key...)
}

93
pkg/feeds/feeds_test.go
View File

@ -4,9 +4,11 @@ package feeds
import (
"testing"
"time"
"github.com/golang/mock/gomock"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
itunes "github.com/mxpv/podcast"
@ -97,7 +99,58 @@ func TestService_QueryFeed(t *testing.T) {
require.NoError(t, err)
}
func TestService_GetFeed(t *testing.T) {
func TestService_GetFromCache(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
item := CacheItem{
UpdatedAt: time.Now().UTC(),
Feed: []byte("test"),
}
cache := NewMockcacheService(ctrl)
cache.EXPECT().GetItem("123", gomock.Any()).DoAndReturn(func(_ string, ret *CacheItem) error {
*ret = item
return nil
})
s := Service{cache: cache}
data, err := s.BuildFeed("123")
assert.NoError(t, err)
assert.Equal(t, item.Feed, data)
}
func TestService_VerifyCache(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cache := NewMockcacheService(ctrl)
cache.EXPECT().GetItem("123", gomock.Any()).DoAndReturn(func(_ string, ret *CacheItem) error {
ret.Feed = []byte("test")
ret.UpdatedAt = time.Now().UTC().Add(-20 * time.Minute)
ret.ItemCount = 30
return nil
})
cache.EXPECT().SaveItem("123", gomock.Any(), 15*24*time.Hour).Times(1).Return(nil)
stor := NewMockstorage(ctrl)
stor.EXPECT().GetFeed(feed.HashID).Times(1).Return(feed, nil)
builder := NewMockBuilder(ctrl)
builder.EXPECT().GetVideoCount(feed).Return(uint64(30), nil)
s := Service{db: stor, cache: cache, builders: map[api.Provider]Builder{
api.ProviderVimeo: builder,
}}
data, err := s.BuildFeed(feed.HashID)
assert.NoError(t, err)
assert.Equal(t, []byte("test"), data)
}
func TestService_BuildFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@ -105,13 +158,45 @@ func TestService_GetFeed(t *testing.T) {
stor.EXPECT().GetFeed(feed.HashID).Times(1).Return(feed, nil)
cache := NewMockcacheService(ctrl)
cache.EXPECT().Get(feed.HashID).Return("", errors.New("not found"))
cache.EXPECT().Set(feed.HashID, gomock.Any(), gomock.Any()).Return(nil)
cache.EXPECT().GetItem(feed.HashID, gomock.Any()).Return(errors.New("not found"))
cache.EXPECT().SaveItem(feed.HashID, gomock.Any(), gomock.Any()).Return(nil)
podcast := itunes.New("", "", "", nil, nil)
builder := NewMockBuilder(ctrl)
builder.EXPECT().Build(feed).Return(&podcast, nil)
builder.EXPECT().GetVideoCount(feed).Return(uint64(25), nil)
s := Service{db: stor, cache: cache, builders: map[api.Provider]Builder{
api.ProviderVimeo: builder,
}}
_, err := s.BuildFeed(feed.HashID)
require.NoError(t, err)
}
func TestService_RebuildCache(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
stor := NewMockstorage(ctrl)
stor.EXPECT().GetFeed(feed.HashID).Times(1).Return(feed, nil)
cache := NewMockcacheService(ctrl)
cache.EXPECT().GetItem("123", gomock.Any()).DoAndReturn(func(_ string, ret *CacheItem) error {
ret.Feed = []byte("test")
ret.UpdatedAt = time.Now().UTC().Add(-20 * time.Minute)
ret.ItemCount = 30
return nil
})
cache.EXPECT().SaveItem(feed.HashID, gomock.Any(), gomock.Any()).Return(nil)
podcast := itunes.New("", "", "", nil, nil)
builder := NewMockBuilder(ctrl)
builder.EXPECT().Build(feed).Return(&podcast, nil)
builder.EXPECT().GetVideoCount(feed).Return(uint64(25), nil)
s := Service{db: stor, cache: cache, builders: map[api.Provider]Builder{
api.ProviderVimeo: builder,
@ -126,7 +211,7 @@ func TestService_WrongID(t *testing.T) {
defer ctrl.Finish()
cache := NewMockcacheService(ctrl)
cache.EXPECT().Get(gomock.Any()).Return("", errors.New("not found"))
cache.EXPECT().GetItem(gomock.Any(), gomock.Any()).Return(errors.New("not found"))
stor := NewMockstorage(ctrl)
stor.EXPECT().GetFeed(gomock.Any()).Times(1).Return(nil, errors.New("not found"))

25
pkg/storage/dynamo.go
View File

@ -241,7 +241,7 @@ func (d Dynamo) GetMetadata(hashID string) (*model.Feed, error) {
return &feed, nil
}
func (d Dynamo) Downgrade(userID string, featureLevel int) error {
func (d Dynamo) Downgrade(userID string, featureLevel int) ([]string, error) {
logger := log.WithFields(log.Fields{
"user_id": userID,
"feature_level": featureLevel,
@ -253,7 +253,7 @@ func (d Dynamo) Downgrade(userID string, featureLevel int) error {
// Max page size: 600
// Format: any
// Quality: any
return nil
return []string{}, nil
}
keyConditionExpression, err := expr.
@ -263,7 +263,7 @@ func (d Dynamo) Downgrade(userID string, featureLevel int) error {
if err != nil {
logger.WithError(err).Error("failed to build key condition")
return err
return nil, err
}
// Query all feed's hash ids for specified
@ -292,12 +292,12 @@ func (d Dynamo) Downgrade(userID string, featureLevel int) error {
if err != nil {
logger.WithError(err).Error("query failed")
return err
return nil, err
}
logger.Debugf("got %d key(s)", len(keys))
if len(keys) == 0 {
return nil
return []string{}, nil
}
if featureLevel == api.ExtendedFeatures {
@ -315,7 +315,7 @@ func (d Dynamo) Downgrade(userID string, featureLevel int) error {
if err != nil {
logger.WithError(err).Error("failed to build update expression")
return err
return nil, err
}
for _, key := range keys {
@ -331,7 +331,7 @@ func (d Dynamo) Downgrade(userID string, featureLevel int) error {
_, err := d.dynamo.UpdateItem(input)
if err != nil {
logger.WithError(err).Error("failed to update item")
return err
return nil, err
}
}
@ -349,7 +349,7 @@ func (d Dynamo) Downgrade(userID string, featureLevel int) error {
Build()
if err != nil {
return err
return nil, err
}
for _, key := range keys {
@ -364,13 +364,18 @@ func (d Dynamo) Downgrade(userID string, featureLevel int) error {
_, err := d.dynamo.UpdateItem(input)
if err != nil {
logger.WithError(err).Error("failed to update item")
return err
return nil, err
}
}
}
hashIDs := make([]string, len(keys))
for i, key := range keys {
hashIDs[i] = *key[feedsPrimaryKey].S
}
logger.Info("successfully downgraded user's feeds")
return nil
return hashIDs, nil
}
func (d Dynamo) AddPledge(pledge *model.Pledge) error {