1
0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00

Refactor caching

This commit is contained in:
Maksym Pavlenko
2019-03-29 19:18:03 -07:00
parent 995120b450
commit 35ad8c3f43
7 changed files with 170 additions and 174 deletions

View File

@@ -8,20 +8,19 @@ import (
"os/signal"
"syscall"
"github.com/mxpv/podsync/pkg/cache"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
log "github.com/sirupsen/logrus"
"github.com/mxpv/podsync/pkg/api"
"github.com/mxpv/podsync/pkg/builders"
"github.com/mxpv/podsync/pkg/cache"
"github.com/mxpv/podsync/pkg/config"
"github.com/mxpv/podsync/pkg/feeds"
"github.com/mxpv/podsync/pkg/handler"
"github.com/mxpv/podsync/pkg/storage"
"github.com/mxpv/podsync/pkg/support"
log "github.com/sirupsen/logrus"
)
func main() {
@@ -78,11 +77,10 @@ func main() {
log.WithError(err).Fatal("failed to create Vimeo builder")
}
feed, err := feeds.NewFeedService(
feeds.WithStorage(database),
feeds.WithBuilder(api.ProviderYoutube, youtube),
feeds.WithBuilder(api.ProviderVimeo, vimeo),
)
feed, err := feeds.NewFeedService(database, redisCache, map[api.Provider]feeds.Builder{
api.ProviderYoutube: youtube,
api.ProviderVimeo: vimeo,
})
if err != nil {
log.WithError(err).Fatal("failed to create feed service")
@@ -90,7 +88,7 @@ func main() {
srv := http.Server{
Addr: fmt.Sprintf(":%d", 5001),
Handler: handler.New(feed, patreon, redisCache, cfg),
Handler: handler.New(feed, patreon, cfg),
}
go func() {

View File

@@ -12,12 +12,9 @@ import (
"github.com/mxpv/podsync/pkg/model"
)
const (
MetricQueries = "queries"
MetricDownloads = "downloads"
)
const feedCacheTTL = 15 * time.Minute
type builder interface {
type Builder interface {
Build(feed *model.Feed) (podcast *itunes.Podcast, err error)
}
@@ -28,10 +25,16 @@ type storage interface {
Downgrade(userID string, featureLevel int) error
}
type cacheService interface {
Set(key, value string, ttl time.Duration) error
Get(key string) (string, error)
}
type Service struct {
generator IDGen
db storage
builders map[api.Provider]builder
builders map[api.Provider]Builder
cache cacheService
}
func (s Service) makeFeed(req *api.CreateFeedRequest, identity *api.Identity) (*model.Feed, error) {
@@ -99,7 +102,15 @@ func (s Service) QueryFeed(hashID string) (*model.Feed, error) {
return s.db.GetFeed(hashID)
}
func (s Service) BuildFeed(hashID string) (*itunes.Podcast, error) {
func (s Service) BuildFeed(hashID string) ([]byte, error) {
// Check cached version first
cached, err := s.cache.Get(hashID)
if err == nil {
return []byte(cached), nil
}
// Query feed metadata
feed, err := s.QueryFeed(hashID)
if err != nil {
return nil, err
@@ -110,12 +121,22 @@ func (s Service) BuildFeed(hashID string) (*itunes.Podcast, error) {
return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashID)
}
// Rebuild feed using YouTube API
podcast, err := builder.Build(feed)
if err != nil {
return nil, err
}
return podcast, nil
data := podcast.String()
// Save to cache
if err := s.cache.Set(hashID, data, feedCacheTTL); err != nil {
log.Printf("failed to cache feed %q: %+v", hashID, err)
}
return []byte(data), nil
}
func (s Service) GetMetadata(hashID string) (*api.Metadata, error) {
@@ -143,23 +164,7 @@ func (s Service) Downgrade(patronID string, featureLevel int) error {
return nil
}
type FeedOption func(*Service)
//noinspection GoExportedFuncWithUnexportedType
func WithStorage(db storage) FeedOption {
return func(service *Service) {
service.db = db
}
}
//noinspection GoExportedFuncWithUnexportedType
func WithBuilder(provider api.Provider, builder builder) FeedOption {
return func(service *Service) {
service.builders[provider] = builder
}
}
func NewFeedService(opts ...FeedOption) (*Service, error) {
func NewFeedService(db storage, cache cacheService, builders map[api.Provider]Builder) (*Service, error) {
idGen, err := NewIDGen()
if err != nil {
return nil, err
@@ -167,11 +172,9 @@ func NewFeedService(opts ...FeedOption) (*Service, error) {
svc := &Service{
generator: idGen,
builders: make(map[api.Provider]builder),
}
for _, fn := range opts {
fn(svc)
db: db,
builders: builders,
cache: cache,
}
return svc, nil

View File

@@ -1,6 +1,7 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: feeds.go
// Package feeds is a generated GoMock package.
package feeds
import (
@@ -8,42 +9,45 @@ import (
podcast "github.com/mxpv/podcast"
model "github.com/mxpv/podsync/pkg/model"
reflect "reflect"
time "time"
)
// Mockbuilder is a mock of builder interface
type Mockbuilder struct {
// MockBuilder is a mock of Builder interface
type MockBuilder struct {
ctrl *gomock.Controller
recorder *MockbuilderMockRecorder
recorder *MockBuilderMockRecorder
}
// MockbuilderMockRecorder is the mock recorder for Mockbuilder
type MockbuilderMockRecorder struct {
mock *Mockbuilder
// MockBuilderMockRecorder is the mock recorder for MockBuilder
type MockBuilderMockRecorder struct {
mock *MockBuilder
}
// NewMockbuilder creates a new mock instance
func NewMockbuilder(ctrl *gomock.Controller) *Mockbuilder {
mock := &Mockbuilder{ctrl: ctrl}
mock.recorder = &MockbuilderMockRecorder{mock}
// NewMockBuilder creates a new mock instance
func NewMockBuilder(ctrl *gomock.Controller) *MockBuilder {
mock := &MockBuilder{ctrl: ctrl}
mock.recorder = &MockBuilderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (_m *Mockbuilder) EXPECT() *MockbuilderMockRecorder {
return _m.recorder
func (m *MockBuilder) EXPECT() *MockBuilderMockRecorder {
return m.recorder
}
// Build mocks base method
func (_m *Mockbuilder) Build(feed *model.Feed) (*podcast.Podcast, error) {
ret := _m.ctrl.Call(_m, "Build", feed)
func (m *MockBuilder) Build(feed *model.Feed) (*podcast.Podcast, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Build", feed)
ret0, _ := ret[0].(*podcast.Podcast)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Build indicates an expected call of Build
func (_mr *MockbuilderMockRecorder) Build(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Build", reflect.TypeOf((*Mockbuilder)(nil).Build), arg0)
func (mr *MockBuilderMockRecorder) Build(feed interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockBuilder)(nil).Build), feed)
}
// Mockstorage is a mock of storage interface
@@ -65,56 +69,116 @@ func NewMockstorage(ctrl *gomock.Controller) *Mockstorage {
}
// EXPECT returns an object that allows the caller to indicate expected use
func (_m *Mockstorage) EXPECT() *MockstorageMockRecorder {
return _m.recorder
func (m *Mockstorage) EXPECT() *MockstorageMockRecorder {
return m.recorder
}
// SaveFeed mocks base method
func (_m *Mockstorage) SaveFeed(feed *model.Feed) error {
ret := _m.ctrl.Call(_m, "SaveFeed", feed)
func (m *Mockstorage) SaveFeed(feed *model.Feed) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SaveFeed", feed)
ret0, _ := ret[0].(error)
return ret0
}
// SaveFeed indicates an expected call of SaveFeed
func (_mr *MockstorageMockRecorder) SaveFeed(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "SaveFeed", reflect.TypeOf((*Mockstorage)(nil).SaveFeed), arg0)
func (mr *MockstorageMockRecorder) SaveFeed(feed interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveFeed", reflect.TypeOf((*Mockstorage)(nil).SaveFeed), feed)
}
// GetFeed mocks base method
func (_m *Mockstorage) GetFeed(hashID string) (*model.Feed, error) {
ret := _m.ctrl.Call(_m, "GetFeed", hashID)
func (m *Mockstorage) GetFeed(hashID string) (*model.Feed, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetFeed", hashID)
ret0, _ := ret[0].(*model.Feed)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetFeed indicates an expected call of GetFeed
func (_mr *MockstorageMockRecorder) GetFeed(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), arg0)
func (mr *MockstorageMockRecorder) GetFeed(hashID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), hashID)
}
// GetMetadata mocks base method
func (_m *Mockstorage) GetMetadata(hashID string) (*model.Feed, error) {
ret := _m.ctrl.Call(_m, "GetMetadata", hashID)
func (m *Mockstorage) GetMetadata(hashID string) (*model.Feed, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetMetadata", hashID)
ret0, _ := ret[0].(*model.Feed)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetMetadata indicates an expected call of GetMetadata
func (_mr *MockstorageMockRecorder) GetMetadata(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetMetadata", reflect.TypeOf((*Mockstorage)(nil).GetMetadata), arg0)
func (mr *MockstorageMockRecorder) GetMetadata(hashID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetadata", reflect.TypeOf((*Mockstorage)(nil).GetMetadata), hashID)
}
// Downgrade mocks base method
func (_m *Mockstorage) Downgrade(userID string, featureLevel int) error {
ret := _m.ctrl.Call(_m, "Downgrade", userID, featureLevel)
func (m *Mockstorage) Downgrade(userID string, featureLevel int) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Downgrade", userID, featureLevel)
ret0, _ := ret[0].(error)
return ret0
}
// Downgrade indicates an expected call of Downgrade
func (_mr *MockstorageMockRecorder) Downgrade(arg0, arg1 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Downgrade", reflect.TypeOf((*Mockstorage)(nil).Downgrade), arg0, arg1)
func (mr *MockstorageMockRecorder) Downgrade(userID, featureLevel interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Downgrade", reflect.TypeOf((*Mockstorage)(nil).Downgrade), userID, featureLevel)
}
// MockcacheService is a mock of cacheService interface
type MockcacheService struct {
ctrl *gomock.Controller
recorder *MockcacheServiceMockRecorder
}
// MockcacheServiceMockRecorder is the mock recorder for MockcacheService
type MockcacheServiceMockRecorder struct {
mock *MockcacheService
}
// NewMockcacheService creates a new mock instance
func NewMockcacheService(ctrl *gomock.Controller) *MockcacheService {
mock := &MockcacheService{ctrl: ctrl}
mock.recorder = &MockcacheServiceMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockcacheService) EXPECT() *MockcacheServiceMockRecorder {
return m.recorder
}
// Set mocks base method
func (m *MockcacheService) Set(key, value string, ttl time.Duration) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Set", key, value, ttl)
ret0, _ := ret[0].(error)
return ret0
}
// Set indicates an expected call of Set
func (mr *MockcacheServiceMockRecorder) Set(key, value, ttl interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockcacheService)(nil).Set), key, value, ttl)
}
// Get mocks base method
func (m *MockcacheService) Get(key string) (string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Get", key)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Get indicates an expected call of Get
func (mr *MockcacheServiceMockRecorder) Get(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockcacheService)(nil).Get), key)
}

View File

@@ -9,6 +9,8 @@ import (
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/podsync/pkg/api"
"github.com/mxpv/podsync/pkg/model"
)
@@ -35,7 +37,7 @@ func TestService_CreateFeed(t *testing.T) {
s := Service{
generator: gen,
db: db,
builders: map[api.Provider]builder{api.ProviderYoutube: nil},
builders: map[api.Provider]Builder{api.ProviderYoutube: nil},
}
req := &api.CreateFeedRequest{
@@ -102,7 +104,18 @@ func TestService_GetFeed(t *testing.T) {
stor := NewMockstorage(ctrl)
stor.EXPECT().GetFeed(feed.HashID).Times(1).Return(feed, nil)
s := Service{db: stor}
cache := NewMockcacheService(ctrl)
cache.EXPECT().Get(feed.HashID).Return("", errors.New("not found"))
cache.EXPECT().Set(feed.HashID, gomock.Any(), gomock.Any()).Return(nil)
podcast := itunes.New("", "", "", nil, nil)
builder := NewMockBuilder(ctrl)
builder.EXPECT().Build(feed).Return(&podcast, nil)
s := Service{db: stor, cache: cache, builders: map[api.Provider]Builder{
api.ProviderVimeo: builder,
}}
_, err := s.BuildFeed(feed.HashID)
require.NoError(t, err)
@@ -112,10 +125,13 @@ func TestService_WrongID(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cache := NewMockcacheService(ctrl)
cache.EXPECT().Get(gomock.Any()).Return("", errors.New("not found"))
stor := NewMockstorage(ctrl)
stor.EXPECT().GetFeed(gomock.Any()).Times(1).Return(nil, errors.New("not found"))
s := Service{db: stor}
s := Service{db: stor, cache: cache}
_, err := s.BuildFeed("invalid_feed_id")
require.Error(t, err)

View File

@@ -5,12 +5,10 @@ import (
"io/ioutil"
"net/http"
"strings"
"time"
"github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin"
patreon "github.com/mxpv/patreon-go"
itunes "github.com/mxpv/podcast"
"github.com/mxpv/patreon-go"
"golang.org/x/oauth2"
"github.com/mxpv/podsync/pkg/api"
@@ -26,7 +24,7 @@ const (
type feedService interface {
CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error)
BuildFeed(hashID string) (*itunes.Podcast, error)
BuildFeed(hashID string) ([]byte, error)
GetMetadata(hashID string) (*api.Metadata, error)
Downgrade(patronID string, featureLevel int) error
}
@@ -37,20 +35,14 @@ type patreonService interface {
GetFeatureLevelFromAmount(amount int) int
}
type cacheService interface {
Set(key, value string, ttl time.Duration) error
Get(key string) (string, error)
}
type handler struct {
feed feedService
cfg *config.AppConfig
oauth2 oauth2.Config
patreon patreonService
cache cacheService
}
func New(feed feedService, support patreonService, cache cacheService, cfg *config.AppConfig) http.Handler {
func New(feed feedService, support patreonService, cfg *config.AppConfig) http.Handler {
r := gin.New()
r.Use(gin.Recovery())
@@ -60,7 +52,6 @@ func New(feed feedService, support patreonService, cache cacheService, cfg *conf
h := handler{
feed: feed,
patreon: support,
cache: cache,
cfg: cfg,
}
@@ -222,14 +213,6 @@ func (h handler) getFeed(c *gin.Context) {
hashID = strings.TrimSuffix(hashID, ".xml")
}
const feedContentType = "application/rss+xml; charset=UTF-8"
cached, err := h.cache.Get(hashID)
if err == nil {
c.Data(http.StatusOK, feedContentType, []byte(cached))
return
}
podcast, err := h.feed.BuildFeed(hashID)
if err != nil {
code := http.StatusInternalServerError
@@ -248,13 +231,8 @@ func (h handler) getFeed(c *gin.Context) {
return
}
data := podcast.String()
if err := h.cache.Set(hashID, data, 10*time.Minute); err != nil {
log.WithError(err).Warnf("failed to cache feed %q", hashID)
}
c.Data(http.StatusOK, feedContentType, []byte(data))
const feedContentType = "application/rss+xml; charset=UTF-8"
c.Data(http.StatusOK, feedContentType, podcast)
}
func (h handler) metadata(c *gin.Context) {

View File

@@ -7,10 +7,8 @@ package handler
import (
gomock "github.com/golang/mock/gomock"
patreon_go "github.com/mxpv/patreon-go"
podcast "github.com/mxpv/podcast"
api "github.com/mxpv/podsync/pkg/api"
reflect "reflect"
time "time"
)
// MockfeedService is a mock of feedService interface
@@ -52,10 +50,10 @@ func (mr *MockfeedServiceMockRecorder) CreateFeed(req, identity interface{}) *go
}
// BuildFeed mocks base method
func (m *MockfeedService) BuildFeed(hashID string) (*podcast.Podcast, error) {
func (m *MockfeedService) BuildFeed(hashID string) ([]byte, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "BuildFeed", hashID)
ret0, _ := ret[0].(*podcast.Podcast)
ret0, _ := ret[0].([]byte)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -159,55 +157,3 @@ func (mr *MockpatreonServiceMockRecorder) GetFeatureLevelFromAmount(amount inter
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeatureLevelFromAmount", reflect.TypeOf((*MockpatreonService)(nil).GetFeatureLevelFromAmount), amount)
}
// MockcacheService is a mock of cacheService interface
type MockcacheService struct {
ctrl *gomock.Controller
recorder *MockcacheServiceMockRecorder
}
// MockcacheServiceMockRecorder is the mock recorder for MockcacheService
type MockcacheServiceMockRecorder struct {
mock *MockcacheService
}
// NewMockcacheService creates a new mock instance
func NewMockcacheService(ctrl *gomock.Controller) *MockcacheService {
mock := &MockcacheService{ctrl: ctrl}
mock.recorder = &MockcacheServiceMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockcacheService) EXPECT() *MockcacheServiceMockRecorder {
return m.recorder
}
// Set mocks base method
func (m *MockcacheService) Set(key, value string, ttl time.Duration) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Set", key, value, ttl)
ret0, _ := ret[0].(error)
return ret0
}
// Set indicates an expected call of Set
func (mr *MockcacheServiceMockRecorder) Set(key, value, ttl interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockcacheService)(nil).Set), key, value, ttl)
}
// Get mocks base method
func (m *MockcacheService) Get(key string) (string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Get", key)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Get indicates an expected call of Get
func (mr *MockcacheServiceMockRecorder) Get(key interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockcacheService)(nil).Get), key)
}

View File

@@ -9,10 +9,7 @@ import (
"strings"
"testing"
"github.com/pkg/errors"
"github.com/golang/mock/gomock"
itunes "github.com/mxpv/podcast"
"github.com/stretchr/testify/require"
"github.com/mxpv/podsync/pkg/api"
@@ -38,7 +35,7 @@ func TestCreateFeed(t *testing.T) {
patreon := NewMockpatreonService(ctrl)
patreon.EXPECT().GetFeatureLevelByID(gomock.Any()).Return(api.DefaultFeatures)
srv := httptest.NewServer(New(feed, patreon, nil, cfg))
srv := httptest.NewServer(New(feed, patreon, cfg))
defer srv.Close()
query := `{"url": "https://youtube.com/channel/123", "page_size": 55, "quality": "low", "format": "audio"}`
@@ -53,7 +50,7 @@ func TestCreateInvalidFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
srv := httptest.NewServer(New(NewMockfeedService(ctrl), nil, nil, cfg))
srv := httptest.NewServer(New(NewMockfeedService(ctrl), nil, cfg))
defer srv.Close()
query := `{}`
@@ -101,16 +98,10 @@ func TestGetFeed(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
podcast := itunes.New("", "", "", nil, nil)
feed := NewMockfeedService(ctrl)
feed.EXPECT().BuildFeed("123").Return(&podcast, nil)
feed.EXPECT().BuildFeed("123").Return([]byte("Test"), nil)
cache := NewMockcacheService(ctrl)
cache.EXPECT().Get("123").Times(1).Return("", errors.New("not found"))
cache.EXPECT().Set("123", podcast.String(), gomock.Any()).Return(nil).Times(1)
srv := httptest.NewServer(New(feed, nil, cache, cfg))
srv := httptest.NewServer(New(feed, nil, cfg))
defer srv.Close()
resp, err := http.Get(srv.URL + "/123")
@@ -125,7 +116,7 @@ func TestGetMetadata(t *testing.T) {
feed := NewMockfeedService(ctrl)
feed.EXPECT().GetMetadata("123").Times(1).Return(&api.Metadata{}, nil)
srv := httptest.NewServer(New(feed, nil, nil, cfg))
srv := httptest.NewServer(New(feed, nil, cfg))
defer srv.Close()
resp, err := http.Get(srv.URL + "/api/metadata/123")