From 1f302c185f1ec89e1e7809ab55840253af3ace28 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Sun, 24 Feb 2019 14:39:58 -0800 Subject: [PATCH] Drop Redis --- cmd/app/main.go | 11 ---- docker-compose.yml | 12 ----- go.mod | 1 - go.sum | 2 - pkg/config/config.go | 1 - pkg/feeds/feeds.go | 34 ++----------- pkg/feeds/feeds_mock_test.go | 98 +++++++++--------------------------- pkg/feeds/feeds_test.go | 38 ++------------ pkg/stats/redis.go | 94 ---------------------------------- pkg/stats/redis_test.go | 79 ----------------------------- 10 files changed, 30 insertions(+), 340 deletions(-) delete mode 100644 pkg/stats/redis.go delete mode 100644 pkg/stats/redis_test.go diff --git a/cmd/app/main.go b/cmd/app/main.go index 2be78f4..5c582d1 100644 --- a/cmd/app/main.go +++ b/cmd/app/main.go @@ -16,7 +16,6 @@ import ( "github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/feeds" "github.com/mxpv/podsync/pkg/handler" - "github.com/mxpv/podsync/pkg/stats" "github.com/mxpv/podsync/pkg/storage" "github.com/mxpv/podsync/pkg/support" @@ -39,11 +38,6 @@ func main() { log.WithError(err).Fatal("failed to read configuration") } - statistics, err := stats.NewRedisStats(cfg.RedisURL) - if err != nil { - log.WithError(err).Fatal("failed to create redis") - } - database, err := storage.NewDynamo(&aws.Config{ Region: aws.String("us-east-1"), Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKey, cfg.AWSAccessSecret, ""), @@ -77,7 +71,6 @@ func main() { feed, err := feeds.NewFeedService( feeds.WithStorage(database), - feeds.WithStats(statistics), feeds.WithBuilder(api.ProviderYoutube, youtube), feeds.WithBuilder(api.ProviderVimeo, vimeo), ) @@ -110,9 +103,5 @@ func main() { log.WithError(err).Error("failed to close database") } - if err := statistics.Close(); err != nil { - log.WithError(err).Error("failed to close stats storage") - } - log.Info("server gracefully stopped") } diff --git a/docker-compose.yml b/docker-compose.yml index ca93aa8..6ee04ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,6 @@ services: ports: - 5001 environment: - - REDIS_CONNECTION_URL=redis://redis - POSTGRES_CONNECTION_URL={POSTGRES_CONNECTION_URL} - YOUTUBE_API_KEY={YOUTUBE_API_KEY} - VIMEO_API_KEY={VIMEO_API_KEY} @@ -37,17 +36,6 @@ services: - 5002 environment: - METADATA_URL=http://app:5001/api/metadata/{feed_id} - redis: - image: redis:5.0.1 - container_name: redis - command: redis-server --appendonly no --save 900 1 --save 300 10 --save 60 10000 - restart: always - volumes: - - /data/redis:/data - sysctls: - net.core.somaxconn: 1024 - mem_limit: 268435456 # 256MB - oom_kill_disable: true nginx: image: mxpv/nginx:latest container_name: nginx diff --git a/go.mod b/go.mod index f3382f2..60f434a 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07 github.com/go-pg/pg v6.14.2+incompatible - github.com/go-redis/redis v6.12.0+incompatible github.com/golang/mock v1.1.1 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect github.com/gorilla/sessions v1.1.1 // indirect diff --git a/go.sum b/go.sum index 260ba0b..21e798f 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,6 @@ github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07 h1:cZPJWzd2oNeoS0oJM github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/go-pg/pg v6.14.2+incompatible h1:FrOgsHDUhC3V3wkBGAIN5LVj4nJczFPyy1YNFnetfIQ= github.com/go-pg/pg v6.14.2+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA= -github.com/go-redis/redis v6.12.0+incompatible h1:s+64XI+z/RXqGHz2fQSgRJOEwqqSXeX3dliF7iVkMbE= -github.com/go-redis/redis v6.12.0+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= diff --git a/pkg/config/config.go b/pkg/config/config.go index 3b5749d..da877ec 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,7 +16,6 @@ type AppConfig struct { PatreonRedirectURL string `yaml:"patreonRedirectUrl"` PatreonWebhooksSecret string `json:"patreonWebhooksSecret"` PostgresConnectionURL string `yaml:"postgresConnectionUrl"` - RedisURL string `yaml:"redisUrl"` CookieSecret string `yaml:"cookieSecret"` AWSAccessKey string `yaml:"awsAccessKey"` AWSAccessSecret string `yaml:"awsAccessSecret"` diff --git a/pkg/feeds/feeds.go b/pkg/feeds/feeds.go index 474c6a6..949df4b 100644 --- a/pkg/feeds/feeds.go +++ b/pkg/feeds/feeds.go @@ -17,11 +17,6 @@ const ( MetricDownloads = "downloads" ) -type stats interface { - Inc(metric, hashID string) (int64, error) - Get(metric, hashID string) (int64, error) -} - type builder interface { Build(feed *model.Feed) (podcast *itunes.Podcast, err error) } @@ -35,7 +30,6 @@ type storage interface { type Service struct { generator IDGen - stats stats db storage builders map[api.Provider]builder } @@ -111,15 +105,6 @@ func (s Service) BuildFeed(hashID string) (*itunes.Podcast, error) { return nil, err } - count, err := s.stats.Inc(MetricQueries, feed.HashID) - if err != nil { - return nil, errors.Wrapf(err, "failed to update metrics for feed: %s", hashID) - } - - if feed.PageSize > 150 && count > api.ExtendedPaginationQueryLimit { - return nil, api.ErrQuotaExceeded - } - builder, ok := s.builders[feed.Provider] if !ok { return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashID) @@ -139,16 +124,10 @@ func (s Service) GetMetadata(hashID string) (*api.Metadata, error) { return nil, err } - downloads, err := s.stats.Inc(MetricDownloads, hashID) - if err != nil { - return nil, err - } - return &api.Metadata{ - Provider: feed.Provider, - Format: feed.Format, - Quality: feed.Quality, - Downloads: downloads, + Provider: feed.Provider, + Format: feed.Format, + Quality: feed.Quality, }, nil } @@ -180,13 +159,6 @@ func WithBuilder(provider api.Provider, builder builder) FeedOption { } } -//noinspection GoExportedFuncWithUnexportedType -func WithStats(m stats) FeedOption { - return func(service *Service) { - service.stats = m - } -} - func NewFeedService(opts ...FeedOption) (*Service, error) { idGen, err := NewIDGen() if err != nil { diff --git a/pkg/feeds/feeds_mock_test.go b/pkg/feeds/feeds_mock_test.go index 21ac12a..f794e65 100644 --- a/pkg/feeds/feeds_mock_test.go +++ b/pkg/feeds/feeds_mock_test.go @@ -1,7 +1,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: feeds.go -// Package feeds is a generated GoMock package. package feeds import ( @@ -11,55 +10,6 @@ import ( reflect "reflect" ) -// Mockstats is a mock of stats interface -type Mockstats struct { - ctrl *gomock.Controller - recorder *MockstatsMockRecorder -} - -// MockstatsMockRecorder is the mock recorder for Mockstats -type MockstatsMockRecorder struct { - mock *Mockstats -} - -// NewMockstats creates a new mock instance -func NewMockstats(ctrl *gomock.Controller) *Mockstats { - mock := &Mockstats{ctrl: ctrl} - mock.recorder = &MockstatsMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *Mockstats) EXPECT() *MockstatsMockRecorder { - return m.recorder -} - -// Inc mocks base method -func (m *Mockstats) Inc(metric, hashID string) (int64, error) { - ret := m.ctrl.Call(m, "Inc", metric, hashID) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Inc indicates an expected call of Inc -func (mr *MockstatsMockRecorder) Inc(metric, hashID interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inc", reflect.TypeOf((*Mockstats)(nil).Inc), metric, hashID) -} - -// Get mocks base method -func (m *Mockstats) Get(metric, hashID string) (int64, error) { - ret := m.ctrl.Call(m, "Get", metric, hashID) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Get indicates an expected call of Get -func (mr *MockstatsMockRecorder) Get(metric, hashID interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*Mockstats)(nil).Get), metric, hashID) -} - // Mockbuilder is a mock of builder interface type Mockbuilder struct { ctrl *gomock.Controller @@ -79,21 +29,21 @@ func NewMockbuilder(ctrl *gomock.Controller) *Mockbuilder { } // EXPECT returns an object that allows the caller to indicate expected use -func (m *Mockbuilder) EXPECT() *MockbuilderMockRecorder { - return m.recorder +func (_m *Mockbuilder) EXPECT() *MockbuilderMockRecorder { + return _m.recorder } // Build mocks base method -func (m *Mockbuilder) Build(feed *model.Feed) (*podcast.Podcast, error) { - ret := m.ctrl.Call(m, "Build", feed) +func (_m *Mockbuilder) Build(feed *model.Feed) (*podcast.Podcast, error) { + ret := _m.ctrl.Call(_m, "Build", feed) ret0, _ := ret[0].(*podcast.Podcast) ret1, _ := ret[1].(error) return ret0, ret1 } // Build indicates an expected call of Build -func (mr *MockbuilderMockRecorder) Build(feed interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*Mockbuilder)(nil).Build), feed) +func (_mr *MockbuilderMockRecorder) Build(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Build", reflect.TypeOf((*Mockbuilder)(nil).Build), arg0) } // Mockstorage is a mock of storage interface @@ -115,56 +65,56 @@ func NewMockstorage(ctrl *gomock.Controller) *Mockstorage { } // EXPECT returns an object that allows the caller to indicate expected use -func (m *Mockstorage) EXPECT() *MockstorageMockRecorder { - return m.recorder +func (_m *Mockstorage) EXPECT() *MockstorageMockRecorder { + return _m.recorder } // SaveFeed mocks base method -func (m *Mockstorage) SaveFeed(feed *model.Feed) error { - ret := m.ctrl.Call(m, "SaveFeed", feed) +func (_m *Mockstorage) SaveFeed(feed *model.Feed) error { + ret := _m.ctrl.Call(_m, "SaveFeed", feed) ret0, _ := ret[0].(error) return ret0 } // SaveFeed indicates an expected call of SaveFeed -func (mr *MockstorageMockRecorder) SaveFeed(feed interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveFeed", reflect.TypeOf((*Mockstorage)(nil).SaveFeed), feed) +func (_mr *MockstorageMockRecorder) SaveFeed(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "SaveFeed", reflect.TypeOf((*Mockstorage)(nil).SaveFeed), arg0) } // GetFeed mocks base method -func (m *Mockstorage) GetFeed(hashID string) (*model.Feed, error) { - ret := m.ctrl.Call(m, "GetFeed", hashID) +func (_m *Mockstorage) GetFeed(hashID string) (*model.Feed, error) { + ret := _m.ctrl.Call(_m, "GetFeed", hashID) ret0, _ := ret[0].(*model.Feed) ret1, _ := ret[1].(error) return ret0, ret1 } // GetFeed indicates an expected call of GetFeed -func (mr *MockstorageMockRecorder) GetFeed(hashID interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), hashID) +func (_mr *MockstorageMockRecorder) GetFeed(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), arg0) } // GetMetadata mocks base method -func (m *Mockstorage) GetMetadata(hashID string) (*model.Feed, error) { - ret := m.ctrl.Call(m, "GetMetadata", hashID) +func (_m *Mockstorage) GetMetadata(hashID string) (*model.Feed, error) { + ret := _m.ctrl.Call(_m, "GetMetadata", hashID) ret0, _ := ret[0].(*model.Feed) ret1, _ := ret[1].(error) return ret0, ret1 } // GetMetadata indicates an expected call of GetMetadata -func (mr *MockstorageMockRecorder) GetMetadata(hashID interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetadata", reflect.TypeOf((*Mockstorage)(nil).GetMetadata), hashID) +func (_mr *MockstorageMockRecorder) GetMetadata(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetMetadata", reflect.TypeOf((*Mockstorage)(nil).GetMetadata), arg0) } // Downgrade mocks base method -func (m *Mockstorage) Downgrade(userID string, featureLevel int) error { - ret := m.ctrl.Call(m, "Downgrade", userID, featureLevel) +func (_m *Mockstorage) Downgrade(userID string, featureLevel int) error { + ret := _m.ctrl.Call(_m, "Downgrade", userID, featureLevel) ret0, _ := ret[0].(error) return ret0 } // Downgrade indicates an expected call of Downgrade -func (mr *MockstorageMockRecorder) Downgrade(userID, featureLevel interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Downgrade", reflect.TypeOf((*Mockstorage)(nil).Downgrade), userID, featureLevel) +func (_mr *MockstorageMockRecorder) Downgrade(arg0, arg1 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Downgrade", reflect.TypeOf((*Mockstorage)(nil).Downgrade), arg0, arg1) } diff --git a/pkg/feeds/feeds_test.go b/pkg/feeds/feeds_test.go index 33fef22..535f19e 100644 --- a/pkg/feeds/feeds_test.go +++ b/pkg/feeds/feeds_test.go @@ -99,44 +99,15 @@ func TestService_GetFeed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - stats := NewMockstats(ctrl) - stats.EXPECT().Inc(MetricQueries, feed.HashID).Return(int64(10), nil) - stor := NewMockstorage(ctrl) stor.EXPECT().GetFeed(feed.HashID).Times(1).Return(feed, nil) - s := Service{db: stor, stats: stats} + s := Service{db: stor} _, err := s.BuildFeed(feed.HashID) require.NoError(t, err) } -func TestService_BuildFeedQuotaCheck(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - f := &model.Feed{ - HashID: "321", - ItemID: "xyz", - Provider: api.ProviderVimeo, - LinkType: api.LinkTypeChannel, - PageSize: 600, - Quality: api.QualityHigh, - Format: api.FormatVideo, - } - - stats := NewMockstats(ctrl) - stats.EXPECT().Inc(MetricQueries, f.HashID).Return(int64(api.ExtendedPaginationQueryLimit)+1, nil) - - stor := NewMockstorage(ctrl) - stor.EXPECT().GetFeed(f.HashID).Times(1).Return(f, nil) - - s := Service{db: stor, stats: stats} - - _, err := s.BuildFeed(f.HashID) - require.Equal(t, api.ErrQuotaExceeded, err) -} - func TestService_WrongID(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -157,12 +128,9 @@ func TestService_GetMetadata(t *testing.T) { stor := NewMockstorage(ctrl) stor.EXPECT().GetMetadata(feed.HashID).Times(1).Return(feed, nil) - stats := NewMockstats(ctrl) - stats.EXPECT().Inc(MetricDownloads, feed.HashID).Return(int64(10), nil) - - s := Service{db: stor, stats: stats} + s := Service{db: stor} m, err := s.GetMetadata(feed.HashID) require.NoError(t, err) - require.Equal(t, int64(10), m.Downloads) + require.EqualValues(t, 0, m.Downloads) } diff --git a/pkg/stats/redis.go b/pkg/stats/redis.go deleted file mode 100644 index 56ae28e..0000000 --- a/pkg/stats/redis.go +++ /dev/null @@ -1,94 +0,0 @@ -package stats - -import ( - "fmt" - "time" - - "github.com/go-redis/redis" -) - -// RedisStats implement stats Redis backend -// Inside docker can be connected as: -// docker exec -it redis redis-cli -// View available stats keys -// 127.0.0.1:6379> keys stats/top/* -// Get stats top: -// 127.0.0.1:6379> zrevrange stats/top/2018/12/queries 0 100 withscores -// 127.0.0.1:6379> zrevrange stats/top/2018/12/downloads 0 100 withscores -// Query specific feed stats: -// 127.0.0.1:6379> hgetall "stats/2018/12/p2AZoiTNO" -type RedisStats struct { - client *redis.Client -} - -func (r RedisStats) Inc(metric, hashID string) (int64, error) { - now := time.Now().UTC() - - key := r.makeKey(now, hashID) - top := r.makeTop(now, metric) - - var cmd *redis.IntCmd - _, err := r.client.TxPipelined(func(p redis.Pipeliner) error { - cmd = p.HIncrBy(key, metric, 1) - p.ZIncrBy(top, 1, hashID) - return nil - }) - - if err != nil { - return 0, err - } - - return cmd.Result() -} - -func (r RedisStats) Get(metric, hashID string) (int64, error) { - now := time.Now().UTC() - key := r.makeKey(now, hashID) - return r.client.HGet(key, metric).Int64() -} - -func (r RedisStats) Top(metric string) (map[string]int64, error) { - now := time.Now().UTC() - top := r.makeTop(now, metric) - - zrange, err := r.client.ZRevRangeWithScores(top, 0, 10).Result() - if err != nil { - return nil, err - } - - ret := make(map[string]int64) - for _, x := range zrange { - key := x.Member.(string) - val := int64(x.Score) - - ret[key] = val - } - - return ret, nil -} - -func (r RedisStats) makeKey(now time.Time, hashID string) string { - return fmt.Sprintf("stats/%d/%d/%s", now.Year(), now.Month(), hashID) -} - -func (r RedisStats) makeTop(now time.Time, metric string) string { - return fmt.Sprintf("stats/top/%d/%d/%s", now.Year(), now.Month(), metric) -} - -func (r RedisStats) Close() error { - return r.client.Close() -} - -func NewRedisStats(redisURL string) (*RedisStats, error) { - opts, err := redis.ParseURL(redisURL) - if err != nil { - return nil, err - } - - client := redis.NewClient(opts) - if err := client.Ping().Err(); err != nil { - return nil, err - } - - return &RedisStats{client}, nil -} diff --git a/pkg/stats/redis_test.go b/pkg/stats/redis_test.go deleted file mode 100644 index e7174ae..0000000 --- a/pkg/stats/redis_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package stats - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -const metric = "downloads" - -func TestRedisStats_IncAndGet(t *testing.T) { - t.Skip("run redis tests manually") - - s := createRedisClient(t) - - const hashID = "321" - - v, err := s.Inc(metric, hashID) - require.NoError(t, err) - require.Equal(t, int64(1), v) - - v, err = s.Inc(metric, hashID) - require.NoError(t, err) - require.Equal(t, int64(2), v) - - v, err = s.Get(metric, hashID) - require.NoError(t, err) - require.Equal(t, int64(2), v) -} - -func TestRedisStats_Top(t *testing.T) { - t.Skip("run redis tests manually") - - s := createRedisClient(t) - - // 3 - s.Inc(metric, "123") - s.Inc(metric, "123") - s.Inc(metric, "123") - - // 2 - s.Inc(metric, "321") - s.Inc(metric, "321") - - // 1 - s.Inc(metric, "213") - - top, err := s.Top(metric) - require.NoError(t, err) - require.Len(t, top, 3) - - // 3 - h3, ok := top["123"] - require.True(t, ok) - require.Equal(t, int64(3), h3) - - // 2 - h2, ok := top["321"] - require.True(t, ok) - require.Equal(t, int64(2), h2) - - // 1 - h1, ok := top["213"] - require.True(t, ok) - require.Equal(t, int64(1), h1) -} - -func createRedisClient(t *testing.T) *RedisStats { - client, err := NewRedisStats("redis://localhost") - require.NoError(t, err) - - keys, err := client.client.Keys("*").Result() - require.NoError(t, err) - - err = client.client.Del(keys...).Err() - require.NoError(t, err) - - return client -}