Drop Redis

cmd/app/main.go

@@ -16,7 +16,6 @@ import (
 	"github.com/mxpv/podsync/pkg/config"
 	"github.com/mxpv/podsync/pkg/feeds"
 	"github.com/mxpv/podsync/pkg/handler"
-	"github.com/mxpv/podsync/pkg/stats"
 	"github.com/mxpv/podsync/pkg/storage"
 	"github.com/mxpv/podsync/pkg/support"
 
@@ -39,11 +38,6 @@ func main() {
 		log.WithError(err).Fatal("failed to read configuration")
 	}
 
-	statistics, err := stats.NewRedisStats(cfg.RedisURL)
-	if err != nil {
-		log.WithError(err).Fatal("failed to create redis")
-	}
-
 	database, err := storage.NewDynamo(&aws.Config{
 		Region:      aws.String("us-east-1"),
 		Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKey, cfg.AWSAccessSecret, ""),
@@ -77,7 +71,6 @@ func main() {
 
 	feed, err := feeds.NewFeedService(
 		feeds.WithStorage(database),
-		feeds.WithStats(statistics),
 		feeds.WithBuilder(api.ProviderYoutube, youtube),
 		feeds.WithBuilder(api.ProviderVimeo, vimeo),
 	)
@@ -110,9 +103,5 @@ func main() {
 		log.WithError(err).Error("failed to close database")
 	}
 
-	if err := statistics.Close(); err != nil {
-		log.WithError(err).Error("failed to close stats storage")
-	}
-
 	log.Info("server gracefully stopped")
 }

docker-compose.yml

@@ -8,7 +8,6 @@ services:
     ports:
       - 5001
     environment:
-      - REDIS_CONNECTION_URL=redis://redis
       - POSTGRES_CONNECTION_URL={POSTGRES_CONNECTION_URL}
       - YOUTUBE_API_KEY={YOUTUBE_API_KEY}
       - VIMEO_API_KEY={VIMEO_API_KEY}
@@ -37,17 +36,6 @@ services:
       - 5002
     environment:
       - METADATA_URL=http://app:5001/api/metadata/{feed_id}
-  redis:
-    image: redis:5.0.1
-    container_name: redis
-    command: redis-server --appendonly no --save 900 1 --save 300 10 --save 60 10000
-    restart: always
-    volumes:
-      - /data/redis:/data
-    sysctls:
-      net.core.somaxconn: 1024
-    mem_limit: 268435456 # 256MB
-    oom_kill_disable: true
   nginx:
     image: mxpv/nginx:latest
     container_name: nginx

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect
 	github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07
 	github.com/go-pg/pg v6.14.2+incompatible
-	github.com/go-redis/redis v6.12.0+incompatible
 	github.com/golang/mock v1.1.1
 	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
 	github.com/gorilla/sessions v1.1.1 // indirect

go.sum

@@ -30,8 +30,6 @@ github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07 h1:cZPJWzd2oNeoS0oJM
 github.com/gin-gonic/gin v0.0.0-20170702092826-d459835d2b07/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
 github.com/go-pg/pg v6.14.2+incompatible h1:FrOgsHDUhC3V3wkBGAIN5LVj4nJczFPyy1YNFnetfIQ=
 github.com/go-pg/pg v6.14.2+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA=
-github.com/go-redis/redis v6.12.0+incompatible h1:s+64XI+z/RXqGHz2fQSgRJOEwqqSXeX3dliF7iVkMbE=
-github.com/go-redis/redis v6.12.0+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=

pkg/config/config.go

@@ -16,7 +16,6 @@ type AppConfig struct {
 	PatreonRedirectURL     string `yaml:"patreonRedirectUrl"`
 	PatreonWebhooksSecret  string `json:"patreonWebhooksSecret"`
 	PostgresConnectionURL  string `yaml:"postgresConnectionUrl"`
-	RedisURL               string `yaml:"redisUrl"`
 	CookieSecret           string `yaml:"cookieSecret"`
 	AWSAccessKey           string `yaml:"awsAccessKey"`
 	AWSAccessSecret        string `yaml:"awsAccessSecret"`

pkg/feeds/feeds.go

@@ -17,11 +17,6 @@ const (
 	MetricDownloads = "downloads"
 )
 
-type stats interface {
-	Inc(metric, hashID string) (int64, error)
-	Get(metric, hashID string) (int64, error)
-}
-
 type builder interface {
 	Build(feed *model.Feed) (podcast *itunes.Podcast, err error)
 }
@@ -35,7 +30,6 @@ type storage interface {
 
 type Service struct {
 	generator IDGen
-	stats     stats
 	db        storage
 	builders  map[api.Provider]builder
 }
@@ -111,15 +105,6 @@ func (s Service) BuildFeed(hashID string) (*itunes.Podcast, error) {
 		return nil, err
 	}
 
-	count, err := s.stats.Inc(MetricQueries, feed.HashID)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to update metrics for feed: %s", hashID)
-	}
-
-	if feed.PageSize > 150 && count > api.ExtendedPaginationQueryLimit {
-		return nil, api.ErrQuotaExceeded
-	}
-
 	builder, ok := s.builders[feed.Provider]
 	if !ok {
 		return nil, errors.Wrapf(err, "failed to get builder for feed: %s", hashID)
@@ -139,16 +124,10 @@ func (s Service) GetMetadata(hashID string) (*api.Metadata, error) {
 		return nil, err
 	}
 
-	downloads, err := s.stats.Inc(MetricDownloads, hashID)
-	if err != nil {
-		return nil, err
-	}
-
 	return &api.Metadata{
-		Provider:  feed.Provider,
-		Format:    feed.Format,
-		Quality:   feed.Quality,
-		Downloads: downloads,
+		Provider: feed.Provider,
+		Format:   feed.Format,
+		Quality:  feed.Quality,
 	}, nil
 }
 
@@ -180,13 +159,6 @@ func WithBuilder(provider api.Provider, builder builder) FeedOption {
 	}
 }
 
-//noinspection GoExportedFuncWithUnexportedType
-func WithStats(m stats) FeedOption {
-	return func(service *Service) {
-		service.stats = m
-	}
-}
-
 func NewFeedService(opts ...FeedOption) (*Service, error) {
 	idGen, err := NewIDGen()
 	if err != nil {

pkg/feeds/feeds_mock_test.go

@@ -1,7 +1,6 @@
 // Code generated by MockGen. DO NOT EDIT.
 // Source: feeds.go
 
-// Package feeds is a generated GoMock package.
 package feeds
 
 import (
@@ -11,55 +10,6 @@ import (
 	reflect "reflect"
 )
 
-// Mockstats is a mock of stats interface
-type Mockstats struct {
-	ctrl     *gomock.Controller
-	recorder *MockstatsMockRecorder
-}
-
-// MockstatsMockRecorder is the mock recorder for Mockstats
-type MockstatsMockRecorder struct {
-	mock *Mockstats
-}
-
-// NewMockstats creates a new mock instance
-func NewMockstats(ctrl *gomock.Controller) *Mockstats {
-	mock := &Mockstats{ctrl: ctrl}
-	mock.recorder = &MockstatsMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *Mockstats) EXPECT() *MockstatsMockRecorder {
-	return m.recorder
-}
-
-// Inc mocks base method
-func (m *Mockstats) Inc(metric, hashID string) (int64, error) {
-	ret := m.ctrl.Call(m, "Inc", metric, hashID)
-	ret0, _ := ret[0].(int64)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Inc indicates an expected call of Inc
-func (mr *MockstatsMockRecorder) Inc(metric, hashID interface{}) *gomock.Call {
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inc", reflect.TypeOf((*Mockstats)(nil).Inc), metric, hashID)
-}
-
-// Get mocks base method
-func (m *Mockstats) Get(metric, hashID string) (int64, error) {
-	ret := m.ctrl.Call(m, "Get", metric, hashID)
-	ret0, _ := ret[0].(int64)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Get indicates an expected call of Get
-func (mr *MockstatsMockRecorder) Get(metric, hashID interface{}) *gomock.Call {
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*Mockstats)(nil).Get), metric, hashID)
-}
-
 // Mockbuilder is a mock of builder interface
 type Mockbuilder struct {
 	ctrl     *gomock.Controller
@@ -79,21 +29,21 @@ func NewMockbuilder(ctrl *gomock.Controller) *Mockbuilder {
 }
 
 // EXPECT returns an object that allows the caller to indicate expected use
-func (m *Mockbuilder) EXPECT() *MockbuilderMockRecorder {
-	return m.recorder
+func (_m *Mockbuilder) EXPECT() *MockbuilderMockRecorder {
+	return _m.recorder
 }
 
 // Build mocks base method
-func (m *Mockbuilder) Build(feed *model.Feed) (*podcast.Podcast, error) {
-	ret := m.ctrl.Call(m, "Build", feed)
+func (_m *Mockbuilder) Build(feed *model.Feed) (*podcast.Podcast, error) {
+	ret := _m.ctrl.Call(_m, "Build", feed)
 	ret0, _ := ret[0].(*podcast.Podcast)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // Build indicates an expected call of Build
-func (mr *MockbuilderMockRecorder) Build(feed interface{}) *gomock.Call {
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*Mockbuilder)(nil).Build), feed)
+func (_mr *MockbuilderMockRecorder) Build(arg0 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Build", reflect.TypeOf((*Mockbuilder)(nil).Build), arg0)
 }
 
 // Mockstorage is a mock of storage interface
@@ -115,56 +65,56 @@ func NewMockstorage(ctrl *gomock.Controller) *Mockstorage {
 }
 
 // EXPECT returns an object that allows the caller to indicate expected use
-func (m *Mockstorage) EXPECT() *MockstorageMockRecorder {
-	return m.recorder
+func (_m *Mockstorage) EXPECT() *MockstorageMockRecorder {
+	return _m.recorder
 }
 
 // SaveFeed mocks base method
-func (m *Mockstorage) SaveFeed(feed *model.Feed) error {
-	ret := m.ctrl.Call(m, "SaveFeed", feed)
+func (_m *Mockstorage) SaveFeed(feed *model.Feed) error {
+	ret := _m.ctrl.Call(_m, "SaveFeed", feed)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // SaveFeed indicates an expected call of SaveFeed
-func (mr *MockstorageMockRecorder) SaveFeed(feed interface{}) *gomock.Call {
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveFeed", reflect.TypeOf((*Mockstorage)(nil).SaveFeed), feed)
+func (_mr *MockstorageMockRecorder) SaveFeed(arg0 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "SaveFeed", reflect.TypeOf((*Mockstorage)(nil).SaveFeed), arg0)
 }
 
 // GetFeed mocks base method
-func (m *Mockstorage) GetFeed(hashID string) (*model.Feed, error) {
-	ret := m.ctrl.Call(m, "GetFeed", hashID)
+func (_m *Mockstorage) GetFeed(hashID string) (*model.Feed, error) {
+	ret := _m.ctrl.Call(_m, "GetFeed", hashID)
 	ret0, _ := ret[0].(*model.Feed)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // GetFeed indicates an expected call of GetFeed
-func (mr *MockstorageMockRecorder) GetFeed(hashID interface{}) *gomock.Call {
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), hashID)
+func (_mr *MockstorageMockRecorder) GetFeed(arg0 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetFeed", reflect.TypeOf((*Mockstorage)(nil).GetFeed), arg0)
 }
 
 // GetMetadata mocks base method
-func (m *Mockstorage) GetMetadata(hashID string) (*model.Feed, error) {
-	ret := m.ctrl.Call(m, "GetMetadata", hashID)
+func (_m *Mockstorage) GetMetadata(hashID string) (*model.Feed, error) {
+	ret := _m.ctrl.Call(_m, "GetMetadata", hashID)
 	ret0, _ := ret[0].(*model.Feed)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // GetMetadata indicates an expected call of GetMetadata
-func (mr *MockstorageMockRecorder) GetMetadata(hashID interface{}) *gomock.Call {
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetadata", reflect.TypeOf((*Mockstorage)(nil).GetMetadata), hashID)
+func (_mr *MockstorageMockRecorder) GetMetadata(arg0 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetMetadata", reflect.TypeOf((*Mockstorage)(nil).GetMetadata), arg0)
 }
 
 // Downgrade mocks base method
-func (m *Mockstorage) Downgrade(userID string, featureLevel int) error {
-	ret := m.ctrl.Call(m, "Downgrade", userID, featureLevel)
+func (_m *Mockstorage) Downgrade(userID string, featureLevel int) error {
+	ret := _m.ctrl.Call(_m, "Downgrade", userID, featureLevel)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // Downgrade indicates an expected call of Downgrade
-func (mr *MockstorageMockRecorder) Downgrade(userID, featureLevel interface{}) *gomock.Call {
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Downgrade", reflect.TypeOf((*Mockstorage)(nil).Downgrade), userID, featureLevel)
+func (_mr *MockstorageMockRecorder) Downgrade(arg0, arg1 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Downgrade", reflect.TypeOf((*Mockstorage)(nil).Downgrade), arg0, arg1)
 }

pkg/feeds/feeds_test.go

@@ -99,44 +99,15 @@ func TestService_GetFeed(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	stats := NewMockstats(ctrl)
-	stats.EXPECT().Inc(MetricQueries, feed.HashID).Return(int64(10), nil)
-
 	stor := NewMockstorage(ctrl)
 	stor.EXPECT().GetFeed(feed.HashID).Times(1).Return(feed, nil)
 
-	s := Service{db: stor, stats: stats}
+	s := Service{db: stor}
 
 	_, err := s.BuildFeed(feed.HashID)
 	require.NoError(t, err)
 }
 
-func TestService_BuildFeedQuotaCheck(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-
-	f := &model.Feed{
-		HashID:   "321",
-		ItemID:   "xyz",
-		Provider: api.ProviderVimeo,
-		LinkType: api.LinkTypeChannel,
-		PageSize: 600,
-		Quality:  api.QualityHigh,
-		Format:   api.FormatVideo,
-	}
-
-	stats := NewMockstats(ctrl)
-	stats.EXPECT().Inc(MetricQueries, f.HashID).Return(int64(api.ExtendedPaginationQueryLimit)+1, nil)
-
-	stor := NewMockstorage(ctrl)
-	stor.EXPECT().GetFeed(f.HashID).Times(1).Return(f, nil)
-
-	s := Service{db: stor, stats: stats}
-
-	_, err := s.BuildFeed(f.HashID)
-	require.Equal(t, api.ErrQuotaExceeded, err)
-}
-
 func TestService_WrongID(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -157,12 +128,9 @@ func TestService_GetMetadata(t *testing.T) {
 	stor := NewMockstorage(ctrl)
 	stor.EXPECT().GetMetadata(feed.HashID).Times(1).Return(feed, nil)
 
-	stats := NewMockstats(ctrl)
-	stats.EXPECT().Inc(MetricDownloads, feed.HashID).Return(int64(10), nil)
-
-	s := Service{db: stor, stats: stats}
+	s := Service{db: stor}
 
 	m, err := s.GetMetadata(feed.HashID)
 	require.NoError(t, err)
-	require.Equal(t, int64(10), m.Downloads)
+	require.EqualValues(t, 0, m.Downloads)
 }

pkg/stats/redis.go

@@ -1,94 +0,0 @@
-package stats
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/go-redis/redis"
-)
-
-// RedisStats implement stats Redis backend
-// Inside docker can be connected as:
-//      docker exec -it redis redis-cli
-// View available stats keys
-//      127.0.0.1:6379> keys stats/top/*
-// Get stats top:
-//      127.0.0.1:6379> zrevrange stats/top/2018/12/queries 0 100 withscores
-//      127.0.0.1:6379> zrevrange stats/top/2018/12/downloads 0 100 withscores
-// Query specific feed stats:
-//      127.0.0.1:6379> hgetall "stats/2018/12/p2AZoiTNO"
-type RedisStats struct {
-	client *redis.Client
-}
-
-func (r RedisStats) Inc(metric, hashID string) (int64, error) {
-	now := time.Now().UTC()
-
-	key := r.makeKey(now, hashID)
-	top := r.makeTop(now, metric)
-
-	var cmd *redis.IntCmd
-	_, err := r.client.TxPipelined(func(p redis.Pipeliner) error {
-		cmd = p.HIncrBy(key, metric, 1)
-		p.ZIncrBy(top, 1, hashID)
-		return nil
-	})
-
-	if err != nil {
-		return 0, err
-	}
-
-	return cmd.Result()
-}
-
-func (r RedisStats) Get(metric, hashID string) (int64, error) {
-	now := time.Now().UTC()
-	key := r.makeKey(now, hashID)
-	return r.client.HGet(key, metric).Int64()
-}
-
-func (r RedisStats) Top(metric string) (map[string]int64, error) {
-	now := time.Now().UTC()
-	top := r.makeTop(now, metric)
-
-	zrange, err := r.client.ZRevRangeWithScores(top, 0, 10).Result()
-	if err != nil {
-		return nil, err
-	}
-
-	ret := make(map[string]int64)
-	for _, x := range zrange {
-		key := x.Member.(string)
-		val := int64(x.Score)
-
-		ret[key] = val
-	}
-
-	return ret, nil
-}
-
-func (r RedisStats) makeKey(now time.Time, hashID string) string {
-	return fmt.Sprintf("stats/%d/%d/%s", now.Year(), now.Month(), hashID)
-}
-
-func (r RedisStats) makeTop(now time.Time, metric string) string {
-	return fmt.Sprintf("stats/top/%d/%d/%s", now.Year(), now.Month(), metric)
-}
-
-func (r RedisStats) Close() error {
-	return r.client.Close()
-}
-
-func NewRedisStats(redisURL string) (*RedisStats, error) {
-	opts, err := redis.ParseURL(redisURL)
-	if err != nil {
-		return nil, err
-	}
-
-	client := redis.NewClient(opts)
-	if err := client.Ping().Err(); err != nil {
-		return nil, err
-	}
-
-	return &RedisStats{client}, nil
-}

pkg/stats/redis_test.go

@@ -1,79 +0,0 @@
-package stats
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-const metric = "downloads"
-
-func TestRedisStats_IncAndGet(t *testing.T) {
-	t.Skip("run redis tests manually")
-
-	s := createRedisClient(t)
-
-	const hashID = "321"
-
-	v, err := s.Inc(metric, hashID)
-	require.NoError(t, err)
-	require.Equal(t, int64(1), v)
-
-	v, err = s.Inc(metric, hashID)
-	require.NoError(t, err)
-	require.Equal(t, int64(2), v)
-
-	v, err = s.Get(metric, hashID)
-	require.NoError(t, err)
-	require.Equal(t, int64(2), v)
-}
-
-func TestRedisStats_Top(t *testing.T) {
-	t.Skip("run redis tests manually")
-
-	s := createRedisClient(t)
-
-	// 3
-	s.Inc(metric, "123")
-	s.Inc(metric, "123")
-	s.Inc(metric, "123")
-
-	// 2
-	s.Inc(metric, "321")
-	s.Inc(metric, "321")
-
-	// 1
-	s.Inc(metric, "213")
-
-	top, err := s.Top(metric)
-	require.NoError(t, err)
-	require.Len(t, top, 3)
-
-	// 3
-	h3, ok := top["123"]
-	require.True(t, ok)
-	require.Equal(t, int64(3), h3)
-
-	// 2
-	h2, ok := top["321"]
-	require.True(t, ok)
-	require.Equal(t, int64(2), h2)
-
-	// 1
-	h1, ok := top["213"]
-	require.True(t, ok)
-	require.Equal(t, int64(1), h1)
-}
-
-func createRedisClient(t *testing.T) *RedisStats {
-	client, err := NewRedisStats("redis://localhost")
-	require.NoError(t, err)
-
-	keys, err := client.client.Keys("*").Result()
-	require.NoError(t, err)
-
-	err = client.client.Del(keys...).Err()
-	require.NoError(t, err)
-
-	return client
-}