Merge pull request #132 from mxpv/key-rotation

Support multiple API keys

README.md

@@ -30,6 +30,7 @@ any device in podcast client.
 - Runs on Windows, Mac OS, Linux, and Docker.
 - Supports ARM.
 - Automatic youtube-dl self update.
+- Supports API keys rotation.
 
 ## Dependencies
 
@@ -58,9 +59,13 @@ Here is an example how configuration might look like:
 port = 8080
 data_dir = "/app/data" # Don't change if you run podsync via docker
 
+# Tokens from `Access tokens` section
 [tokens]
-youtube = "{YOUTUBE_API_TOKEN}" # Tokens from `Access tokens` section
-vimeo = "{VIMEO_API_TOKEN}"
+youtube = "YOUTUBE_API_TOKEN" # YouTube API Key. See https://developers.google.com/youtube/registering_an_application
+vimeo = [ # Multiple keys will be rotated.
+  "VIMEO_API_KEY_1", # Vimeo developer keys. See https://developer.vimeo.com/api/guides/start#generate-access-token
+  "VIMEO_API_KEY_2"
+]
 
 [feeds]
   [feeds.ID1]

cmd/podsync/updater.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/mxpv/podsync/pkg/builder"
 	"github.com/mxpv/podsync/pkg/config"
 	"github.com/mxpv/podsync/pkg/db"
 	"github.com/mxpv/podsync/pkg/feed"
@@ -31,14 +32,26 @@ type Updater struct {
 	downloader Downloader
 	db         db.Storage
 	fs         fs.Storage
+	keys       map[model.Provider]feed.KeyProvider
 }
 
 func NewUpdater(config *config.Config, downloader Downloader, db db.Storage, fs fs.Storage) (*Updater, error) {
+	keys := map[model.Provider]feed.KeyProvider{}
+
+	for name, list := range config.Tokens {
+		provider, err := feed.NewKeyProvider(list)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create key provider for %q", name)
+		}
+		keys[name] = provider
+	}
+
 	return &Updater{
 		config:     config,
 		downloader: downloader,
 		db:         db,
 		fs:         fs,
+		keys:       keys,
 	}, nil
 }
 
@@ -78,8 +91,18 @@ func (u *Updater) Update(ctx context.Context, feedConfig *config.Feed) error {
 
 // updateFeed pulls API for new episodes and saves them to database
 func (u *Updater) updateFeed(ctx context.Context, feedConfig *config.Feed) error {
+	info, err := builder.ParseURL(feedConfig.URL)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse URL: %s", feedConfig.URL)
+	}
+
+	keyProvider, ok := u.keys[info.Provider]
+	if !ok {
+		return errors.Errorf("key provider %q not loaded", info.Provider)
+	}
+
 	// Create an updater for this feed type
-	provider, err := feed.New(ctx, feedConfig, u.config.Tokens)
+	provider, err := builder.New(ctx, info.Provider, keyProvider.Get())
 	if err != nil {
 		return err
 	}

go.mod

@@ -2,13 +2,15 @@ module github.com/mxpv/podsync
 
 require (
 	github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec
-	github.com/BurntSushi/toml v0.3.1
 	github.com/dgraph-io/badger v1.6.0
 	github.com/eduncan911/podcast v1.4.2
 	github.com/gilliek/go-opml v1.0.0
 	github.com/golang/mock v1.4.3
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/jessevdk/go-flags v1.4.0
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/naoina/toml v0.1.1
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c

go.sum

@@ -37,9 +37,15 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

pkg/builder/builder.go

@@ -0,0 +1,25 @@
+package builder
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/mxpv/podsync/pkg/config"
+	"github.com/mxpv/podsync/pkg/model"
+)
+
+type Builder interface {
+	Build(ctx context.Context, cfg *config.Feed) (*model.Feed, error)
+}
+
+func New(ctx context.Context, provider model.Provider, key string) (Builder, error) {
+	switch provider {
+	case model.ProviderYoutube:
+		return NewYouTubeBuilder(key)
+	case model.ProviderVimeo:
+		return NewVimeoBuilder(ctx, key)
+	default:
+		return nil, errors.Errorf("unsupported provider %q", provider)
+	}
+}

pkg/builder/url.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"net/url"

pkg/builder/url_test.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"net/url"

pkg/builder/vimeo.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"net/http"

pkg/builder/vimeo_test.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"context"

pkg/builder/youtube.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"context"

pkg/builder/youtube_test.go

@@ -1,4 +1,4 @@
-package feed
+package builder
 
 import (
 	"context"

pkg/config/config.go

@@ -2,11 +2,11 @@ package config
 
 import (
 	"fmt"
+	"io/ioutil"
 	"path/filepath"
-	"time"
 
-	"github.com/BurntSushi/toml"
 	"github.com/hashicorp/go-multierror"
+	"github.com/naoina/toml"
 	"github.com/pkg/errors"
 
 	"github.com/mxpv/podsync/pkg/model"
@@ -44,6 +44,11 @@ type Feed struct {
 	OPML bool `toml:"opml"`
 }
 
+type Filters struct {
+	Title string `toml:"title"`
+	// More filters to be added here
+}
+
 type Custom struct {
 	CoverArt string `toml:"cover_art"`
 	Category string `toml:"category"`
@@ -51,15 +56,6 @@ type Custom struct {
 	Language string `toml:"lang"`
 }
 
-type Tokens struct {
-	// YouTube API key.
-	// See https://developers.google.com/youtube/registering_an_application
-	YouTube string `toml:"youtube"`
-	// Vimeo developer key.
-	// See https://developer.vimeo.com/api/guides/start#generate-access-token
-	Vimeo string `toml:"vimeo"`
-}
-
 type Server struct {
 	// Hostname to use for download links
 	Hostname string `toml:"hostname"`
@@ -118,17 +114,21 @@ type Config struct {
 	// ID will be used as feed ID in http://podsync.net/{FEED_ID}.xml
 	Feeds map[string]*Feed
 	// Tokens is API keys to use to access YouTube/Vimeo APIs.
-	Tokens Tokens `toml:"tokens"`
+	Tokens map[model.Provider]StringSlice `toml:"tokens"`
 	// Downloader (youtube-dl) configuration
 	Downloader Downloader `toml:"downloader"`
 }
 
 // LoadConfig loads TOML configuration from a file path
 func LoadConfig(path string) (*Config, error) {
-	config := Config{}
-	_, err := toml.DecodeFile(path, &config)
+	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to load config file")
+		return nil, errors.Wrapf(err, "failed to read config file: %s", path)
+	}
+
+	config := Config{}
+	if err := toml.Unmarshal(data, &config); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal toml")
 	}
 
 	for id, feed := range config.Feeds {
@@ -207,18 +207,3 @@ func (c *Config) applyDefaults(configPath string) {
 		}
 	}
 }
-
-type Duration struct {
-	time.Duration
-}
-
-func (d *Duration) UnmarshalText(text []byte) error {
-	var err error
-	d.Duration, err = time.ParseDuration(string(text))
-	return err
-}
-
-type Filters struct {
-	Title string `toml:"title"`
-	// More filters to be added here
-}

pkg/config/config_test.go

@@ -16,7 +16,10 @@ func TestLoadConfig(t *testing.T) {
 	const file = `
 [tokens]
 youtube = "123"
-vimeo = "321"
+vimeo = [
+  "321",
+  "456"
+]
 
 [server]
 port = 80
@@ -44,15 +47,18 @@ self_update = true
 
 	config, err := LoadConfig(path)
 	assert.NoError(t, err)
-	assert.NotNil(t, config)
+	require.NotNil(t, config)
 
 	assert.Equal(t, "test/data/", config.Server.DataDir)
 	assert.EqualValues(t, 80, config.Server.Port)
 
 	assert.Equal(t, "/home/user/db/", config.Database.Dir)
 
-	assert.Equal(t, "123", config.Tokens.YouTube)
-	assert.Equal(t, "321", config.Tokens.Vimeo)
+	require.Len(t, config.Tokens["youtube"], 1)
+	assert.Equal(t, "123", config.Tokens["youtube"][0])
+	require.Len(t, config.Tokens["vimeo"], 2)
+	assert.Equal(t, "321", config.Tokens["vimeo"][0])
+	assert.Equal(t, "456", config.Tokens["vimeo"][1])
 
 	assert.Len(t, config.Feeds, 1)
 	feed, ok := config.Feeds["XYZ"]
@@ -75,6 +81,28 @@ self_update = true
 	assert.True(t, config.Downloader.SelfUpdate)
 }
 
+func TestLoadEmptyKeyList(t *testing.T) {
+	const file = `
+[tokens]
+vimeo = []
+
+[server]
+data_dir = "/data"
+[feeds]
+  [feeds.A]
+  url = "https://youtube.com/watch?v=ygIUF678y40"
+`
+	path := setup(t, file)
+	defer os.Remove(path)
+
+	config, err := LoadConfig(path)
+	assert.NoError(t, err)
+	require.NotNil(t, config)
+
+	require.Len(t, config.Tokens, 1)
+	require.Len(t, config.Tokens["vimeo"], 0)
+}
+
 func TestApplyDefaults(t *testing.T) {
 	const file = `
 [server]

pkg/config/toml.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type Duration struct {
+	time.Duration
+}
+
+func (d *Duration) UnmarshalText(text []byte) error {
+	res, err := time.ParseDuration(string(text))
+	if err != nil {
+		return err
+	}
+
+	*d = Duration{res}
+	return nil
+}
+
+// StringSlice is a toml extension that lets you to specify either a string
+// value (a slice with just one element) or a string slice.
+type StringSlice []string
+
+func (s *StringSlice) UnmarshalTOML(decode func(interface{}) error) error {
+	var single string
+	if err := decode(&single); err == nil {
+		*s = []string{single}
+		return nil
+	}
+
+	var slice []string
+	if err := decode(&slice); err == nil {
+		*s = slice
+		return nil
+	}
+
+	return errors.New("failed to decode string (slice) field")
+}

pkg/feed/common.go

@@ -1,45 +0,0 @@
-package feed
-
-import (
-	"context"
-
-	"github.com/pkg/errors"
-
-	"github.com/mxpv/podsync/pkg/config"
-	"github.com/mxpv/podsync/pkg/model"
-)
-
-type Builder interface {
-	Build(ctx context.Context, cfg *config.Feed) (*model.Feed, error)
-}
-
-func New(ctx context.Context, cfg *config.Feed, tokens config.Tokens) (Builder, error) {
-	var (
-		provider Builder
-		err      error
-	)
-
-	info, err := ParseURL(cfg.URL)
-	if err != nil {
-		return nil, err
-	}
-
-	switch info.Provider {
-	case model.ProviderYoutube:
-		provider, err = NewYouTubeBuilder(tokens.YouTube)
-	case model.ProviderVimeo:
-		provider, err = NewVimeoBuilder(ctx, tokens.Vimeo)
-	default:
-		return nil, errors.Errorf("unsupported provider %q", info.Provider)
-	}
-
-	return provider, err
-}
-
-type feedProvider interface {
-	GetFeed(ctx context.Context, feedID string) (*model.Feed, error)
-}
-
-type urlProvider interface {
-	URL(ctx context.Context, ns string, fileName string) (string, error)
-}

pkg/feed/deps.go

@@ -0,0 +1,17 @@
+//go:generate mockgen -source=deps.go -destination=deps_mock_test.go -package=feed
+
+package feed
+
+import (
+	"context"
+
+	"github.com/mxpv/podsync/pkg/model"
+)
+
+type feedProvider interface {
+	GetFeed(ctx context.Context, feedID string) (*model.Feed, error)
+}
+
+type urlProvider interface {
+	URL(ctx context.Context, ns string, fileName string) (string, error)
+}

pkg/feed/deps_mock_test.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: common.go
+// Source: deps.go
 
 // Package feed is a generated GoMock package.
 package feed
@@ -7,49 +7,10 @@ package feed
 import (
 	context "context"
 	gomock "github.com/golang/mock/gomock"
-	config "github.com/mxpv/podsync/pkg/config"
 	model "github.com/mxpv/podsync/pkg/model"
 	reflect "reflect"
 )
 
-// MockBuilder is a mock of Builder interface
-type MockBuilder struct {
-	ctrl     *gomock.Controller
-	recorder *MockBuilderMockRecorder
-}
-
-// MockBuilderMockRecorder is the mock recorder for MockBuilder
-type MockBuilderMockRecorder struct {
-	mock *MockBuilder
-}
-
-// NewMockBuilder creates a new mock instance
-func NewMockBuilder(ctrl *gomock.Controller) *MockBuilder {
-	mock := &MockBuilder{ctrl: ctrl}
-	mock.recorder = &MockBuilderMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockBuilder) EXPECT() *MockBuilderMockRecorder {
-	return m.recorder
-}
-
-// Build mocks base method
-func (m *MockBuilder) Build(ctx context.Context, cfg *config.Feed) (*model.Feed, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Build", ctx, cfg)
-	ret0, _ := ret[0].(*model.Feed)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Build indicates an expected call of Build
-func (mr *MockBuilderMockRecorder) Build(ctx, cfg interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockBuilder)(nil).Build), ctx, cfg)
-}
-
 // MockfeedProvider is a mock of feedProvider interface
 type MockfeedProvider struct {
 	ctrl     *gomock.Controller

pkg/feed/key.go

@@ -0,0 +1,62 @@
+package feed
+
+import (
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+type KeyProvider interface {
+	Get() string
+}
+
+func NewKeyProvider(keys []string) (KeyProvider, error) {
+	switch len(keys) {
+	case 0:
+		return nil, errors.New("no keys")
+	case 1:
+		return NewFixedKey(keys[0])
+	default:
+		return NewRotatedKeys(keys)
+	}
+}
+
+type FixedKeyProvider struct {
+	key string
+}
+
+func NewFixedKey(key string) (KeyProvider, error) {
+	if key == "" {
+		return nil, errors.New("key can't be empty")
+	}
+
+	return &FixedKeyProvider{key: key}, nil
+}
+
+func (p FixedKeyProvider) Get() string {
+	return p.key
+}
+
+type RotatedKeyProvider struct {
+	keys  []string
+	lock  sync.Mutex
+	index int
+}
+
+func NewRotatedKeys(keys []string) (KeyProvider, error) {
+	if len(keys) < 2 {
+		return nil, errors.Errorf("at least 2 keys required (got %d)", len(keys))
+	}
+
+	return &RotatedKeyProvider{keys: keys, index: 0}, nil
+}
+
+func (p *RotatedKeyProvider) Get() string {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	current := p.index % len(p.keys)
+	p.index++
+
+	return p.keys[current]
+}

pkg/feed/key_test.go

@@ -0,0 +1,26 @@
+package feed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFixedKey(t *testing.T) {
+	key, err := NewFixedKey("123")
+	assert.NoError(t, err)
+
+	assert.EqualValues(t, "123", key.Get())
+	assert.EqualValues(t, "123", key.Get())
+}
+
+func TestNewRotatedKeys(t *testing.T) {
+	key, err := NewRotatedKeys([]string{"123", "456"})
+	assert.NoError(t, err)
+
+	assert.EqualValues(t, "123", key.Get())
+	assert.EqualValues(t, "456", key.Get())
+
+	assert.EqualValues(t, "123", key.Get())
+	assert.EqualValues(t, "456", key.Get())
+}

pkg/feed/opml_test.go

@@ -1,5 +1,3 @@
-//go:generate mockgen -source=common.go -destination=common_mock_test.go -package=feed
-
 package feed
 
 import (