Decouple config package

cmd/podsync/config.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+
+	"github.com/mxpv/podsync/pkg/db"
+	"github.com/mxpv/podsync/pkg/feed"
+	"github.com/mxpv/podsync/pkg/model"
+	"github.com/mxpv/podsync/pkg/ytdl"
+)
+
+type ServerConfig struct {
+	// Hostname to use for download links
+	Hostname string `toml:"hostname"`
+	// Port is a server port to listen to
+	Port int `toml:"port"`
+	// Bind a specific IP addresses for server
+	// "*": bind all IP addresses which is default option
+	// localhost or 127.0.0.1  bind a single IPv4 address
+	BindAddress string `toml:"bind_address"`
+	// Specify path for reverse proxy and only [A-Za-z0-9]
+	Path string `toml:"path"`
+	// DataDir is a path to a directory to keep XML feeds and downloaded episodes,
+	// that will be available to user via web server for download.
+	DataDir string `toml:"data_dir"`
+}
+
+type Log struct {
+	// Filename to write the log to (instead of stdout)
+	Filename string `toml:"filename"`
+	// MaxSize is the maximum size of the log file in MB
+	MaxSize int `toml:"max_size"`
+	// MaxBackups is the maximum number of log file backups to keep after rotation
+	MaxBackups int `toml:"max_backups"`
+	// MaxAge is the maximum number of days to keep the logs for
+	MaxAge int `toml:"max_age"`
+	// Compress old backups
+	Compress bool `toml:"compress"`
+}
+
+type Config struct {
+	// Server is the web server configuration
+	Server ServerConfig `toml:"server"`
+	// Log is the optional logging configuration
+	Log Log `toml:"log"`
+	// Database configuration
+	Database db.Config `toml:"database"`
+	// Feeds is a list of feeds to host by this app.
+	// ID will be used as feed ID in http://podsync.net/{FEED_ID}.xml
+	Feeds map[string]*feed.Config
+	// Tokens is API keys to use to access YouTube/Vimeo APIs.
+	Tokens map[model.Provider][]string `toml:"tokens"`
+	// Downloader (youtube-dl) configuration
+	Downloader ytdl.Config `toml:"downloader"`
+}
+
+// LoadConfig loads TOML configuration from a file path
+func LoadConfig(path string) (*Config, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read config file: %s", path)
+	}
+
+	config := Config{}
+	if err := toml.Unmarshal(data, &config); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal toml")
+	}
+
+	for id, f := range config.Feeds {
+		f.ID = id
+	}
+
+	config.applyDefaults(path)
+
+	if err := config.validate(); err != nil {
+		return nil, err
+	}
+
+	return &config, nil
+}
+
+func (c *Config) validate() error {
+	var result *multierror.Error
+
+	if c.Server.DataDir == "" {
+		result = multierror.Append(result, errors.New("data directory is required"))
+	}
+
+	if c.Server.Path != "" {
+		var pathReg = regexp.MustCompile(model.PathRegex)
+		if !pathReg.MatchString(c.Server.Path) {
+			result = multierror.Append(result, errors.Errorf("Server handle path must be match %s or empty", model.PathRegex))
+		}
+	}
+
+	if len(c.Feeds) == 0 {
+		result = multierror.Append(result, errors.New("at least one feed must be specified"))
+	}
+
+	for id, f := range c.Feeds {
+		if f.URL == "" {
+			result = multierror.Append(result, errors.Errorf("URL is required for %q", id))
+		}
+	}
+
+	return result.ErrorOrNil()
+}
+
+func (c *Config) applyDefaults(configPath string) {
+	if c.Server.Hostname == "" {
+		if c.Server.Port != 0 && c.Server.Port != 80 {
+			c.Server.Hostname = fmt.Sprintf("http://localhost:%d", c.Server.Port)
+		} else {
+			c.Server.Hostname = "http://localhost"
+		}
+	}
+
+	if c.Log.Filename != "" {
+		if c.Log.MaxSize == 0 {
+			c.Log.MaxSize = model.DefaultLogMaxSize
+		}
+		if c.Log.MaxAge == 0 {
+			c.Log.MaxAge = model.DefaultLogMaxAge
+		}
+		if c.Log.MaxBackups == 0 {
+			c.Log.MaxBackups = model.DefaultLogMaxBackups
+		}
+	}
+
+	if c.Database.Dir == "" {
+		c.Database.Dir = filepath.Join(filepath.Dir(configPath), "db")
+	}
+
+	for _, feed := range c.Feeds {
+		if feed.UpdatePeriod == 0 {
+			feed.UpdatePeriod = model.DefaultUpdatePeriod
+		}
+
+		if feed.Quality == "" {
+			feed.Quality = model.DefaultQuality
+		}
+
+		if feed.Custom.CoverArtQuality == "" {
+			feed.Custom.CoverArtQuality = model.DefaultQuality
+		}
+
+		if feed.Format == "" {
+			feed.Format = model.DefaultFormat
+		}
+
+		if feed.PageSize == 0 {
+			feed.PageSize = model.DefaultPageSize
+		}
+
+		if feed.PlaylistSort == "" {
+			feed.PlaylistSort = model.SortingAsc
+		}
+	}
+}

cmd/podsync/config_test.go

@@ -0,0 +1,241 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mxpv/podsync/pkg/model"
+)
+
+func TestLoadConfig(t *testing.T) {
+	const file = `
+[tokens]
+youtube = ["123"]
+vimeo = ["321", "456"]
+
+[server]
+port = 80
+data_dir = "test/data/"
+
+[database]
+dir = "/home/user/db/"
+
+[downloader]
+self_update = true
+timeout = 15
+
+[feeds]
+  [feeds.XYZ]
+  url = "https://youtube.com/watch?v=ygIUF678y40"
+  page_size = 48
+  update_period = "5h"
+  format = "audio"
+  quality = "low"
+  filters = { title = "regex for title here" }
+  playlist_sort = "desc"
+  clean = { keep_last = 10 }
+  	[feeds.XYZ.custom]
+	cover_art = "http://img"
+	cover_art_quality = "high"
+	category = "TV"
+	subcategories = ["1", "2"]
+	explicit = true
+	lang = "en"
+	author = "Mrs. Smith (mrs@smith.org)"
+	ownerName = "Mrs. Smith"
+	ownerEmail = "mrs@smith.org"
+`
+	path := setup(t, file)
+	defer os.Remove(path)
+
+	config, err := LoadConfig(path)
+	assert.NoError(t, err)
+	require.NotNil(t, config)
+
+	assert.Equal(t, "test/data/", config.Server.DataDir)
+	assert.EqualValues(t, 80, config.Server.Port)
+
+	assert.Equal(t, "/home/user/db/", config.Database.Dir)
+
+	require.Len(t, config.Tokens["youtube"], 1)
+	assert.Equal(t, "123", config.Tokens["youtube"][0])
+	require.Len(t, config.Tokens["vimeo"], 2)
+	assert.Equal(t, "321", config.Tokens["vimeo"][0])
+	assert.Equal(t, "456", config.Tokens["vimeo"][1])
+
+	assert.Len(t, config.Feeds, 1)
+	feed, ok := config.Feeds["XYZ"]
+	assert.True(t, ok)
+	assert.Equal(t, "https://youtube.com/watch?v=ygIUF678y40", feed.URL)
+	assert.EqualValues(t, 48, feed.PageSize)
+	assert.EqualValues(t, 5*time.Hour, feed.UpdatePeriod)
+	assert.EqualValues(t, "audio", feed.Format)
+	assert.EqualValues(t, "low", feed.Quality)
+	assert.EqualValues(t, "regex for title here", feed.Filters.Title)
+	assert.EqualValues(t, 10, feed.Clean.KeepLast)
+	assert.EqualValues(t, model.SortingDesc, feed.PlaylistSort)
+
+	assert.EqualValues(t, "http://img", feed.Custom.CoverArt)
+	assert.EqualValues(t, "high", feed.Custom.CoverArtQuality)
+	assert.EqualValues(t, "TV", feed.Custom.Category)
+	assert.True(t, feed.Custom.Explicit)
+	assert.EqualValues(t, "en", feed.Custom.Language)
+	assert.EqualValues(t, "Mrs. Smith (mrs@smith.org)", feed.Custom.Author)
+	assert.EqualValues(t, "Mrs. Smith", feed.Custom.OwnerName)
+	assert.EqualValues(t, "mrs@smith.org", feed.Custom.OwnerEmail)
+
+	assert.EqualValues(t, feed.Custom.Subcategories, []string{"1", "2"})
+
+	assert.Nil(t, config.Database.Badger)
+
+	assert.True(t, config.Downloader.SelfUpdate)
+	assert.EqualValues(t, 15, config.Downloader.Timeout)
+}
+
+func TestLoadEmptyKeyList(t *testing.T) {
+	const file = `
+[tokens]
+vimeo = []
+
+[server]
+data_dir = "/data"
+[feeds]
+  [feeds.A]
+  url = "https://youtube.com/watch?v=ygIUF678y40"
+`
+	path := setup(t, file)
+	defer os.Remove(path)
+
+	config, err := LoadConfig(path)
+	assert.NoError(t, err)
+	require.NotNil(t, config)
+
+	require.Len(t, config.Tokens, 1)
+	require.Len(t, config.Tokens["vimeo"], 0)
+}
+
+func TestApplyDefaults(t *testing.T) {
+	const file = `
+[server]
+data_dir = "/data"
+
+[feeds]
+  [feeds.A]
+  url = "https://youtube.com/watch?v=ygIUF678y40"
+`
+	path := setup(t, file)
+	defer os.Remove(path)
+
+	config, err := LoadConfig(path)
+	assert.NoError(t, err)
+	assert.NotNil(t, config)
+
+	assert.Len(t, config.Feeds, 1)
+	feed, ok := config.Feeds["A"]
+	require.True(t, ok)
+
+	assert.EqualValues(t, feed.UpdatePeriod, model.DefaultUpdatePeriod)
+	assert.EqualValues(t, feed.PageSize, 50)
+	assert.EqualValues(t, feed.Quality, "high")
+	assert.EqualValues(t, feed.Custom.CoverArtQuality, "high")
+	assert.EqualValues(t, feed.Format, "video")
+}
+
+func TestHttpServerListenAddress(t *testing.T) {
+	const file = `
+[server]
+bind_address = "172.20.10.2"
+port = 8080
+path = "test"
+data_dir = "/data"
+
+[feeds]
+  [feeds.A]
+  url = "https://youtube.com/watch?v=ygIUF678y40"
+
+[database]
+  badger = { truncate = true, file_io = true }
+`
+	path := setup(t, file)
+	defer os.Remove(path)
+
+	config, err := LoadConfig(path)
+	assert.NoError(t, err)
+	require.NotNil(t, config)
+	require.NotNil(t, config.Server.BindAddress)
+	require.NotNil(t, config.Server.Path)
+}
+
+func TestDefaultHostname(t *testing.T) {
+	cfg := Config{
+		Server: ServerConfig{},
+	}
+
+	t.Run("empty hostname", func(t *testing.T) {
+		cfg.applyDefaults("")
+		assert.Equal(t, "http://localhost", cfg.Server.Hostname)
+	})
+
+	t.Run("empty hostname with port", func(t *testing.T) {
+		cfg.Server.Hostname = ""
+		cfg.Server.Port = 7979
+		cfg.applyDefaults("")
+		assert.Equal(t, "http://localhost:7979", cfg.Server.Hostname)
+	})
+
+	t.Run("skip overwrite", func(t *testing.T) {
+		cfg.Server.Hostname = "https://my.host:4443"
+		cfg.Server.Port = 80
+		cfg.applyDefaults("")
+		assert.Equal(t, "https://my.host:4443", cfg.Server.Hostname)
+	})
+}
+
+func TestDefaultDatabasePath(t *testing.T) {
+	cfg := Config{}
+	cfg.applyDefaults("/home/user/podsync/config.toml")
+	assert.Equal(t, "/home/user/podsync/db", cfg.Database.Dir)
+}
+
+func TestLoadBadgerConfig(t *testing.T) {
+	const file = `
+[server]
+data_dir = "/data"
+
+[feeds]
+  [feeds.A]
+  url = "https://youtube.com/watch?v=ygIUF678y40"
+
+[database]
+  badger = { truncate = true, file_io = true }
+`
+	path := setup(t, file)
+	defer os.Remove(path)
+
+	config, err := LoadConfig(path)
+	assert.NoError(t, err)
+	require.NotNil(t, config)
+	require.NotNil(t, config.Database.Badger)
+
+	assert.True(t, config.Database.Badger.Truncate)
+	assert.True(t, config.Database.Badger.FileIO)
+}
+
+func setup(t *testing.T, file string) string {
+	t.Helper()
+
+	f, err := ioutil.TempFile("", "")
+	require.NoError(t, err)
+
+	defer f.Close()
+
+	_, err = f.WriteString(file)
+	require.NoError(t, err)
+
+	return f.Name()
+}

cmd/podsync/main.go

@@ -10,12 +10,12 @@ import (
 	"time"
 
 	"github.com/jessevdk/go-flags"
+	"github.com/mxpv/podsync/pkg/feed"
 	"github.com/robfig/cron/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/natefinch/lumberjack.v2"
 
-	"github.com/mxpv/podsync/pkg/config"
 	"github.com/mxpv/podsync/pkg/db"
 	"github.com/mxpv/podsync/pkg/fs"
 	"github.com/mxpv/podsync/pkg/ytdl"
@@ -75,7 +75,7 @@ func main() {
 
 	// Load TOML file
 	log.Debugf("loading configuration %q", opts.ConfigPath)
-	cfg, err := config.LoadConfig(opts.ConfigPath)
+	cfg, err := LoadConfig(opts.ConfigPath)
 	if err != nil {
 		log.WithError(err).Fatal("failed to load configuration file")
 	}
@@ -121,7 +121,7 @@ func main() {
 	}
 
 	// Queue of feeds to update
-	updates := make(chan *config.Feed, 16)
+	updates := make(chan *feed.Config, 16)
 	defer close(updates)
 
 	// Create Cron

cmd/podsync/server.go

@@ -5,15 +5,13 @@ import (
 	"net/http"
 
 	log "github.com/sirupsen/logrus"
-
-	"github.com/mxpv/podsync/pkg/config"
 )
 
 type Server struct {
 	http.Server
 }
 
-func NewServer(cfg *config.Config, storage http.FileSystem) *Server {
+func NewServer(cfg *Config, storage http.FileSystem) *Server {
 	port := cfg.Server.Port
 	if port == 0 {
 		port = 8080

cmd/podsync/updater.go

@@ -15,7 +15,6 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/mxpv/podsync/pkg/builder"
-	"github.com/mxpv/podsync/pkg/config"
 	"github.com/mxpv/podsync/pkg/db"
 	"github.com/mxpv/podsync/pkg/feed"
 	"github.com/mxpv/podsync/pkg/fs"
@@ -24,18 +23,18 @@ import (
 )
 
 type Downloader interface {
-	Download(ctx context.Context, feedConfig *config.Feed, episode *model.Episode) (io.ReadCloser, error)
+	Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (io.ReadCloser, error)
 }
 
 type Updater struct {
-	config     *config.Config
+	config     *Config
 	downloader Downloader
 	db         db.Storage
 	fs         fs.Storage
 	keys       map[model.Provider]feed.KeyProvider
 }
 
-func NewUpdater(config *config.Config, downloader Downloader, db db.Storage, fs fs.Storage) (*Updater, error) {
+func NewUpdater(config *Config, downloader Downloader, db db.Storage, fs fs.Storage) (*Updater, error) {
 	keys := map[model.Provider]feed.KeyProvider{}
 
 	for name, list := range config.Tokens {
@@ -55,7 +54,7 @@ func NewUpdater(config *config.Config, downloader Downloader, db db.Storage, fs
 	}, nil
 }
 
-func (u *Updater) Update(ctx context.Context, feedConfig *config.Feed) error {
+func (u *Updater) Update(ctx context.Context, feedConfig *feed.Config) error {
 	log.WithFields(log.Fields{
 		"feed_id": feedConfig.ID,
 		"format":  feedConfig.Format,
@@ -90,7 +89,7 @@ func (u *Updater) Update(ctx context.Context, feedConfig *config.Feed) error {
 }
 
 // updateFeed pulls API for new episodes and saves them to database
-func (u *Updater) updateFeed(ctx context.Context, feedConfig *config.Feed) error {
+func (u *Updater) updateFeed(ctx context.Context, feedConfig *feed.Config) error {
 	info, err := builder.ParseURL(feedConfig.URL)
 	if err != nil {
 		return errors.Wrapf(err, "failed to parse URL: %s", feedConfig.URL)
@@ -162,7 +161,7 @@ func (u *Updater) matchRegexpFilter(pattern, str string, negative bool, logger l
 	return true
 }
 
-func (u *Updater) matchFilters(episode *model.Episode, filters *config.Filters) bool {
+func (u *Updater) matchFilters(episode *model.Episode, filters *feed.Filters) bool {
 	logger := log.WithFields(log.Fields{"episode_id": episode.ID})
 	if !u.matchRegexpFilter(filters.Title, episode.Title, false, logger.WithField("filter", "title")) {
 		return false
@@ -181,7 +180,7 @@ func (u *Updater) matchFilters(episode *model.Episode, filters *config.Filters)
 	return true
 }
 
-func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed) error {
+func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *feed.Config) error {
 	var (
 		feedID       = feedConfig.ID
 		downloadList []*model.Episode
@@ -308,7 +307,7 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed)
 	return nil
 }
 
-func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {
+func (u *Updater) buildXML(ctx context.Context, feedConfig *feed.Config) error {
 	f, err := u.db.GetFeed(ctx, feedConfig.ID)
 	if err != nil {
 		return err
@@ -336,7 +335,7 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {
 func (u *Updater) buildOPML(ctx context.Context) error {
 	// Build OPML with data received from builder
 	log.Debug("building podcast OPML")
-	opml, err := feed.BuildOPML(ctx, u.config, u.db, u.config.Server.Hostname)
+	opml, err := feed.BuildOPML(ctx, u.config.Feeds, u.db, u.config.Server.Hostname)
 	if err != nil {
 		return err
 	}
@@ -353,7 +352,7 @@ func (u *Updater) buildOPML(ctx context.Context) error {
 	return nil
 }
 
-func (u *Updater) cleanup(ctx context.Context, feedConfig *config.Feed) error {
+func (u *Updater) cleanup(ctx context.Context, feedConfig *feed.Config) error {
 	var (
 		feedID = feedConfig.ID
 		logger = log.WithField("feed_id", feedID)