Add storage config section

This also deprecate `server.data_dir` in favor of `storage.local.data_dir`

cmd/podsync/config.go

@@ -9,9 +9,11 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/mxpv/podsync/pkg/db"
 	"github.com/mxpv/podsync/pkg/feed"
+	"github.com/mxpv/podsync/pkg/fs"
 	"github.com/mxpv/podsync/pkg/model"
 	"github.com/mxpv/podsync/pkg/ytdl"
 	"github.com/mxpv/podsync/services/web"
@@ -21,7 +23,7 @@ type Config struct {
 	// Server is the web server configuration
 	Server web.Config `toml:"server"`
 	// S3 is the optional configuration for S3-compatible storage provider
-	S3 S3 `toml:"s3"`
+	Storage fs.Config `toml:"storage"`
 	// Log is the optional logging configuration
 	Log Log `toml:"log"`
 	// Database configuration
@@ -35,15 +37,6 @@ type Config struct {
 	Downloader ytdl.Config `toml:"downloader"`
 }
 
-type S3 struct {
-	// S3 Bucket to store files
-	Bucket string `toml:"bucket"`
-	// Region of the S3 service
-	Region string `toml:"region"`
-	// EndpointURL is an HTTP endpoint of the S3 API
-	EndpointURL string `toml:"endpoint_url"`
-}
-
 type Log struct {
 	// Filename to write the log to (instead of stdout)
 	Filename string `toml:"filename"`
@@ -85,8 +78,17 @@ func LoadConfig(path string) (*Config, error) {
 func (c *Config) validate() error {
 	var result *multierror.Error
 
-	if c.Server.DataDir == "" && c.S3.Bucket == "" {
-		result = multierror.Append(result, errors.New("data directory or S3 bucket is required"))
+	if c.Server.DataDir != "" {
+		log.Warnf(`server.data_dir is deprecated, and will be removed in a future release. Use the following config instead:
+
+[storage]
+  [storage.local]
+  data_dir = "%s"
+
+`, c.Server.DataDir)
+		if c.Storage.Local.DataDir == "" {
+			c.Storage.Local.DataDir = c.Server.DataDir
+		}
 	}
 
 	if c.Server.Path != "" {
@@ -96,6 +98,19 @@ func (c *Config) validate() error {
 		}
 	}
 
+	switch c.Storage.Type {
+	case "local":
+		if c.Storage.Local.DataDir == "" {
+			result = multierror.Append(result, errors.New("data directory is required for local storage"))
+		}
+	case "s3":
+		if c.Storage.S3.EndpointURL == "" || c.Storage.S3.Region == "" || c.Storage.S3.Bucket == "" {
+			result = multierror.Append(result, errors.New("S3 storage requires endpoint_url, region and bucket to be set"))
+		}
+	default:
+		result = multierror.Append(result, errors.Errorf("unknown storage type: %s", c.Storage.Type))
+	}
+
 	if len(c.Feeds) == 0 {
 		result = multierror.Append(result, errors.New("at least one feed must be specified"))
 	}
@@ -118,6 +133,10 @@ func (c *Config) applyDefaults(configPath string) {
 		}
 	}
 
+	if c.Storage.Type == "" {
+		c.Storage.Type = "local"
+	}
+
 	if c.Log.Filename != "" {
 		if c.Log.MaxSize == 0 {
 			c.Log.MaxSize = model.DefaultLogMaxSize

cmd/podsync/main.go

@@ -116,10 +116,13 @@ func main() {
 	}()
 
 	var storage fs.Storage
-	if cfg.S3.Bucket != "" {
-		storage, err = fs.NewS3(cfg.S3.EndpointURL, cfg.S3.Region, cfg.S3.Bucket)
-	} else {
-		storage, err = fs.NewLocal(cfg.Server.DataDir)
+	switch cfg.Storage.Type {
+	case "local":
+		storage, err = fs.NewLocal(cfg.Storage.Local.DataDir)
+	case "s3":
+		storage, err = fs.NewS3(cfg.Storage.S3)
+	default:
+		log.Fatalf("unknown storage type: %s", cfg.Storage.Type)
 	}
 	if err != nil {
 		log.WithError(err).Fatal("failed to open storage")
@@ -218,7 +221,7 @@ func main() {
 		}
 	})
 
-	if cfg.S3.Bucket != "" {
+	if cfg.Storage.Type == "s3" {
 		return // S3 content is hosted externally
 	}
 

pkg/fs/local.go

@@ -11,6 +11,11 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// LocalConfig is the storage configuration for local file system
+type LocalConfig struct {
+	DataDir string `yaml:"data_dir"`
+}
+
 // Local implements local file storage
 type Local struct {
 	rootDir string

pkg/fs/s3.go

@@ -16,6 +16,16 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// S3Config is the configuration for a S3-compatible storage provider
+type S3Config struct {
+	// S3 Bucket to store files
+	Bucket string `toml:"bucket"`
+	// Region of the S3 service
+	Region string `toml:"region"`
+	// EndpointURL is an HTTP endpoint of the S3 API
+	EndpointURL string `toml:"endpoint_url"`
+}
+
 // S3 implements file storage for S3-compatible providers.
 type S3 struct {
 	api      s3iface.S3API
@@ -23,10 +33,10 @@ type S3 struct {
 	bucket   string
 }
 
-func NewS3(endpointURL, region, bucket string) (*S3, error) {
+func NewS3(c S3Config) (*S3, error) {
 	cfg := aws.NewConfig().
-		WithEndpoint(endpointURL).
-		WithRegion(region).
+		WithEndpoint(c.EndpointURL).
+		WithRegion(c.Region).
 		WithLogger(s3logger{}).
 		WithLogLevel(aws.LogDebug)
 	sess, err := session.NewSessionWithOptions(session.Options{Config: *cfg})
@@ -36,7 +46,7 @@ func NewS3(endpointURL, region, bucket string) (*S3, error) {
 	return &S3{
 		api:      s3.New(sess),
 		uploader: s3manager.NewUploader(sess),
-		bucket:   bucket,
+		bucket:   c.Bucket,
 	}, nil
 }
 

pkg/fs/storage.go

@@ -20,3 +20,11 @@ type Storage interface {
 	// Size returns a storage object's size in bytes
 	Size(ctx context.Context, name string) (int64, error)
 }
+
+// Config is a configuration for the file storage backend
+type Config struct {
+	// Type is the type of file system to use
+	Type  string      `toml:"type"`
+	Local LocalConfig `toml:"local"`
+	S3    S3Config    `toml:"s3"`
+}