diff --git a/cmd/podsync/config.go b/cmd/podsync/config.go index 70cfc4b..cf33531 100644 --- a/cmd/podsync/config.go +++ b/cmd/podsync/config.go @@ -9,9 +9,11 @@ import ( "github.com/hashicorp/go-multierror" "github.com/pelletier/go-toml" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "github.com/mxpv/podsync/pkg/db" "github.com/mxpv/podsync/pkg/feed" + "github.com/mxpv/podsync/pkg/fs" "github.com/mxpv/podsync/pkg/model" "github.com/mxpv/podsync/pkg/ytdl" "github.com/mxpv/podsync/services/web" @@ -21,7 +23,7 @@ type Config struct { // Server is the web server configuration Server web.Config `toml:"server"` // S3 is the optional configuration for S3-compatible storage provider - S3 S3 `toml:"s3"` + Storage fs.Config `toml:"storage"` // Log is the optional logging configuration Log Log `toml:"log"` // Database configuration @@ -35,15 +37,6 @@ type Config struct { Downloader ytdl.Config `toml:"downloader"` } -type S3 struct { - // S3 Bucket to store files - Bucket string `toml:"bucket"` - // Region of the S3 service - Region string `toml:"region"` - // EndpointURL is an HTTP endpoint of the S3 API - EndpointURL string `toml:"endpoint_url"` -} - type Log struct { // Filename to write the log to (instead of stdout) Filename string `toml:"filename"` @@ -85,8 +78,17 @@ func LoadConfig(path string) (*Config, error) { func (c *Config) validate() error { var result *multierror.Error - if c.Server.DataDir == "" && c.S3.Bucket == "" { - result = multierror.Append(result, errors.New("data directory or S3 bucket is required")) + if c.Server.DataDir != "" { + log.Warnf(`server.data_dir is deprecated, and will be removed in a future release. Use the following config instead: + +[storage] + [storage.local] + data_dir = "%s" + +`, c.Server.DataDir) + if c.Storage.Local.DataDir == "" { + c.Storage.Local.DataDir = c.Server.DataDir + } } if c.Server.Path != "" { @@ -96,6 +98,19 @@ func (c *Config) validate() error { } } + switch c.Storage.Type { + case "local": + if c.Storage.Local.DataDir == "" { + result = multierror.Append(result, errors.New("data directory is required for local storage")) + } + case "s3": + if c.Storage.S3.EndpointURL == "" || c.Storage.S3.Region == "" || c.Storage.S3.Bucket == "" { + result = multierror.Append(result, errors.New("S3 storage requires endpoint_url, region and bucket to be set")) + } + default: + result = multierror.Append(result, errors.Errorf("unknown storage type: %s", c.Storage.Type)) + } + if len(c.Feeds) == 0 { result = multierror.Append(result, errors.New("at least one feed must be specified")) } @@ -118,6 +133,10 @@ func (c *Config) applyDefaults(configPath string) { } } + if c.Storage.Type == "" { + c.Storage.Type = "local" + } + if c.Log.Filename != "" { if c.Log.MaxSize == 0 { c.Log.MaxSize = model.DefaultLogMaxSize diff --git a/cmd/podsync/main.go b/cmd/podsync/main.go index a73923e..f42250d 100644 --- a/cmd/podsync/main.go +++ b/cmd/podsync/main.go @@ -116,10 +116,13 @@ func main() { }() var storage fs.Storage - if cfg.S3.Bucket != "" { - storage, err = fs.NewS3(cfg.S3.EndpointURL, cfg.S3.Region, cfg.S3.Bucket) - } else { - storage, err = fs.NewLocal(cfg.Server.DataDir) + switch cfg.Storage.Type { + case "local": + storage, err = fs.NewLocal(cfg.Storage.Local.DataDir) + case "s3": + storage, err = fs.NewS3(cfg.Storage.S3) + default: + log.Fatalf("unknown storage type: %s", cfg.Storage.Type) } if err != nil { log.WithError(err).Fatal("failed to open storage") @@ -218,7 +221,7 @@ func main() { } }) - if cfg.S3.Bucket != "" { + if cfg.Storage.Type == "s3" { return // S3 content is hosted externally } diff --git a/pkg/fs/local.go b/pkg/fs/local.go index 7118e82..687ba54 100644 --- a/pkg/fs/local.go +++ b/pkg/fs/local.go @@ -11,6 +11,11 @@ import ( log "github.com/sirupsen/logrus" ) +// LocalConfig is the storage configuration for local file system +type LocalConfig struct { + DataDir string `yaml:"data_dir"` +} + // Local implements local file storage type Local struct { rootDir string diff --git a/pkg/fs/s3.go b/pkg/fs/s3.go index cb1ca80..688d3a0 100644 --- a/pkg/fs/s3.go +++ b/pkg/fs/s3.go @@ -16,6 +16,16 @@ import ( log "github.com/sirupsen/logrus" ) +// S3Config is the configuration for a S3-compatible storage provider +type S3Config struct { + // S3 Bucket to store files + Bucket string `toml:"bucket"` + // Region of the S3 service + Region string `toml:"region"` + // EndpointURL is an HTTP endpoint of the S3 API + EndpointURL string `toml:"endpoint_url"` +} + // S3 implements file storage for S3-compatible providers. type S3 struct { api s3iface.S3API @@ -23,10 +33,10 @@ type S3 struct { bucket string } -func NewS3(endpointURL, region, bucket string) (*S3, error) { +func NewS3(c S3Config) (*S3, error) { cfg := aws.NewConfig(). - WithEndpoint(endpointURL). - WithRegion(region). + WithEndpoint(c.EndpointURL). + WithRegion(c.Region). WithLogger(s3logger{}). WithLogLevel(aws.LogDebug) sess, err := session.NewSessionWithOptions(session.Options{Config: *cfg}) @@ -36,7 +46,7 @@ func NewS3(endpointURL, region, bucket string) (*S3, error) { return &S3{ api: s3.New(sess), uploader: s3manager.NewUploader(sess), - bucket: bucket, + bucket: c.Bucket, }, nil } diff --git a/pkg/fs/storage.go b/pkg/fs/storage.go index bc56f74..efdf190 100644 --- a/pkg/fs/storage.go +++ b/pkg/fs/storage.go @@ -20,3 +20,11 @@ type Storage interface { // Size returns a storage object's size in bytes Size(ctx context.Context, name string) (int64, error) } + +// Config is a configuration for the file storage backend +type Config struct { + // Type is the type of file system to use + Type string `toml:"type"` + Local LocalConfig `toml:"local"` + S3 S3Config `toml:"s3"` +}