mirror/mxpv-podsync
1
0
Fork 0
mirror of https://github.com/mxpv/podsync.git synced 2024-05-11 05:55:04 +00:00
Code Issues Releases Activity

Merge pull request #265 from mxpv/storage

Browse Source 
Refactor code to allow pluggable storages
This commit is contained in:
Maksym Pavlenko
2022-01-02 13:19:11 +02:00
committed by GitHub
parent 26f4b2a47a 12cdbf61db
commit ce2df20d8d
12 changed files with 121 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/podsync/main.go
+3 -4
View File
@@ -13,13 +13,12 @@ import (
"github.com/robfig/cron/v3"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"gopkg.in/natefinch/lumberjack.v2"
"github.com/mxpv/podsync/pkg/config"
"github.com/mxpv/podsync/pkg/db"
"github.com/mxpv/podsync/pkg/fs"
"github.com/mxpv/podsync/pkg/ytdl"
"gopkg.in/natefinch/lumberjack.v2"
)
type Opts struct {
@@ -107,7 +106,7 @@ func main() {
log.WithError(err).Fatal("failed to open database")
}
storage, err := fs.NewLocal(cfg.Server.DataDir, cfg.Server.Hostname)
storage, err := fs.NewLocal(cfg.Server.DataDir)
if err != nil {
log.WithError(err).Fatal("failed to open storage")
}
@@ -178,7 +177,7 @@ func main() {
})
// Run web server
srv := NewServer(cfg)
srv := NewServer(cfg, storage)
group.Go(func() error {
log.Infof("running listener at %s", srv.Addr)
cmd/podsync/server.go
+7 -5
View File
@@ -13,24 +13,26 @@ type Server struct {
http.Server
}
func NewServer(cfg *config.Config) *Server {
func NewServer(cfg *config.Config, storage http.FileSystem) *Server {
port := cfg.Server.Port
if port == 0 {
port = 8080
}
bindAddress := cfg.Server.BindAddress
if bindAddress == "*" {
bindAddress = ""
}
srv := Server{}
srv.Addr = fmt.Sprintf("%s:%d", bindAddress, port)
log.Debugf("using address: %s:%s", bindAddress, srv.Addr)
fs := http.FileServer(http.Dir(cfg.Server.DataDir))
path := cfg.Server.Path
http.Handle(fmt.Sprintf("/%s", path), fs)
log.Debugf("handle path: /%s", path)
fileServer := http.FileServer(storage)
log.Debugf("handle path: /%s", cfg.Server.Path)
http.Handle(fmt.Sprintf("/%s", cfg.Server.Path), fileServer)
return &srv
}
cmd/podsync/updater.go
+13 -8
View File
@@ -203,7 +203,7 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed)
// Limit the number of episodes downloaded at once
pageSize--
if pageSize <= 0 {
if pageSize < 0 {
return nil
}
@@ -235,7 +235,7 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed)
)
// Check whether episode already exists
size, err := u.fs.Size(ctx, feedID, episodeName)
size, err := fs.Size(u.fs, fmt.Sprintf("%s/%s", feedID, episodeName))
if err == nil {
logger.Infof("episode %q already exists on disk", episode.ID)
@@ -283,7 +283,7 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed)
}
logger.Debug("copying file")
fileSize, err := u.fs.Create(ctx, feedID, episodeName, tempFile)
fileSize, err := u.fs.Create(ctx, fmt.Sprintf("%s/%s", feedID, episodeName), tempFile)
tempFile.Close()
if err != nil {
logger.WithError(err).Error("failed to copy file")
@@ -316,7 +316,7 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {
// Build iTunes XML feed with data received from builder
log.Debug("building iTunes podcast feed")
podcast, err := feed.Build(ctx, f, feedConfig, u.fs)
podcast, err := feed.Build(ctx, f, feedConfig, u.config.Server.Hostname)
if err != nil {
return err
}
@@ -326,7 +326,7 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {
xmlName = fmt.Sprintf("%s.xml", feedConfig.ID)
)
if _, err := u.fs.Create(ctx, "", xmlName, reader); err != nil {
if _, err := u.fs.Create(ctx, xmlName, reader); err != nil {
return errors.Wrap(err, "failed to upload new XML feed")
}
@@ -336,7 +336,7 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {
func (u *Updater) buildOPML(ctx context.Context) error {
// Build OPML with data received from builder
log.Debug("building podcast OPML")
opml, err := feed.BuildOPML(ctx, u.config, u.db, u.fs)
opml, err := feed.BuildOPML(ctx, u.config, u.db, u.config.Server.Hostname)
if err != nil {
return err
}
@@ -346,7 +346,7 @@ func (u *Updater) buildOPML(ctx context.Context) error {
xmlName = fmt.Sprintf("%s.opml", "podsync")
)
if _, err := u.fs.Create(ctx, "", xmlName, reader); err != nil {
if _, err := u.fs.Create(ctx, xmlName, reader); err != nil {
return errors.Wrap(err, "failed to upload OPML")
}
@@ -388,7 +388,12 @@ func (u *Updater) cleanup(ctx context.Context, feedConfig *config.Feed) error {
for _, episode := range list[count:] {
logger.WithField("episode_id", episode.ID).Infof("deleting %q", episode.Title)
if err := u.fs.Delete(ctx, feedConfig.ID, feed.EpisodeName(feedConfig, episode)); err != nil {
var (
episodeName = feed.EpisodeName(feedConfig, episode)
path = fmt.Sprintf("%s/%s", feedConfig.ID, episodeName)
)
if err := u.fs.Delete(ctx, path); err != nil {
result = multierror.Append(result, errors.Wrapf(err, "failed to delete episode: %s", episode.ID))
continue
}
pkg/feed/deps.go
-4
View File
@@ -11,7 +11,3 @@ import (
type feedProvider interface {
GetFeed(ctx context.Context, feedID string) (*model.Feed, error)
}
type urlProvider interface {
URL(ctx context.Context, ns string, fileName string) (string, error)
}
pkg/feed/deps_mock_test.go
+8 -45
View File
@@ -6,35 +6,36 @@ package feed
import (
context "context"
reflect "reflect"
gomock "github.com/golang/mock/gomock"
model "github.com/mxpv/podsync/pkg/model"
reflect "reflect"
)
// MockfeedProvider is a mock of feedProvider interface
// MockfeedProvider is a mock of feedProvider interface.
type MockfeedProvider struct {
ctrl *gomock.Controller
recorder *MockfeedProviderMockRecorder
}
// MockfeedProviderMockRecorder is the mock recorder for MockfeedProvider
// MockfeedProviderMockRecorder is the mock recorder for MockfeedProvider.
type MockfeedProviderMockRecorder struct {
mock *MockfeedProvider
}
// NewMockfeedProvider creates a new mock instance
// NewMockfeedProvider creates a new mock instance.
func NewMockfeedProvider(ctrl *gomock.Controller) *MockfeedProvider {
mock := &MockfeedProvider{ctrl: ctrl}
mock.recorder = &MockfeedProviderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockfeedProvider) EXPECT() *MockfeedProviderMockRecorder {
return m.recorder
}
// GetFeed mocks base method
// GetFeed mocks base method.
func (m *MockfeedProvider) GetFeed(ctx context.Context, feedID string) (*model.Feed, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetFeed", ctx, feedID)
@@ -43,46 +44,8 @@ func (m *MockfeedProvider) GetFeed(ctx context.Context, feedID string) (*model.F
return ret0, ret1
}
// GetFeed indicates an expected call of GetFeed
// GetFeed indicates an expected call of GetFeed.
func (mr *MockfeedProviderMockRecorder) GetFeed(ctx, feedID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeed", reflect.TypeOf((*MockfeedProvider)(nil).GetFeed), ctx, feedID)
}
// MockurlProvider is a mock of urlProvider interface
type MockurlProvider struct {
ctrl *gomock.Controller
recorder *MockurlProviderMockRecorder
}
// MockurlProviderMockRecorder is the mock recorder for MockurlProvider
type MockurlProviderMockRecorder struct {
mock *MockurlProvider
}
// NewMockurlProvider creates a new mock instance
func NewMockurlProvider(ctrl *gomock.Controller) *MockurlProvider {
mock := &MockurlProvider{ctrl: ctrl}
mock.recorder = &MockurlProviderMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockurlProvider) EXPECT() *MockurlProviderMockRecorder {
return m.recorder
}
// URL mocks base method
func (m *MockurlProvider) URL(ctx context.Context, ns, fileName string) (string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "URL", ctx, ns, fileName)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// URL indicates an expected call of URL
func (mr *MockurlProviderMockRecorder) URL(ctx, ns, fileName interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "URL", reflect.TypeOf((*MockurlProvider)(nil).URL), ctx, ns, fileName)
}
pkg/feed/opml.go
+3 -7
View File
@@ -3,6 +3,7 @@ package feed
import (
"context"
"fmt"
"strings"
"github.com/gilliek/go-opml/opml"
"github.com/pkg/errors"
@@ -12,7 +13,7 @@ import (
"github.com/mxpv/podsync/pkg/model"
)
func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, provider urlProvider) (string, error) {
func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, hostname string) (string, error) {
doc := opml.OPML{Version: "1.0"}
doc.Head = opml.Head{Title: "Podsync feeds"}
doc.Body = opml.Body{}
@@ -31,16 +32,11 @@ func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, prov
continue
}
downloadURL, err := provider.URL(ctx, "", fmt.Sprintf("%s.xml", feed.ID))
if err != nil {
return "", errors.Wrapf(err, "failed to get feed URL for %q", feed.ID)
}
outline := opml.Outline{
Title: f.Title,
Text: f.Description,
Type: "rss",
XMLURL: downloadURL,
XMLURL: fmt.Sprintf("%s/%s.xml", strings.TrimRight(hostname, "/"), feed.ID),
}
doc.Body.Outlines = append(doc.Body.Outlines, outline)
pkg/feed/opml_test.go
+1 -4
View File
@@ -25,9 +25,6 @@ func TestBuildOPML(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
urlMock := NewMockurlProvider(ctrl)
urlMock.EXPECT().URL(gomock.Any(), "", "1.xml").Return("https://url/1.xml", nil)
dbMock := NewMockfeedProvider(ctrl)
dbMock.EXPECT().GetFeed(gomock.Any(), "1").Return(&model.Feed{Title: "1", Description: "desc"}, nil)
@@ -37,7 +34,7 @@ func TestBuildOPML(t *testing.T) {
},
}
out, err := BuildOPML(context.Background(), &cfg, dbMock, urlMock)
out, err := BuildOPML(context.Background(), &cfg, dbMock, "https://url/")
assert.NoError(t, err)
assert.Equal(t, expected, out)
}
pkg/feed/xml.go
+6 -6
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"sort"
"strconv"
"strings"
"time"
itunes "github.com/eduncan911/podcast"
@@ -30,7 +31,7 @@ func (p timeSlice) Swap(i, j int) {
p[i], p[j] = p[j], p[i]
}
func Build(ctx context.Context, feed *model.Feed, cfg *config.Feed, provider urlProvider) (*itunes.Podcast, error) {
func Build(_ctx context.Context, feed *model.Feed, cfg *config.Feed, hostname string) (*itunes.Podcast, error) {
const (
podsyncGenerator = "Podsync generator (support us at https://github.com/mxpv/podsync)"
defaultCategory = "TV & Film"
@@ -125,11 +126,10 @@ func Build(ctx context.Context, feed *model.Feed, cfg *config.Feed, provider url
enclosureType = itunes.MP3
}
episodeName := EpisodeName(cfg, episode)
downloadURL, err := provider.URL(ctx, cfg.ID, episodeName)
if err != nil {
return nil, errors.Wrapf(err, "failed to obtain download URL for: %s", episodeName)
}
var (
episodeName = EpisodeName(cfg, episode)
downloadURL = fmt.Sprintf("%s/%s/%s", strings.TrimRight(hostname, "/"), cfg.ID, episodeName)
)
item.AddEnclosure(downloadURL, enclosureType, episode.Size)
pkg/feed/xml_test.go
+21 -11
View File
@@ -4,27 +4,31 @@ import (
"context"
"testing"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
itunes "github.com/eduncan911/podcast"
"github.com/mxpv/podsync/pkg/config"
"github.com/mxpv/podsync/pkg/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBuildXML(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
urlMock := NewMockurlProvider(ctrl)
feed := model.Feed{}
feed := model.Feed{
Episodes: []*model.Episode{
{
ID: "1",
Status: model.EpisodeDownloaded,
Title: "title",
Description: "description",
},
},
}
cfg := config.Feed{
ID: "test",
Custom: config.Custom{Description: "description", Category: "Technology", Subcategories: []string{"Gadgets", "Podcasting"}},
}
out, err := Build(context.Background(), &feed, &cfg, urlMock)
out, err := Build(context.Background(), &feed, &cfg, "http://localhost/")
assert.NoError(t, err)
assert.EqualValues(t, "description", out.Description)
@@ -33,7 +37,13 @@ func TestBuildXML(t *testing.T) {
require.Len(t, out.ICategories, 1)
category := out.ICategories[0]
assert.EqualValues(t, "Technology", category.Text)
require.Len(t, category.ICategories, 2)
assert.EqualValues(t, "Gadgets", category.ICategories[0].Text)
assert.EqualValues(t, "Podcasting", category.ICategories[1].Text)
require.Len(t, out.Items, 1)
require.NotNil(t, out.Items[0].Enclosure)
assert.EqualValues(t, out.Items[0].Enclosure.URL, "http://localhost/test/1.mp4")
assert.EqualValues(t, out.Items[0].Enclosure.Type, itunes.MP4)
}
pkg/fs/local.go
+23 -60
View File
@@ -2,91 +2,54 @@ package fs
import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// Local implements local file storage
type Local struct {
hostname string
rootDir string
rootDir string
}
func NewLocal(rootDir string, hostname string) (*Local, error) {
if hostname == "" {
return nil, errors.New("hostname can't be empty")
}
hostname = strings.TrimSuffix(hostname, "/")
if !strings.HasPrefix(hostname, "http") {
hostname = fmt.Sprintf("http://%s", hostname)
}
return &Local{rootDir: rootDir, hostname: hostname}, nil
func NewLocal(rootDir string) (*Local, error) {
return &Local{rootDir: rootDir}, nil
}
func (l *Local) Create(ctx context.Context, ns string, fileName string, reader io.Reader) (int64, error) {
func (l *Local) Open(name string) (http.File, error) {
path := filepath.Join(l.rootDir, name)
return os.Open(path)
}
func (l *Local) Delete(_ctx context.Context, name string) error {
path := filepath.Join(l.rootDir, name)
return os.Remove(path)
}
func (l *Local) Create(_ctx context.Context, name string, reader io.Reader) (int64, error) {
var (
logger = log.WithField("episode_id", fileName)
feedDir = filepath.Join(l.rootDir, ns)
logger = log.WithField("name", name)
path = filepath.Join(l.rootDir, name)
)
if err := os.MkdirAll(feedDir, 0755); err != nil {
return 0, errors.Wrapf(err, "failed to create a directory for the feed: %s", feedDir)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return 0, errors.Wrapf(err, "failed to mkdir: %s", path)
}
logger.Debugf("creating directory: %s", feedDir)
if err := os.MkdirAll(feedDir, 0755); err != nil {
return 0, errors.Wrapf(err, "failed to create feed dir: %s", feedDir)
}
var (
episodePath = filepath.Join(l.rootDir, ns, fileName)
)
logger.Debugf("copying to: %s", episodePath)
written, err := l.copyFile(reader, episodePath)
logger.Infof("creating file: %s", path)
written, err := l.copyFile(reader, path)
if err != nil {
return 0, errors.Wrap(err, "failed to copy file")
}
logger.Debugf("copied %d bytes", written)
logger.Debugf("written %d bytes", written)
return written, nil
}
func (l *Local) Delete(ctx context.Context, ns string, fileName string) error {
path := filepath.Join(l.rootDir, ns, fileName)
return os.Remove(path)
}
func (l *Local) Size(ctx context.Context, ns string, fileName string) (int64, error) {
path := filepath.Join(l.rootDir, ns, fileName)
stat, err := os.Stat(path)
if err == nil {
return stat.Size(), nil
}
return 0, err
}
func (l *Local) URL(ctx context.Context, ns string, fileName string) (string, error) {
if _, err := l.Size(ctx, ns, fileName); err != nil {
return "", errors.Wrap(err, "failed to check whether file exists")
}
if ns == "" {
return fmt.Sprintf("%s/%s", l.hostname, fileName), nil
}
return fmt.Sprintf("%s/%s/%s", l.hostname, ns, fileName), nil
}
func (l *Local) copyFile(source io.Reader, destinationPath string) (int64, error) {
dest, err := os.Create(destinationPath)
if err != nil {
pkg/fs/local_test.go
+13 -34
View File
@@ -17,13 +17,9 @@ var (
)
func TestNewLocal(t *testing.T) {
local, err := NewLocal("", "localhost")
local, err := NewLocal("")
assert.NoError(t, err)
assert.Equal(t, "http://localhost", local.hostname)
local, err = NewLocal("", "https://localhost:8080/")
assert.NoError(t, err)
assert.Equal(t, "https://localhost:8080", local.hostname)
assert.NotNil(t, local)
}
func TestLocal_Create(t *testing.T) {
@@ -32,10 +28,10 @@ func TestLocal_Create(t *testing.T) {
defer os.RemoveAll(tmpDir)
stor, err := NewLocal(tmpDir, "localhost")
stor, err := NewLocal(tmpDir)
assert.NoError(t, err)
written, err := stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
written, err := stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
assert.NoError(t, err)
assert.EqualValues(t, 5, written)
@@ -50,22 +46,22 @@ func TestLocal_Size(t *testing.T) {
defer os.RemoveAll(tmpDir)
stor, err := NewLocal(tmpDir, "localhost")
stor, err := NewLocal(tmpDir)
assert.NoError(t, err)
_, err = stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
_, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
assert.NoError(t, err)
sz, err := stor.Size(testCtx, "1", "test")
sz, err := Size(stor, "1/test")
assert.NoError(t, err)
assert.EqualValues(t, 5, sz)
}
func TestLocal_NoSize(t *testing.T) {
stor, err := NewLocal("", "localhost")
stor, err := NewLocal("")
assert.NoError(t, err)
_, err = stor.Size(testCtx, "1", "test")
_, err = Size(stor, "1/test")
assert.True(t, os.IsNotExist(err))
}
@@ -75,39 +71,22 @@ func TestLocal_Delete(t *testing.T) {
defer os.RemoveAll(tmpDir)
stor, err := NewLocal(tmpDir, "localhost")
stor, err := NewLocal(tmpDir)
assert.NoError(t, err)
_, err = stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
_, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
assert.NoError(t, err)
err = stor.Delete(testCtx, "1", "test")
err = stor.Delete(testCtx, "1/test")
assert.NoError(t, err)
_, err = stor.Size(testCtx, "1", "test")
_, err = Size(stor, "1/test")
assert.True(t, os.IsNotExist(err))
_, err = os.Stat(filepath.Join(tmpDir, "1", "test"))
assert.True(t, os.IsNotExist(err))
}
func TestLocal_URL(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "podsync-local-stor-")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
stor, err := NewLocal(tmpDir, "localhost")
assert.NoError(t, err)
_, err = stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
assert.NoError(t, err)
url, err := stor.URL(testCtx, "1", "test")
assert.NoError(t, err)
assert.EqualValues(t, "http://localhost/1/test", url)
}
func TestLocal_copyFile(t *testing.T) {
reader := bytes.NewReader([]byte{1, 2, 4})
pkg/fs/storage.go
+23 -8
View File
@@ -3,18 +3,33 @@ package fs
import (
"context"
"io"
"net/http"
)
// Storage is a file system interface to host downloaded episodes and feeds.
type Storage interface {
// FileSystem must be implemented to in order to pass Storage interface to HTTP file server.
http.FileSystem
// Create will create a new file from reader
Create(ctx context.Context, ns string, fileName string, reader io.Reader) (int64, error)
Create(ctx context.Context, name string, reader io.Reader) (int64, error)
// Delete deletes the file
Delete(ctx context.Context, ns string, fileName string) error
// Size returns the size of a file in bytes
Size(ctx context.Context, ns string, fileName string) (int64, error)
// URL will generate a download link for a file
URL(ctx context.Context, ns string, fileName string) (string, error)
Delete(ctx context.Context, name string) error
}
// Size returns storage object's size in bytes.
func Size(storage http.FileSystem, name string) (int64, error) {
file, err := storage.Open(name)
if err != nil {
return 0, err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return 0, err
}
return stat.Size(), nil
}