diff --git a/cmd/api/main.go b/cmd/api/main.go index e422780..281ce2e 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -33,6 +33,7 @@ type Opts struct { DynamoFeedsTableName string `long:"dynamo-feeds-table" env:"DYNAMO_FEEDS_TABLE_NAME"` DynamoPledgesTableName string `long:"dynamo-pledges-table" env:"DYNAMO_PLEDGES_TABLE_NAME"` RedisURL string `long:"redis-url" required:"true" env:"REDIS_CONNECTION_URL"` + UpdaterURL string `long:"updater-url" required:"true" env:"UPDATER_URL"` Debug bool `long:"debug" env:"DEBUG"` } @@ -90,10 +91,7 @@ func main() { log.WithError(err).Fatal("failed to create Vimeo builder") } - generic, err := builders.NewLambda() - if err != nil { - log.WithError(err).Fatal("failed to create Lambda builder") - } + generic := builders.NewRemote(opts.UpdaterURL) feed, err := feeds.NewFeedService(database, redisCache, map[api.Provider]feeds.Builder{ api.ProviderYoutube: youtube, diff --git a/cmd/updater/server.py b/cmd/updater/server.py index 23c8f92..8e1bacd 100644 --- a/cmd/updater/server.py +++ b/cmd/updater/server.py @@ -8,8 +8,8 @@ app = Sanic() @app.get('/update') async def update(req): url = req.args.get('url', None) - start = req.args.get('start', 1) - count = req.args.get('count', updater.DEFAULT_PAGE_SIZE) + start = int(req.args.get('start', 1)) + count = int(req.args.get('count', updater.DEFAULT_PAGE_SIZE)) # Last seen video ID last_id = req.args.get('last_id', None) diff --git a/docker-compose.yml b/docker-compose.yml index f10b94d..355a96e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ services: environment: - REDIS_CONNECTION_URL=redis://redis - POSTGRES_CONNECTION_URL={POSTGRES_CONNECTION_URL} + - UPDATER_URL=http://updater:8080/update - YOUTUBE_API_KEY={YOUTUBE_API_KEY} - VIMEO_API_KEY={VIMEO_API_KEY} - PATREON_CLIENT_ID={PATREON_CLIENT_ID} @@ -49,3 +50,10 @@ services: - AWS_DEFAULT_REGION=us-east-1 - DYNAMO_FEEDS_TABLE_NAME=Prod_Feeds - DYNAMO_RESOLVE_COUNTERS_TABLE=Prod_ResolveCounters + updater: + image: mxpv/updater:latest + container_name: updater + command: --workers 256 + restart: always + ports: + - 8080 diff --git a/pkg/builders/remote.go b/pkg/builders/remote.go new file mode 100644 index 0000000..541be80 --- /dev/null +++ b/pkg/builders/remote.go @@ -0,0 +1,79 @@ +package builders + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/pkg/errors" + + "github.com/mxpv/podsync/pkg/model" +) + +type Remote struct { + url string +} + +func NewRemote(url string) Remote { + return Remote{url: url} +} + +func (r Remote) Build(feed *model.Feed) error { + addr, err := r.makeURL(feed) + if err != nil { + return err + } + + client := http.Client{ + Timeout: 5 * time.Minute, + } + + resp, err := client.Get(addr) + if err != nil { + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrapf(err, "failed to read error body (status: %d)", resp.StatusCode) + } + + return errors.Errorf("unexpected response (%d) from updater: %q", resp.StatusCode, body) + } + + var out responsePayload + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + feed.LastID = out.LastID + feed.Episodes = append(out.Episodes, feed.Episodes...) + feed.UpdatedAt = time.Now().UTC() + + return nil +} + +func (r Remote) makeURL(feed *model.Feed) (string, error) { + qs := url.Values{} + qs.Add("url", feed.ItemURL) + qs.Add("start", "1") + qs.Add("count", strconv.Itoa(feed.PageSize)) + qs.Add("last_id", feed.LastID) + qs.Add("format", string(feed.Format)) + qs.Add("quality", string(feed.Quality)) + + parsed, err := url.Parse(r.url) + if err != nil { + return "", err + } + + parsed.RawQuery = qs.Encode() + + return parsed.String(), nil +} diff --git a/pkg/builders/remote_test.go b/pkg/builders/remote_test.go new file mode 100644 index 0000000..1f5002d --- /dev/null +++ b/pkg/builders/remote_test.go @@ -0,0 +1,22 @@ +package builders + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mxpv/podsync/pkg/model" +) + +func TestRemote_makeURL(t *testing.T) { + feed := &model.Feed{ + ItemURL: "https://youtube.com/channel/UCupvZG-5ko_eiXAupbDfxWw", + PageSize: 2, + Format: "video", + Quality: "high", + } + + out, err := Remote{url: "http://updater:8080/update"}.makeURL(feed) + assert.NoError(t, err) + assert.EqualValues(t, "http://updater:8080/update?count=2&format=video&last_id=&quality=high&start=1&url=https%3A%2F%2Fyoutube.com%2Fchannel%2FUCupvZG-5ko_eiXAupbDfxWw", out) +}