Implement remote updater

cmd/api/main.go

@@ -33,6 +33,7 @@ type Opts struct {
 	DynamoFeedsTableName   string `long:"dynamo-feeds-table" env:"DYNAMO_FEEDS_TABLE_NAME"`
 	DynamoPledgesTableName string `long:"dynamo-pledges-table" env:"DYNAMO_PLEDGES_TABLE_NAME"`
 	RedisURL               string `long:"redis-url" required:"true" env:"REDIS_CONNECTION_URL"`
+	UpdaterURL             string `long:"updater-url" required:"true" env:"UPDATER_URL"`
 	Debug                  bool   `long:"debug" env:"DEBUG"`
 }
 
@@ -90,10 +91,7 @@ func main() {
 		log.WithError(err).Fatal("failed to create Vimeo builder")
 	}
 
-	generic, err := builders.NewLambda()
-	if err != nil {
-		log.WithError(err).Fatal("failed to create Lambda builder")
-	}
+	generic := builders.NewRemote(opts.UpdaterURL)
 
 	feed, err := feeds.NewFeedService(database, redisCache, map[api.Provider]feeds.Builder{
 		api.ProviderYoutube: youtube,

cmd/updater/server.py

@@ -8,8 +8,8 @@ app = Sanic()
 @app.get('/update')
 async def update(req):
     url = req.args.get('url', None)
-    start = req.args.get('start', 1)
-    count = req.args.get('count', updater.DEFAULT_PAGE_SIZE)
+    start = int(req.args.get('start', 1))
+    count = int(req.args.get('count', updater.DEFAULT_PAGE_SIZE))
 
     # Last seen video ID
     last_id = req.args.get('last_id', None)

docker-compose.yml

@@ -10,6 +10,7 @@ services:
     environment:
       - REDIS_CONNECTION_URL=redis://redis
       - POSTGRES_CONNECTION_URL={POSTGRES_CONNECTION_URL}
+      - UPDATER_URL=http://updater:8080/update
       - YOUTUBE_API_KEY={YOUTUBE_API_KEY}
       - VIMEO_API_KEY={VIMEO_API_KEY}
       - PATREON_CLIENT_ID={PATREON_CLIENT_ID}
@@ -49,3 +50,10 @@ services:
       - AWS_DEFAULT_REGION=us-east-1
       - DYNAMO_FEEDS_TABLE_NAME=Prod_Feeds
       - DYNAMO_RESOLVE_COUNTERS_TABLE=Prod_ResolveCounters
+  updater:
+    image: mxpv/updater:latest
+    container_name: updater
+    command: --workers 256
+    restart: always
+    ports:
+      - 8080

pkg/builders/remote.go

@@ -0,0 +1,79 @@
+package builders
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/mxpv/podsync/pkg/model"
+)
+
+type Remote struct {
+	url string
+}
+
+func NewRemote(url string) Remote {
+	return Remote{url: url}
+}
+
+func (r Remote) Build(feed *model.Feed) error {
+	addr, err := r.makeURL(feed)
+	if err != nil {
+		return err
+	}
+
+	client := http.Client{
+		Timeout: 5 * time.Minute,
+	}
+
+	resp, err := client.Get(addr)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read error body (status: %d)", resp.StatusCode)
+		}
+
+		return errors.Errorf("unexpected response (%d) from updater: %q", resp.StatusCode, body)
+	}
+
+	var out responsePayload
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return err
+	}
+
+	feed.LastID = out.LastID
+	feed.Episodes = append(out.Episodes, feed.Episodes...)
+	feed.UpdatedAt = time.Now().UTC()
+
+	return nil
+}
+
+func (r Remote) makeURL(feed *model.Feed) (string, error) {
+	qs := url.Values{}
+	qs.Add("url", feed.ItemURL)
+	qs.Add("start", "1")
+	qs.Add("count", strconv.Itoa(feed.PageSize))
+	qs.Add("last_id", feed.LastID)
+	qs.Add("format", string(feed.Format))
+	qs.Add("quality", string(feed.Quality))
+
+	parsed, err := url.Parse(r.url)
+	if err != nil {
+		return "", err
+	}
+
+	parsed.RawQuery = qs.Encode()
+
+	return parsed.String(), nil
+}

pkg/builders/remote_test.go

@@ -0,0 +1,22 @@
+package builders
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mxpv/podsync/pkg/model"
+)
+
+func TestRemote_makeURL(t *testing.T) {
+	feed := &model.Feed{
+		ItemURL:  "https://youtube.com/channel/UCupvZG-5ko_eiXAupbDfxWw",
+		PageSize: 2,
+		Format:   "video",
+		Quality:  "high",
+	}
+
+	out, err := Remote{url: "http://updater:8080/update"}.makeURL(feed)
+	assert.NoError(t, err)
+	assert.EqualValues(t, "http://updater:8080/update?count=2&format=video&last_id=&quality=high&start=1&url=https%3A%2F%2Fyoutube.com%2Fchannel%2FUCupvZG-5ko_eiXAupbDfxWw", out)
+}