From d76493c07cd609a071fdc90d34bb702536813e7e Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 26 Oct 2017 12:45:10 -0700 Subject: [PATCH 01/10] Add webhooks secret configuration key --- docker-compose.yml | 1 + pkg/config/config.go | 2 ++ pkg/config/config_test.go | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 13e7a39..6733308 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: - PATREON_CLIENT_ID={PATREON_CLIENT_ID} - PATREON_SECRET={PATREON_SECRET} - PATREON_REDIRECT_URL=https://podsync.net/patreon + - PATREON_WEBHOOKS_SECRET={PATREON_WEBHOOKS_SECRET} - COOKIE_SECRET={COOKIE_SECRET} ytdl: image: gcr.io/pod-sync/ytdl:latest diff --git a/pkg/config/config.go b/pkg/config/config.go index 8719a55..dadf2e8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,6 +14,7 @@ type AppConfig struct { PatreonClientId string `yaml:"patreonClientId"` PatreonSecret string `yaml:"patreonSecret"` PatreonRedirectURL string `yaml:"patreonRedirectUrl"` + PatreonWebhooksSecret string `json:"patreonWebhooksSecret"` PostgresConnectionURL string `yaml:"postgresConnectionUrl"` RedisURL string `yaml:"redisUrl"` CookieSecret string `yaml:"cookieSecret"` @@ -38,6 +39,7 @@ func ReadConfiguration() (cfg *AppConfig, err error) { "patreonClientId": "PATREON_CLIENT_ID", "patreonSecret": "PATREON_SECRET", "patreonRedirectUrl": "PATREON_REDIRECT_URL", + "patreonWebhooksSecret": "PATREON_WEBHOOKS_SECRET", "postgresConnectionUrl": "POSTGRES_CONNECTION_URL", "redisUrl": "REDIS_CONNECTION_URL", "cookieSecret": "COOKIE_SECRET", diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 437a495..e80868c 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -19,6 +19,7 @@ cookieSecret: "6" patreonRedirectUrl: "7" assetsPath: "8" templatesPath: "9" +patreonWebhooksSecret: "10" ` func TestReadYaml(t *testing.T) { @@ -40,6 +41,7 @@ func TestReadYaml(t *testing.T) { require.Equal(t, "7", cfg.PatreonRedirectURL) require.Equal(t, "8", cfg.AssetsPath) require.Equal(t, "9", cfg.TemplatesPath) + require.Equal(t, "10", cfg.PatreonWebhooksSecret) } func TestReadEnv(t *testing.T) { @@ -55,6 +57,7 @@ func TestReadEnv(t *testing.T) { os.Setenv("PATREON_REDIRECT_URL", "77") os.Setenv("ASSETS_PATH", "88") os.Setenv("TEMPLATES_PATH", "99") + os.Setenv("PATREON_WEBHOOKS_SECRET", "1010") cfg, err := ReadConfiguration() require.NoError(t, err) @@ -68,4 +71,5 @@ func TestReadEnv(t *testing.T) { require.Equal(t, "77", cfg.PatreonRedirectURL) require.Equal(t, "88", cfg.AssetsPath) require.Equal(t, "99", cfg.TemplatesPath) + require.Equal(t, "1010", cfg.PatreonWebhooksSecret) } From 2f2c392b7d64c9dd53f621f5d208ad7b97968620 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 26 Oct 2017 14:36:52 -0700 Subject: [PATCH 02/10] Implement hook handler --- Gopkg.toml | 2 +- pkg/models/models.go | 14 +++++++ pkg/models/pg.sql | 16 +++++++ pkg/patreon/hook.go | 90 ++++++++++++++++++++++++++++++++++++++++ pkg/patreon/hook_test.go | 82 ++++++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 pkg/models/models.go create mode 100644 pkg/models/pg.sql create mode 100644 pkg/patreon/hook.go create mode 100644 pkg/patreon/hook_test.go diff --git a/Gopkg.toml b/Gopkg.toml index cef0d61..a52729e 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -13,7 +13,7 @@ [[constraint]] name = "github.com/mxpv/patreon-go" - version = "1.2" + revision = "181da1e272784f51dea234c0e58e595321abd1ed" [[constraint]] name = "github.com/ventu-io/go-shortid" diff --git a/pkg/models/models.go b/pkg/models/models.go new file mode 100644 index 0000000..96f7afd --- /dev/null +++ b/pkg/models/models.go @@ -0,0 +1,14 @@ +package models + +import "time" + +type Pledge struct { + PledgeID int64 `sql:",pk"` + PatronID int64 + CreatedAt time.Time + DeclinedSince time.Time + AmountCents int + TotalHistoricalAmountCents int + OutstandingPaymentAmountCents int + IsPaused bool +} diff --git a/pkg/models/pg.sql b/pkg/models/pg.sql new file mode 100644 index 0000000..f05d0cd --- /dev/null +++ b/pkg/models/pg.sql @@ -0,0 +1,16 @@ +BEGIN; + +CREATE TABLE IF NOT EXISTS pledges ( + pledge_id BIGSERIAL PRIMARY KEY, + patron_id BIGINT NOT NULL , + created_at TIMESTAMPTZ NOT NULL, + declined_since TIMESTAMPTZ NULL, + amount_cents INT NOT NULL, + total_historical_amount_cents INT, + outstanding_payment_amount_cents INT, + is_paused BOOLEAN +); + +CREATE INDEX patron_id_idx ON pledges(patron_id); + +COMMIT; \ No newline at end of file diff --git a/pkg/patreon/hook.go b/pkg/patreon/hook.go new file mode 100644 index 0000000..87130c2 --- /dev/null +++ b/pkg/patreon/hook.go @@ -0,0 +1,90 @@ +package patreon + +import ( + "fmt" + "strconv" + + "github.com/go-pg/pg" + "github.com/mxpv/patreon-go" + "github.com/mxpv/podsync/pkg/models" + "github.com/pkg/errors" +) + +const ( + EventHeader = "X-Patreon-Event" + SignatureHeader = "X-Patreon-Signature" + + EventNameCreatePledge = "pledges:create" + EventNameUpdatePledge = "pledges:update" + EventNameDeletePledge = "pledges:delete" +) + +type Hook struct { + db *pg.DB +} + +func (h Hook) toModel(pledge *patreon.Pledge) (*models.Pledge, error) { + pledgeID, err := strconv.ParseInt(pledge.Id, 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse pledge id: %s", pledge.Id) + } + + patronID, err := strconv.ParseInt(pledge.Relationships.Patron.Data.Id, 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse patron id: %s", pledge.Relationships.Patron.Data.Id) + } + + model := &models.Pledge{ + PledgeID: pledgeID, + PatronID: patronID, + AmountCents: pledge.Attributes.AmountCents, + } + + if pledge.Attributes.CreatedAt.Valid { + model.CreatedAt = pledge.Attributes.CreatedAt.Time + } + + if pledge.Attributes.DeclinedSince.Valid { + model.DeclinedSince = pledge.Attributes.DeclinedSince.Time + } + + // Read optional fields + + if pledge.Attributes.TotalHistoricalAmountCents != nil { + model.TotalHistoricalAmountCents = *pledge.Attributes.TotalHistoricalAmountCents + } + + if pledge.Attributes.OutstandingPaymentAmountCents != nil { + model.OutstandingPaymentAmountCents = *pledge.Attributes.OutstandingPaymentAmountCents + } + + if pledge.Attributes.IsPaused != nil { + model.IsPaused = *pledge.Attributes.IsPaused + } + + return model, nil +} + +func (h Hook) Handle(pledge *patreon.Pledge, event string) error { + model, err := h.toModel(pledge) + if err != nil { + return err + } + + switch event { + case EventNameCreatePledge: + return h.db.Insert(model) + case EventNameUpdatePledge: + return h.db.Update(model) + case EventNameDeletePledge: + return h.db.Delete(model) + default: + return fmt.Errorf("unknown event: %s", event) + } + + return nil +} + +func NewHookHandler(db *pg.DB) *Hook { + return &Hook{db: db} +} diff --git a/pkg/patreon/hook_test.go b/pkg/patreon/hook_test.go new file mode 100644 index 0000000..76815bf --- /dev/null +++ b/pkg/patreon/hook_test.go @@ -0,0 +1,82 @@ +package patreon + +import ( + "testing" + "time" + + "github.com/go-pg/pg" + "github.com/mxpv/patreon-go" + "github.com/mxpv/podsync/pkg/models" + "github.com/stretchr/testify/require" +) + +func TestCreate(t *testing.T) { + pledge := createPledge() + + hook := createHandler(t) + err := hook.Handle(pledge, EventNameCreatePledge) + require.NoError(t, err) + + model := &models.Pledge{PledgeID: 12345} + err = hook.db.Select(model) + require.NoError(t, err) + require.Equal(t, pledge.Attributes.AmountCents, model.AmountCents) +} + +func TestUpdate(t *testing.T) { + pledge := createPledge() + + hook := createHandler(t) + err := hook.Handle(pledge, EventNameCreatePledge) + require.NoError(t, err) + + pledge.Attributes.AmountCents = 999 + + err = hook.Handle(pledge, EventNameUpdatePledge) + require.NoError(t, err) + + model := &models.Pledge{PledgeID: 12345} + err = hook.db.Select(model) + require.NoError(t, err) + require.Equal(t, 999, model.AmountCents) +} + +func TestDelete(t *testing.T) { + pledge := createPledge() + hook := createHandler(t) + + err := hook.Handle(pledge, EventNameCreatePledge) + require.NoError(t, err) + + err = hook.Handle(pledge, EventNameDeletePledge) + require.NoError(t, err) +} + +func createHandler(t *testing.T) *Hook { + opts, err := pg.ParseURL("postgres://postgres:@localhost/podsync?sslmode=disable") + if err != nil { + require.NoError(t, err) + } + + db := pg.Connect(opts) + + _, err = db.Model(&models.Pledge{}).Where("1=1").Delete() + require.NoError(t, err) + + return NewHookHandler(db) +} + +func createPledge() *patreon.Pledge { + pledge := &patreon.Pledge{ + Id: "12345", + Type: "pledge", + } + + pledge.Attributes.AmountCents = 400 + pledge.Attributes.CreatedAt = patreon.NullTime{Valid: true, Time: time.Now().UTC()} + + pledge.Relationships.Patron = &patreon.PatronRelationship{} + pledge.Relationships.Patron.Data.Id = "67890" + + return pledge +} From bca16c1dc97da6cb6153fc64003cc525bcaf3a2a Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 26 Oct 2017 16:48:22 -0700 Subject: [PATCH 03/10] Executee pg.sql on Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3edb6e3..a08964c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ addons: postgresql: "9.6" before_script: - psql -a -c "CREATE DATABASE podsync;" -U postgres + - psql -a -f ./pkg/models/pg.sql -d podsync -U postgres script: - set -e - go test -v -short $(go list -e ./... | grep -v vendor) From b70e6228e7912f4a1c14573396c9e10a761c36a6 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 26 Oct 2017 16:52:40 -0700 Subject: [PATCH 04/10] Fix go linter --- pkg/patreon/hook.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/patreon/hook.go b/pkg/patreon/hook.go index 87130c2..3f1aecc 100644 --- a/pkg/patreon/hook.go +++ b/pkg/patreon/hook.go @@ -81,8 +81,6 @@ func (h Hook) Handle(pledge *patreon.Pledge, event string) error { default: return fmt.Errorf("unknown event: %s", event) } - - return nil } func NewHookHandler(db *pg.DB) *Hook { From 75bd90ff49958e66940c11f9e9f3a6dc7f5a0c82 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 26 Oct 2017 19:26:59 -0700 Subject: [PATCH 05/10] Rename patreon package --- pkg/{patreon => webhooks}/hook.go | 12 ++++++------ pkg/{patreon => webhooks}/hook_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) rename pkg/{patreon => webhooks}/hook.go (87%) rename pkg/{patreon => webhooks}/hook_test.go (96%) diff --git a/pkg/patreon/hook.go b/pkg/webhooks/hook.go similarity index 87% rename from pkg/patreon/hook.go rename to pkg/webhooks/hook.go index 3f1aecc..6977aa9 100644 --- a/pkg/patreon/hook.go +++ b/pkg/webhooks/hook.go @@ -1,4 +1,4 @@ -package patreon +package webhooks import ( "fmt" @@ -19,11 +19,11 @@ const ( EventNameDeletePledge = "pledges:delete" ) -type Hook struct { +type Handler struct { db *pg.DB } -func (h Hook) toModel(pledge *patreon.Pledge) (*models.Pledge, error) { +func (h Handler) toModel(pledge *patreon.Pledge) (*models.Pledge, error) { pledgeID, err := strconv.ParseInt(pledge.Id, 10, 64) if err != nil { return nil, errors.Wrapf(err, "failed to parse pledge id: %s", pledge.Id) @@ -65,7 +65,7 @@ func (h Hook) toModel(pledge *patreon.Pledge) (*models.Pledge, error) { return model, nil } -func (h Hook) Handle(pledge *patreon.Pledge, event string) error { +func (h Handler) Handle(pledge *patreon.Pledge, event string) error { model, err := h.toModel(pledge) if err != nil { return err @@ -83,6 +83,6 @@ func (h Hook) Handle(pledge *patreon.Pledge, event string) error { } } -func NewHookHandler(db *pg.DB) *Hook { - return &Hook{db: db} +func NewHookHandler(db *pg.DB) *Handler { + return &Handler{db: db} } diff --git a/pkg/patreon/hook_test.go b/pkg/webhooks/hook_test.go similarity index 96% rename from pkg/patreon/hook_test.go rename to pkg/webhooks/hook_test.go index 76815bf..d1edcc2 100644 --- a/pkg/patreon/hook_test.go +++ b/pkg/webhooks/hook_test.go @@ -1,4 +1,4 @@ -package patreon +package webhooks import ( "testing" @@ -52,7 +52,7 @@ func TestDelete(t *testing.T) { require.NoError(t, err) } -func createHandler(t *testing.T) *Hook { +func createHandler(t *testing.T) *Handler { opts, err := pg.ParseURL("postgres://postgres:@localhost/podsync?sslmode=disable") if err != nil { require.NoError(t, err) From 813717a2c34c05a27d07f3ccef0e1b5cd8ded328 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Sat, 28 Oct 2017 01:11:49 -0700 Subject: [PATCH 06/10] Refactor server package --- cmd/app/main.go | 4 +- pkg/handler/handler.go | 256 ++++++++++++++++++ .../handler_mock_test.go} | 4 +- .../handler_test.go} | 12 +- pkg/server/server.go | 232 ---------------- 5 files changed, 266 insertions(+), 242 deletions(-) create mode 100644 pkg/handler/handler.go rename pkg/{server/server_mock_test.go => handler/handler_mock_test.go} (98%) rename pkg/{server/server_test.go => handler/handler_test.go} (92%) delete mode 100644 pkg/server/server.go diff --git a/cmd/app/main.go b/cmd/app/main.go index a187158..f291c9a 100644 --- a/cmd/app/main.go +++ b/cmd/app/main.go @@ -13,8 +13,8 @@ import ( "github.com/mxpv/podsync/pkg/builders" "github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/feeds" + "github.com/mxpv/podsync/pkg/handler" "github.com/mxpv/podsync/pkg/id" - "github.com/mxpv/podsync/pkg/server" "github.com/mxpv/podsync/pkg/storage" ) @@ -63,7 +63,7 @@ func main() { srv := http.Server{ Addr: fmt.Sprintf(":%d", 5001), - Handler: server.MakeHandlers(feed, cfg), + Handler: handler.New(feed, cfg), } go func() { diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go new file mode 100644 index 0000000..18266ea --- /dev/null +++ b/pkg/handler/handler.go @@ -0,0 +1,256 @@ +package handler + +import ( + "log" + "net/http" + "path" + "strings" + + "github.com/gin-contrib/sessions" + "github.com/gin-gonic/gin" + "github.com/mxpv/patreon-go" + itunes "github.com/mxpv/podcast" + "github.com/mxpv/podsync/pkg/api" + "github.com/mxpv/podsync/pkg/config" + "github.com/mxpv/podsync/pkg/session" + "golang.org/x/oauth2" +) + +const ( + creatorID = "2822191" + maxHashIDLength = 16 +) + +type feed interface { + CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) + GetFeed(hashId string) (*itunes.Podcast, error) + GetMetadata(hashId string) (*api.Feed, error) +} + +type handler struct { + feed feed + cfg *config.AppConfig + oauth2 oauth2.Config +} + +func (h handler) index(c *gin.Context) { + identity, err := session.GetIdentity(c) + if err != nil { + identity = &api.Identity{} + } + + c.HTML(http.StatusOK, "index.html", identity) +} + +func (h handler) login(c *gin.Context) { + state, err := session.SetState(c) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + + authURL := h.oauth2.AuthCodeURL(state) + c.Redirect(http.StatusFound, authURL) +} + +func (h handler) logout(c *gin.Context) { + session.Clear(c) + + c.Redirect(http.StatusFound, "/") +} + +func (h handler) patreonCallback(c *gin.Context) { + // Validate session state + if session.GetSetate(c) != c.Query("state") { + c.String(http.StatusUnauthorized, "invalid state") + return + } + + // Exchange code with tokens + token, err := h.oauth2.Exchange(c.Request.Context(), c.Query("code")) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + // Create Patreon client + tc := h.oauth2.Client(c.Request.Context(), token) + client := patreon.NewClient(tc) + + // Query user info from Patreon + user, err := client.FetchUser() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + + // Determine feature level + level := api.DefaultFeatures + + if user.Data.Id == creatorID { + level = api.PodcasterFeature + } else { + amount := 0 + for _, item := range user.Included.Items { + pledge, ok := item.(*patreon.Pledge) + if ok { + amount += pledge.Attributes.AmountCents + } + } + + if amount >= 100 { + level = api.ExtendedFeatures + } + } + + identity := &api.Identity{ + UserId: user.Data.Id, + FullName: user.Data.Attributes.FullName, + Email: user.Data.Attributes.Email, + ProfileURL: user.Data.Attributes.URL, + FeatureLevel: level, + } + + session.SetIdentity(c, identity) + c.Redirect(http.StatusFound, "/") +} + +func (h handler) robots(c *gin.Context) { + c.String(http.StatusOK, `User-agent: * +Allow: /$ +Disallow: / +Host: www.podsync.net`) +} + +func (h handler) ping(c *gin.Context) { + c.String(http.StatusOK, "ok") +} + +func (h handler) create(c *gin.Context) { + req := &api.CreateFeedRequest{} + + if err := c.BindJSON(req); err != nil { + c.JSON(badRequest(err)) + return + } + + identity, err := session.GetIdentity(c) + if err != nil { + c.JSON(internalError(err)) + return + } + + hashId, err := h.feed.CreateFeed(req, identity) + if err != nil { + c.JSON(internalError(err)) + return + } + + c.JSON(http.StatusOK, gin.H{"id": hashId}) +} + +func (h handler) getFeed(c *gin.Context) { + hashId := c.Request.URL.Path[1:] + if hashId == "" || len(hashId) > maxHashIDLength { + c.String(http.StatusBadRequest, "invalid feed id") + return + } + + if strings.HasSuffix(hashId, ".xml") { + hashId = strings.TrimSuffix(hashId, ".xml") + } + + podcast, err := h.feed.GetFeed(hashId) + if err != nil { + code := http.StatusInternalServerError + if err == api.ErrNotFound { + code = http.StatusNotFound + } else { + log.Printf("server error (hash id: %s): %v", hashId, err) + } + + c.String(code, err.Error()) + return + } + + c.Data(http.StatusOK, "application/rss+xml; charset=UTF-8", podcast.Bytes()) +} + +func (h handler) metadata(c *gin.Context) { + hashId := c.Param("hashId") + if hashId == "" || len(hashId) > maxHashIDLength { + c.String(http.StatusBadRequest, "invalid feed id") + return + } + + feed, err := h.feed.GetMetadata(hashId) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + + c.JSON(http.StatusOK, feed) +} + +func New(feed feed, cfg *config.AppConfig) http.Handler { + r := gin.New() + r.Use(gin.Recovery()) + + store := sessions.NewCookieStore([]byte(cfg.CookieSecret)) + r.Use(sessions.Sessions("podsync", store)) + + // Static files + HTML + + log.Printf("using assets path: %s", cfg.AssetsPath) + if cfg.AssetsPath != "" { + r.Static("/assets", cfg.AssetsPath) + } + + log.Printf("using templates path: %s", cfg.TemplatesPath) + if cfg.TemplatesPath != "" { + r.LoadHTMLGlob(path.Join(cfg.TemplatesPath, "*.html")) + } + + h := handler{ + feed: feed, + cfg: cfg, + } + + // OAuth 2 configuration + + h.oauth2 = oauth2.Config{ + ClientID: cfg.PatreonClientId, + ClientSecret: cfg.PatreonSecret, + RedirectURL: cfg.PatreonRedirectURL, + Scopes: []string{"users", "pledges-to-me", "my-campaign"}, + Endpoint: oauth2.Endpoint{ + AuthURL: patreon.AuthorizationURL, + TokenURL: patreon.AccessTokenURL, + }, + } + + // Handlers + + r.GET("/", h.index) + r.GET("/login", h.login) + r.GET("/logout", h.logout) + r.GET("/patreon", h.patreonCallback) + r.GET("/robots.txt", h.robots) + + r.GET("/api/ping", h.ping) + r.POST("/api/create", h.create) + r.GET("/api/metadata/:hashId", h.metadata) + + r.NoRoute(h.getFeed) + + return r +} + +func badRequest(err error) (int, interface{}) { + return http.StatusBadRequest, gin.H{"error": err.Error()} +} + +func internalError(err error) (int, interface{}) { + log.Printf("server error: %v", err) + return http.StatusInternalServerError, gin.H{"error": err.Error()} +} diff --git a/pkg/server/server_mock_test.go b/pkg/handler/handler_mock_test.go similarity index 98% rename from pkg/server/server_mock_test.go rename to pkg/handler/handler_mock_test.go index a808718..9c88046 100644 --- a/pkg/server/server_mock_test.go +++ b/pkg/handler/handler_mock_test.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: server.go +// Source: handler.go -package server +package handler import ( gomock "github.com/golang/mock/gomock" diff --git a/pkg/server/server_test.go b/pkg/handler/handler_test.go similarity index 92% rename from pkg/server/server_test.go rename to pkg/handler/handler_test.go index 7a5864d..5c12f40 100644 --- a/pkg/server/server_test.go +++ b/pkg/handler/handler_test.go @@ -1,6 +1,6 @@ -//go:generate mockgen -source=server.go -destination=server_mock_test.go -package=server +//go:generate mockgen -source=handler.go -destination=handler_mock_test.go -package=handler -package server +package handler import ( "io/ioutil" @@ -32,7 +32,7 @@ func TestCreateFeed(t *testing.T) { feed := NewMockfeed(ctrl) feed.EXPECT().CreateFeed(gomock.Eq(req), gomock.Any()).Times(1).Return("456", nil) - srv := httptest.NewServer(MakeHandlers(feed, cfg)) + srv := httptest.NewServer(New(feed, cfg)) defer srv.Close() query := `{"url": "https://youtube.com/channel/123", "page_size": 55, "quality": "low", "format": "audio"}` @@ -47,7 +47,7 @@ func TestCreateInvalidFeed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - srv := httptest.NewServer(MakeHandlers(NewMockfeed(ctrl), cfg)) + srv := httptest.NewServer(New(NewMockfeed(ctrl), cfg)) defer srv.Close() query := `{}` @@ -100,7 +100,7 @@ func TestGetFeed(t *testing.T) { feed := NewMockfeed(ctrl) feed.EXPECT().GetFeed("123").Return(&podcast, nil) - srv := httptest.NewServer(MakeHandlers(feed, cfg)) + srv := httptest.NewServer(New(feed, cfg)) defer srv.Close() resp, err := http.Get(srv.URL + "/123") @@ -115,7 +115,7 @@ func TestGetMetadata(t *testing.T) { feed := NewMockfeed(ctrl) feed.EXPECT().GetMetadata("123").Times(1).Return(&api.Feed{}, nil) - srv := httptest.NewServer(MakeHandlers(feed, cfg)) + srv := httptest.NewServer(New(feed, cfg)) defer srv.Close() resp, err := http.Get(srv.URL + "/api/metadata/123") diff --git a/pkg/server/server.go b/pkg/server/server.go deleted file mode 100644 index 2d292a1..0000000 --- a/pkg/server/server.go +++ /dev/null @@ -1,232 +0,0 @@ -package server - -import ( - "log" - "net/http" - "path" - "strings" - - "github.com/gin-contrib/sessions" - "github.com/gin-gonic/gin" - "github.com/mxpv/patreon-go" - itunes "github.com/mxpv/podcast" - "github.com/mxpv/podsync/pkg/api" - "github.com/mxpv/podsync/pkg/config" - "github.com/mxpv/podsync/pkg/session" - "golang.org/x/oauth2" -) - -const ( - creatorID = "2822191" - maxHashIDLength = 16 -) - -type feed interface { - CreateFeed(req *api.CreateFeedRequest, identity *api.Identity) (string, error) - GetFeed(hashId string) (*itunes.Podcast, error) - GetMetadata(hashId string) (*api.Feed, error) -} - -func MakeHandlers(feed feed, cfg *config.AppConfig) http.Handler { - r := gin.New() - r.Use(gin.Recovery()) - - store := sessions.NewCookieStore([]byte(cfg.CookieSecret)) - r.Use(sessions.Sessions("podsync", store)) - - // Static files + HTML - - log.Printf("using assets path: %s", cfg.AssetsPath) - if cfg.AssetsPath != "" { - r.Static("/assets", cfg.AssetsPath) - } - - log.Printf("using templates path: %s", cfg.TemplatesPath) - if cfg.TemplatesPath != "" { - r.LoadHTMLGlob(path.Join(cfg.TemplatesPath, "*.html")) - } - - conf := &oauth2.Config{ - ClientID: cfg.PatreonClientId, - ClientSecret: cfg.PatreonSecret, - RedirectURL: cfg.PatreonRedirectURL, - Scopes: []string{"users", "pledges-to-me", "my-campaign"}, - Endpoint: oauth2.Endpoint{ - AuthURL: patreon.AuthorizationURL, - TokenURL: patreon.AccessTokenURL, - }, - } - - r.GET("/", func(c *gin.Context) { - identity, err := session.GetIdentity(c) - if err != nil { - identity = &api.Identity{} - } - - c.HTML(http.StatusOK, "index.html", identity) - }) - - r.GET("/login", func(c *gin.Context) { - state, err := session.SetState(c) - if err != nil { - c.String(http.StatusInternalServerError, err.Error()) - return - } - - authURL := conf.AuthCodeURL(state) - c.Redirect(http.StatusFound, authURL) - }) - - r.GET("/logout", func(c *gin.Context) { - session.Clear(c) - - c.Redirect(http.StatusFound, "/") - }) - - r.GET("/patreon", func(c *gin.Context) { - // Validate session state - if session.GetSetate(c) != c.Query("state") { - c.String(http.StatusUnauthorized, "invalid state") - return - } - - // Exchange code with tokens - token, err := conf.Exchange(c.Request.Context(), c.Query("code")) - if err != nil { - c.String(http.StatusBadRequest, err.Error()) - return - } - - // Create Patreon client - tc := conf.Client(c.Request.Context(), token) - client := patreon.NewClient(tc) - - // Query user info from Patreon - user, err := client.FetchUser() - if err != nil { - c.String(http.StatusInternalServerError, err.Error()) - return - } - - // Determine feature level - level := api.DefaultFeatures - - if user.Data.Id == creatorID { - level = api.PodcasterFeature - } else { - amount := 0 - for _, item := range user.Included.Items { - pledge, ok := item.(*patreon.Pledge) - if ok { - amount += pledge.Attributes.AmountCents - } - } - - if amount >= 100 { - level = api.ExtendedFeatures - } - } - - identity := &api.Identity{ - UserId: user.Data.Id, - FullName: user.Data.Attributes.FullName, - Email: user.Data.Attributes.Email, - ProfileURL: user.Data.Attributes.URL, - FeatureLevel: level, - } - - session.SetIdentity(c, identity) - c.Redirect(http.StatusFound, "/") - }) - - // GET /robots.txt - r.GET("/robots.txt", func(c *gin.Context) { - c.String(http.StatusOK, `User-agent: * -Allow: /$ -Disallow: / -Host: www.podsync.net`) - }) - - // REST API - - r.GET("/api/ping", func(c *gin.Context) { - c.String(http.StatusOK, "ok") - }) - - r.POST("/api/create", func(c *gin.Context) { - req := &api.CreateFeedRequest{} - - if err := c.BindJSON(req); err != nil { - c.JSON(badRequest(err)) - return - } - - identity, err := session.GetIdentity(c) - if err != nil { - c.JSON(internalError(err)) - return - } - - hashId, err := feed.CreateFeed(req, identity) - if err != nil { - c.JSON(internalError(err)) - return - } - - c.JSON(http.StatusOK, gin.H{"id": hashId}) - }) - - r.NoRoute(func(c *gin.Context) { - hashId := c.Request.URL.Path[1:] - if hashId == "" || len(hashId) > maxHashIDLength { - c.String(http.StatusBadRequest, "invalid feed id") - return - } - - if strings.HasSuffix(hashId, ".xml") { - hashId = strings.TrimSuffix(hashId, ".xml") - } - - podcast, err := feed.GetFeed(hashId) - if err != nil { - code := http.StatusInternalServerError - if err == api.ErrNotFound { - code = http.StatusNotFound - } else { - log.Printf("server error (hash id: %s): %v", hashId, err) - } - - c.String(code, err.Error()) - return - } - - c.Data(http.StatusOK, "application/rss+xml; charset=UTF-8", podcast.Bytes()) - }) - - r.GET("/api/metadata/:hashId", func(c *gin.Context) { - hashId := c.Param("hashId") - if hashId == "" || len(hashId) > maxHashIDLength { - c.String(http.StatusBadRequest, "invalid feed id") - return - } - - feed, err := feed.GetMetadata(hashId) - if err != nil { - c.String(http.StatusInternalServerError, err.Error()) - return - } - - c.JSON(http.StatusOK, feed) - }) - - return r -} - -func badRequest(err error) (int, interface{}) { - return http.StatusBadRequest, gin.H{"error": err.Error()} -} - -func internalError(err error) (int, interface{}) { - log.Printf("server error: %v", err) - return http.StatusInternalServerError, gin.H{"error": err.Error()} -} From 05b7dd5020bb3d5469cc0ed7e209258213c4531f Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Mon, 30 Oct 2017 15:33:26 -0700 Subject: [PATCH 07/10] Rename package --- pkg/{webhooks => webhook}/hook.go | 2 +- pkg/{webhooks => webhook}/hook_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename pkg/{webhooks => webhook}/hook.go (99%) rename pkg/{webhooks => webhook}/hook_test.go (99%) diff --git a/pkg/webhooks/hook.go b/pkg/webhook/hook.go similarity index 99% rename from pkg/webhooks/hook.go rename to pkg/webhook/hook.go index 6977aa9..217970e 100644 --- a/pkg/webhooks/hook.go +++ b/pkg/webhook/hook.go @@ -1,4 +1,4 @@ -package webhooks +package webhook import ( "fmt" diff --git a/pkg/webhooks/hook_test.go b/pkg/webhook/hook_test.go similarity index 99% rename from pkg/webhooks/hook_test.go rename to pkg/webhook/hook_test.go index d1edcc2..61fd40f 100644 --- a/pkg/webhooks/hook_test.go +++ b/pkg/webhook/hook_test.go @@ -1,4 +1,4 @@ -package webhooks +package webhook import ( "testing" From 0f52d8d3fc613d4bca951f2dd62ce3a7492d651b Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Mon, 30 Oct 2017 17:26:46 -0700 Subject: [PATCH 08/10] Implement http handler for webhooks --- Gopkg.toml | 2 +- cmd/app/main.go | 38 +++++++++++++++++++++++++- docker-compose.yml | 1 + pkg/handler/handler.go | 58 +++++++++++++++++++++++++++++++++++++--- pkg/webhook/hook.go | 23 +++++----------- pkg/webhook/hook_test.go | 14 +++++----- 6 files changed, 108 insertions(+), 28 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index a52729e..5ceb3df 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -13,7 +13,7 @@ [[constraint]] name = "github.com/mxpv/patreon-go" - revision = "181da1e272784f51dea234c0e58e595321abd1ed" + version = "1.3" [[constraint]] name = "github.com/ventu-io/go-shortid" diff --git a/cmd/app/main.go b/cmd/app/main.go index f291c9a..0441eb1 100644 --- a/cmd/app/main.go +++ b/cmd/app/main.go @@ -4,11 +4,15 @@ import ( "context" "fmt" "log" + "net" "net/http" "os" "os/signal" + "strings" "syscall" + "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy" + "github.com/go-pg/pg" "github.com/mxpv/podsync/pkg/api" "github.com/mxpv/podsync/pkg/builders" "github.com/mxpv/podsync/pkg/config" @@ -16,6 +20,7 @@ import ( "github.com/mxpv/podsync/pkg/handler" "github.com/mxpv/podsync/pkg/id" "github.com/mxpv/podsync/pkg/storage" + "github.com/pkg/errors" ) func main() { @@ -42,6 +47,11 @@ func main() { panic(err) } + pg, err := createPg(cfg.PostgresConnectionURL) + if err != nil { + panic(err) + } + // Builders youtube, err := builders.NewYouTubeBuilder(cfg.YouTubeApiKey) @@ -63,7 +73,7 @@ func main() { srv := http.Server{ Addr: fmt.Sprintf(":%d", 5001), - Handler: handler.New(feed, cfg), + Handler: handler.New(feed, pg, cfg), } go func() { @@ -81,3 +91,29 @@ func main() { log.Printf("server gracefully stopped") } + +func createPg(connectionURL string) (*pg.DB, error) { + opts, err := pg.ParseURL(connectionURL) + if err != nil { + return nil, err + } + + // If host format is "projection:region:host", than use Google SQL Proxy + // See https://github.com/go-pg/pg/issues/576 + if strings.Count(opts.Addr, ":") == 2 { + log.Print("using GCP SQL proxy") + opts.Dialer = func(network, addr string) (net.Conn, error) { + return proxy.Dial(addr) + } + } + + db := pg.Connect(opts) + + // Check database connectivity + if _, err := db.ExecOne("SELECT 1"); err != nil { + db.Close() + return nil, errors.Wrap(err, "failed to check database connectivity") + } + + return db, nil +} diff --git a/docker-compose.yml b/docker-compose.yml index 6733308..cef05e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - 5001 environment: - REDIS_CONNECTION_URL=redis://redis + - POSTGRES_CONNECTION_URL={POSTGRES_CONNECTION_URL} - YOUTUBE_API_KEY={YOUTUBE_API_KEY} - VIMEO_API_KEY={VIMEO_API_KEY} - PATREON_CLIENT_ID={PATREON_CLIENT_ID} diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 18266ea..3f360ef 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -1,6 +1,8 @@ package handler import ( + "encoding/json" + "io/ioutil" "log" "net/http" "path" @@ -8,11 +10,13 @@ import ( "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" + "github.com/go-pg/pg" "github.com/mxpv/patreon-go" itunes "github.com/mxpv/podcast" "github.com/mxpv/podsync/pkg/api" "github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/session" + "github.com/mxpv/podsync/pkg/webhook" "golang.org/x/oauth2" ) @@ -31,6 +35,7 @@ type handler struct { feed feed cfg *config.AppConfig oauth2 oauth2.Config + hook *webhook.Handler } func (h handler) index(c *gin.Context) { @@ -87,7 +92,7 @@ func (h handler) patreonCallback(c *gin.Context) { // Determine feature level level := api.DefaultFeatures - if user.Data.Id == creatorID { + if user.Data.ID == creatorID { level = api.PodcasterFeature } else { amount := 0 @@ -104,7 +109,7 @@ func (h handler) patreonCallback(c *gin.Context) { } identity := &api.Identity{ - UserId: user.Data.Id, + UserId: user.Data.ID, FullName: user.Data.Attributes.FullName, Email: user.Data.Attributes.Email, ProfileURL: user.Data.Attributes.URL, @@ -192,7 +197,52 @@ func (h handler) metadata(c *gin.Context) { c.JSON(http.StatusOK, feed) } -func New(feed feed, cfg *config.AppConfig) http.Handler { +func (h handler) webhook(c *gin.Context) { + // Read body to byte array in order to verify signature first + body, err := ioutil.ReadAll(c.Request.Body) + if err != nil { + log.Printf("failed to read webhook body: %v", err) + c.Status(http.StatusBadRequest) + return + } + + // Verify signature + signature := c.GetHeader(patreon.HeaderSignature) + valid, err := patreon.VerifySignature(body, h.cfg.PatreonWebhooksSecret, signature) + if err != nil { + log.Printf("failed to verify signature: %v", err) + c.Status(http.StatusBadRequest) + return + } + + if !valid { + c.Status(http.StatusUnauthorized) + return + } + + // Get event name + eventName := c.GetHeader(patreon.HeaderEventType) + if eventName == "" { + log.Print("event name header is empty") + c.Status(http.StatusBadRequest) + return + } + + pledge := &patreon.WebhookPledge{} + if err := json.Unmarshal(body, pledge); err != nil { + c.JSON(badRequest(err)) + return + } + + if err := h.hook.Handle(&pledge.Data, eventName); err != nil { + c.JSON(internalError(err)) + return + } + + log.Printf("sucessfully processed patreon event %s (%s)", pledge.Data.ID, eventName) +} + +func New(feed feed, db *pg.DB, cfg *config.AppConfig) http.Handler { r := gin.New() r.Use(gin.Recovery()) @@ -214,6 +264,7 @@ func New(feed feed, cfg *config.AppConfig) http.Handler { h := handler{ feed: feed, cfg: cfg, + hook: webhook.NewHookHandler(db), } // OAuth 2 configuration @@ -240,6 +291,7 @@ func New(feed feed, cfg *config.AppConfig) http.Handler { r.GET("/api/ping", h.ping) r.POST("/api/create", h.create) r.GET("/api/metadata/:hashId", h.metadata) + r.POST("/api/webhooks", h.webhook) r.NoRoute(h.getFeed) diff --git a/pkg/webhook/hook.go b/pkg/webhook/hook.go index 217970e..d500d46 100644 --- a/pkg/webhook/hook.go +++ b/pkg/webhook/hook.go @@ -10,28 +10,19 @@ import ( "github.com/pkg/errors" ) -const ( - EventHeader = "X-Patreon-Event" - SignatureHeader = "X-Patreon-Signature" - - EventNameCreatePledge = "pledges:create" - EventNameUpdatePledge = "pledges:update" - EventNameDeletePledge = "pledges:delete" -) - type Handler struct { db *pg.DB } func (h Handler) toModel(pledge *patreon.Pledge) (*models.Pledge, error) { - pledgeID, err := strconv.ParseInt(pledge.Id, 10, 64) + pledgeID, err := strconv.ParseInt(pledge.ID, 10, 64) if err != nil { - return nil, errors.Wrapf(err, "failed to parse pledge id: %s", pledge.Id) + return nil, errors.Wrapf(err, "failed to parse pledge id: %s", pledge.ID) } - patronID, err := strconv.ParseInt(pledge.Relationships.Patron.Data.Id, 10, 64) + patronID, err := strconv.ParseInt(pledge.Relationships.Patron.Data.ID, 10, 64) if err != nil { - return nil, errors.Wrapf(err, "failed to parse patron id: %s", pledge.Relationships.Patron.Data.Id) + return nil, errors.Wrapf(err, "failed to parse patron id: %s", pledge.Relationships.Patron.Data.ID) } model := &models.Pledge{ @@ -72,11 +63,11 @@ func (h Handler) Handle(pledge *patreon.Pledge, event string) error { } switch event { - case EventNameCreatePledge: + case patreon.EventCreatePledge: return h.db.Insert(model) - case EventNameUpdatePledge: + case patreon.EventUpdatePledge: return h.db.Update(model) - case EventNameDeletePledge: + case patreon.EventDeletePledge: return h.db.Delete(model) default: return fmt.Errorf("unknown event: %s", event) diff --git a/pkg/webhook/hook_test.go b/pkg/webhook/hook_test.go index 61fd40f..cb05e79 100644 --- a/pkg/webhook/hook_test.go +++ b/pkg/webhook/hook_test.go @@ -14,7 +14,7 @@ func TestCreate(t *testing.T) { pledge := createPledge() hook := createHandler(t) - err := hook.Handle(pledge, EventNameCreatePledge) + err := hook.Handle(pledge, patreon.EventCreatePledge) require.NoError(t, err) model := &models.Pledge{PledgeID: 12345} @@ -27,12 +27,12 @@ func TestUpdate(t *testing.T) { pledge := createPledge() hook := createHandler(t) - err := hook.Handle(pledge, EventNameCreatePledge) + err := hook.Handle(pledge, patreon.EventCreatePledge) require.NoError(t, err) pledge.Attributes.AmountCents = 999 - err = hook.Handle(pledge, EventNameUpdatePledge) + err = hook.Handle(pledge, patreon.EventUpdatePledge) require.NoError(t, err) model := &models.Pledge{PledgeID: 12345} @@ -45,10 +45,10 @@ func TestDelete(t *testing.T) { pledge := createPledge() hook := createHandler(t) - err := hook.Handle(pledge, EventNameCreatePledge) + err := hook.Handle(pledge, patreon.EventCreatePledge) require.NoError(t, err) - err = hook.Handle(pledge, EventNameDeletePledge) + err = hook.Handle(pledge, patreon.EventDeletePledge) require.NoError(t, err) } @@ -68,7 +68,7 @@ func createHandler(t *testing.T) *Handler { func createPledge() *patreon.Pledge { pledge := &patreon.Pledge{ - Id: "12345", + ID: "12345", Type: "pledge", } @@ -76,7 +76,7 @@ func createPledge() *patreon.Pledge { pledge.Attributes.CreatedAt = patreon.NullTime{Valid: true, Time: time.Now().UTC()} pledge.Relationships.Patron = &patreon.PatronRelationship{} - pledge.Relationships.Patron.Data.Id = "67890" + pledge.Relationships.Patron.Data.ID = "67890" return pledge } From 0f60f4990e9458ddf1400be6e901cb9208dd866e Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Mon, 30 Oct 2017 17:39:42 -0700 Subject: [PATCH 09/10] Update compose file --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index cef05e3..215877f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ services: environment: - REDIS_CONNECTION_URL=redis://redis - POSTGRES_CONNECTION_URL={POSTGRES_CONNECTION_URL} + # https://console.developers.google.com/project/_/apiui/credential/serviceaccount + - GOOGLE_APPLICATION_CREDENTIALS={PATH_TO_GOOGLE_CREDENTIALS_FILE_FOR_USE_OUTSIDE_GOOGLE_CLOUD} - YOUTUBE_API_KEY={YOUTUBE_API_KEY} - VIMEO_API_KEY={VIMEO_API_KEY} - PATREON_CLIENT_ID={PATREON_CLIENT_ID} From 087eb1b6abea2f1228013f46bc5c4548622b3c8b Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Mon, 30 Oct 2017 18:29:33 -0700 Subject: [PATCH 10/10] Fix tests --- pkg/handler/handler_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/handler/handler_test.go b/pkg/handler/handler_test.go index 5c12f40..b0e04a3 100644 --- a/pkg/handler/handler_test.go +++ b/pkg/handler/handler_test.go @@ -32,7 +32,7 @@ func TestCreateFeed(t *testing.T) { feed := NewMockfeed(ctrl) feed.EXPECT().CreateFeed(gomock.Eq(req), gomock.Any()).Times(1).Return("456", nil) - srv := httptest.NewServer(New(feed, cfg)) + srv := httptest.NewServer(New(feed, nil, cfg)) defer srv.Close() query := `{"url": "https://youtube.com/channel/123", "page_size": 55, "quality": "low", "format": "audio"}` @@ -47,7 +47,7 @@ func TestCreateInvalidFeed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - srv := httptest.NewServer(New(NewMockfeed(ctrl), cfg)) + srv := httptest.NewServer(New(NewMockfeed(ctrl), nil, cfg)) defer srv.Close() query := `{}` @@ -100,7 +100,7 @@ func TestGetFeed(t *testing.T) { feed := NewMockfeed(ctrl) feed.EXPECT().GetFeed("123").Return(&podcast, nil) - srv := httptest.NewServer(New(feed, cfg)) + srv := httptest.NewServer(New(feed, nil, cfg)) defer srv.Close() resp, err := http.Get(srv.URL + "/123") @@ -115,7 +115,7 @@ func TestGetMetadata(t *testing.T) { feed := NewMockfeed(ctrl) feed.EXPECT().GetMetadata("123").Times(1).Return(&api.Feed{}, nil) - srv := httptest.NewServer(New(feed, cfg)) + srv := httptest.NewServer(New(feed, nil, cfg)) defer srv.Close() resp, err := http.Get(srv.URL + "/api/metadata/123")