mirror of
https://github.com/mxpv/podsync.git
synced 2024-05-11 05:55:04 +00:00
Implement Redis storage
This commit is contained in:
Maksym Pavlenko
@ -12,13 +12,19 @@ const (
|
||||
VideoFormat = Format("video")
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultPageSize = 50
|
||||
DefaultFormat = VideoFormat
|
||||
DefaultQuality = HighQuality
|
||||
)
|
||||
|
||||
type Feed struct {
|
||||
Id int64
|
||||
HashId string
|
||||
UserId string
|
||||
URL string
|
||||
PageSize int
|
||||
Quality Quality
|
||||
Format Format
|
||||
LastAccess time.Time
|
||||
Id int64 `json:"id"`
|
||||
HashId string `json:"hash_id"`
|
||||
UserId string `json:"user_id"`
|
||||
URL string `json:"url"`
|
||||
PageSize int `json:"page_size"`
|
||||
Quality Quality `json:"quality"`
|
||||
Format Format `json:"format"`
|
||||
LastAccess time.Time `json:"last_access"`
|
||||
}
|
||||
|
||||
@ -14,6 +14,7 @@ type AppConfig struct {
|
||||
PatreonClientId string `yaml:"patreonClientId"`
|
||||
PatreonSecret string `yaml:"patreonSecret"`
|
||||
PostgresConnectionURL string `yaml:"postgresConnectionUrl"`
|
||||
RedisURL string `yaml:"redisUrl"`
|
||||
}
|
||||
|
||||
func ReadConfiguration() (cfg *AppConfig, err error) {
|
||||
|
||||
188
web/pkg/storage/redis.go
Normal file
188
web/pkg/storage/redis.go
Normal file
@ -0,0 +1,188 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-redis/redis"
|
||||
"github.com/mxpv/podsync/web/pkg/api"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const expiration = 24 * time.Hour * 90
|
||||
|
||||
// Backward compatible Redis storage for feeds
|
||||
type RedisStorage struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func (r *RedisStorage) makeURL(m map[string]string) (string, error) {
|
||||
provider := m["provider"]
|
||||
linkType := m["type"]
|
||||
id := m["id"]
|
||||
|
||||
if provider == "" || linkType == "" || id == "" {
|
||||
return "", errors.New("failed to query URL data from storage")
|
||||
}
|
||||
|
||||
url := ""
|
||||
|
||||
if strings.EqualFold(provider, "youtube") {
|
||||
if strings.EqualFold(linkType, "channel") {
|
||||
url = "https://youtube.com/channel/" + id
|
||||
} else if strings.EqualFold(linkType, "playlist") {
|
||||
url = "https://youtube.com/playlist?list=" + id
|
||||
} else if strings.EqualFold(linkType, "user") {
|
||||
url = "https://youtube.com/user/" + id
|
||||
}
|
||||
} else if strings.EqualFold(provider, "vimeo") {
|
||||
if strings.EqualFold(linkType, "channel") {
|
||||
url = "https://vimeo.com/channels/" + id
|
||||
} else if strings.EqualFold(linkType, "user") {
|
||||
url = "https://vimeo.com/" + id
|
||||
} else if strings.EqualFold(linkType, "group") {
|
||||
url = "https://vimeo.com/groups/" + id
|
||||
}
|
||||
}
|
||||
|
||||
if url == "" {
|
||||
return "", fmt.Errorf("failed to query URL (provider: %s, type: %s, id: %s)", provider, linkType, id)
|
||||
}
|
||||
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func (r *RedisStorage) parsePageSize(m map[string]string) (int, error) {
|
||||
str, ok := m["pagesize"]
|
||||
if !ok {
|
||||
return 50, nil
|
||||
}
|
||||
|
||||
size, err := strconv.ParseInt(str, 10, 32)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if size > 150 {
|
||||
return 0, errors.New("invalid page size")
|
||||
}
|
||||
|
||||
return int(size), nil
|
||||
}
|
||||
|
||||
func (r *RedisStorage) parseFormat(m map[string]string) (api.Format, api.Quality, error) {
|
||||
quality, ok := m["quality"]
|
||||
if !ok {
|
||||
return api.VideoFormat, api.HighQuality, nil
|
||||
}
|
||||
|
||||
if quality == "VideoHigh" {
|
||||
return api.VideoFormat, api.HighQuality, nil
|
||||
} else if quality == "VideoLow" {
|
||||
return api.VideoFormat, api.LowQuality, nil
|
||||
} else if quality == "AudioHigh" {
|
||||
return api.AudioFormat, api.HighQuality, nil
|
||||
} else if quality == "AudioLow" {
|
||||
return api.AudioFormat, api.LowQuality, nil
|
||||
}
|
||||
|
||||
return "", "", fmt.Errorf("unsupported formmat %s", quality)
|
||||
}
|
||||
|
||||
func (r *RedisStorage) GetFeed(hashId string) (*api.Feed, error) {
|
||||
result, err := r.client.HGetAll(hashId).Result()
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to query feed with id %s", hashId)
|
||||
}
|
||||
|
||||
// Expire after 3 month if no use
|
||||
if err := r.client.Expire(hashId, expiration).Err(); err != nil {
|
||||
return nil, errors.Wrap(err, "failed query update feed")
|
||||
}
|
||||
|
||||
feed := &api.Feed{
|
||||
PageSize: api.DefaultPageSize,
|
||||
Quality: api.DefaultQuality,
|
||||
Format: api.DefaultFormat,
|
||||
}
|
||||
|
||||
m := make(map[string]string, len(result))
|
||||
for key, val := range result {
|
||||
m[strings.ToLower(key)] = val
|
||||
}
|
||||
|
||||
j, ok := m["json"]
|
||||
if ok {
|
||||
if err := json.Unmarshal([]byte(j), feed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return feed, nil
|
||||
}
|
||||
|
||||
// Construct URL data
|
||||
url, err := r.makeURL(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
feed.URL = url
|
||||
|
||||
// Fetch user id
|
||||
patreonId, ok := m["patreonid"]
|
||||
if ok {
|
||||
feed.UserId = patreonId
|
||||
}
|
||||
|
||||
// Unpack page size
|
||||
pageSize, err := r.parsePageSize(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if patreonId == "" && pageSize > 50 {
|
||||
return nil, errors.New("wrong feed data")
|
||||
}
|
||||
|
||||
// Parse feed's format and quality
|
||||
format, quality, err := r.parseFormat(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
feed.Format = format
|
||||
quality = quality
|
||||
|
||||
return feed, nil
|
||||
}
|
||||
|
||||
func (r *RedisStorage) CreateFeed(feed *api.Feed) error {
|
||||
raw, err := json.Marshal(feed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fields := map[string]interface{}{"json": string(raw)}
|
||||
if err := r.client.HMSet(feed.HashId, fields).Err(); err != nil {
|
||||
return errors.Wrap(err, "failed to save feed")
|
||||
}
|
||||
|
||||
return r.client.Expire(feed.HashId, expiration).Err()
|
||||
}
|
||||
|
||||
func (r *RedisStorage) keys() ([]string, error) {
|
||||
return r.client.Keys("*").Result()
|
||||
}
|
||||
|
||||
func NewRedisStorage(redisUrl string) (*RedisStorage, error) {
|
||||
opts, err := redis.ParseURL(redisUrl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := redis.NewClient(opts)
|
||||
return &RedisStorage{client}, nil
|
||||
}
|
||||
69
web/pkg/storage/redis_test.go
Normal file
69
web/pkg/storage/redis_test.go
Normal file
@ -0,0 +1,69 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mxpv/podsync/web/pkg/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRedisStorage_GetFeed(t *testing.T) {
|
||||
t.Skip("run redis tests manually")
|
||||
|
||||
client := createRedisClient(t)
|
||||
|
||||
keys, err := client.keys()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.True(t, len(keys) > 0)
|
||||
|
||||
for idx, key := range keys {
|
||||
if key == "keygen" {
|
||||
continue
|
||||
}
|
||||
|
||||
feed, err := client.GetFeed(key)
|
||||
require.NoError(t, err, "feed %s (id = %d) failed", key, idx)
|
||||
require.NotNil(t, feed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisStorage_CreateFeed(t *testing.T) {
|
||||
t.Skip("run redis tests manually")
|
||||
|
||||
client := createRedisClient(t)
|
||||
|
||||
hashId := strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
|
||||
|
||||
err := client.CreateFeed(&api.Feed{
|
||||
Id: 123,
|
||||
HashId: hashId,
|
||||
UserId: "321",
|
||||
URL: "https://youtube.com/123",
|
||||
PageSize: 45,
|
||||
Quality: api.LowQuality,
|
||||
Format: api.AudioFormat,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
feed, err := client.GetFeed(hashId)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, int64(123), feed.Id)
|
||||
require.Equal(t, hashId, feed.HashId)
|
||||
require.Equal(t, "321", feed.UserId)
|
||||
require.Equal(t, "https://youtube.com/123", feed.URL)
|
||||
require.Equal(t, 45, feed.PageSize)
|
||||
require.Equal(t, api.LowQuality, feed.Quality)
|
||||
require.Equal(t, api.AudioFormat, feed.Format)
|
||||
}
|
||||
|
||||
func createRedisClient(t *testing.T) *RedisStorage {
|
||||
client, err := NewRedisStorage("redis://localhost")
|
||||
require.NoError(t, err)
|
||||
|
||||
return client
|
||||
}
|
||||
Reference in New Issue
Block a user