From db66f6c73a9e62df190214a3697032d095ace93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=94=D0=B8=D0=BA?= =?UTF-8?q?=D1=82=D0=BE=D1=80?= Date: Sun, 9 Feb 2020 02:51:30 +0300 Subject: [PATCH] Use cron expression to set update intevals #88 #46 --- README.md | 51 ++++++++++++++++++++++++++++---- cmd/podsync/main.go | 69 +++++++++++++++++++++++--------------------- go.mod | 1 + go.sum | 2 ++ pkg/config/config.go | 3 ++ 5 files changed, 87 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 1e26a56..317d508 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Here is an example how configuration might look like: ```toml [server] port = 8080 -data_dir = "/path/to/data/directory" +data_dir = "/app/data" # Don't change if you run podsync via docker [tokens] youtube = "{YOUTUBE_API_TOKEN}" # Tokens from `Access tokens` section @@ -54,8 +54,9 @@ vimeo = "{VIMEO_API_TOKEN}" update_period = "12h" # How often query for updates, examples: "60m", "4h", "2h45m" quality = "high" # or "low" format = "video" # or "audio" - cover_art = "{IMAGE_URL}" # Optional URL address of an image file - max_height = "720" # Optional maximal height of video, example: 720, 1080, 1440, 2160, ... + # cover_art = "{IMAGE_URL}" # Optional URL address of an image file + # max_height = "720" # Optional maximal height of video, example: 720, 1080, 1440, 2160, ... + # cron_schedule = "@every 12h" # Optional cron expression format. If set then overwrite 'update_period'. See details below ``` Episodes files will be kept at: `/path/to/data/directory/ID1`, feed will be accessible from: `http://localhost/ID1.xml` @@ -72,6 +73,44 @@ hostname = "https://my.test.host:4443" ... ``` +## Schedule via cron expression + +A cron expression represents a set of times, using 5 space-separated fields. + + Field name | Mandatory? | Allowed values | Allowed special characters + ---------- | ---------- | -------------- | -------------------------- + Minutes | Yes | 0-59 | * / , - + Hours | Yes | 0-23 | * / , - + Day of month | Yes | 1-31 | * / , - ? + Month | Yes | 1-12 or JAN-DEC | * / , - + Day of week | Yes | 0-6 or SUN-SAT | * / , - ? + +Month and Day-of-week field values are case insensitive. "SUN", "Sun", and "sun" are equally accepted. +The specific interpretation of the format is based on the Cron Wikipedia page: https://en.wikipedia.org/wiki/Cron + +### Predefined schedules + +You may use one of several pre-defined schedules in place of a cron expression. + + Entry | Description | Equivalent To + ----- | ----------- | ------------- + @yearly (or @annually) | Run once a year, midnight, Jan. 1st | 0 0 1 1 * + @monthly | Run once a month, midnight, first of month | 0 0 1 * * + @weekly | Run once a week, midnight between Sat/Sun | 0 0 * * 0 + @daily (or @midnight) | Run once a day, midnight | 0 0 * * * + @hourly | Run once an hour, beginning of hour | 0 * * * * + +### Intervals + +You may also schedule a job to execute at fixed intervals, starting at the time it's added +or cron is run. This is supported by formatting the cron spec like this: + + @every + +where "duration" is a string accepted by [time.ParseDuration](http://golang.org/pkg/time/#ParseDuration). + +For example, "@every 1h30m10s" would indicate a schedule that activates after 1 hour, 30 minutes, 10 seconds, and then every interval after that. + Server will be accessible from `http://localhost:8080`, but episode links will point to `https://my.test.host:4443/ID1/...` ## One click deployment @@ -80,12 +119,12 @@ Server will be accessible from `http://localhost:8080`, but episode links will p ## How to run -Run as binary: +### Run as binary: ``` $ ./podsync --config config.toml ``` -Run via Docker: +### Run via Docker: ``` $ docker pull mxpv/podsync:latest $ docker run \ @@ -95,7 +134,7 @@ $ docker run \ mxpv/podsync:latest ``` -Run via Docker Compose: +### Run via Docker Compose: ``` $ docker-compose up ``` diff --git a/cmd/podsync/main.go b/cmd/podsync/main.go index c5b72d4..42fe389 100644 --- a/cmd/podsync/main.go +++ b/cmd/podsync/main.go @@ -2,12 +2,15 @@ package main import ( "context" + "fmt" + "net/http" "os" "os/signal" "syscall" "time" "github.com/jessevdk/go-flags" + "github.com/robfig/cron/v3" log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" @@ -91,10 +94,6 @@ func main() { log.WithError(err).Fatal("failed to open database") } - // Queue of feeds to update - updates := make(chan *config.Feed, 16) - defer close(updates) - // Run updater thread log.Debug("creating updater") updater, err := NewUpdater(cfg, downloader, database) @@ -102,42 +101,46 @@ func main() { log.WithError(err).Fatal("failed to create updater") } + c := cron.New(cron.WithChain(cron.SkipIfStillRunning(nil))) + group.Go(func() error { - for { - select { - case feed := <-updates: + defer func() { + log.Info("shutting down cron") + c.Stop() + }() + + for _, feed := range cfg.Feeds { + if feed.CronSchedule == "" { + feed.CronSchedule = fmt.Sprintf("@every %s", feed.UpdatePeriod.String()) + } + + _, err = c.AddFunc(feed.CronSchedule, func() { + log.Debugf("adding %q to update queue", feed.URL) + if err := updater.Update(ctx, feed); err != nil { log.WithError(err).Errorf("failed to update feed: %s", feed.URL) } - case <-ctx.Done(): - return ctx.Err() - } - } - }) + }) - // Run wait goroutines for each feed configuration - for _, feed := range cfg.Feeds { - _feed := feed - group.Go(func() error { - log.Debugf("-> %s (update every %s)", _feed.URL, _feed.UpdatePeriod) + if err != nil { + log.WithError(err).Fatalf("can't create cron task for feed: %s", feed.ID) + } + + log.Debugf("-> %s (update '%s')", feed.URL, feed.CronSchedule) // Perform initial update after CLI restart - updates <- _feed - - timer := time.NewTicker(_feed.UpdatePeriod.Duration) - defer timer.Stop() - - for { - select { - case <-timer.C: - log.Debugf("adding %q to update queue", _feed.URL) - updates <- _feed - case <-ctx.Done(): - return ctx.Err() - } + if err := updater.Update(ctx, feed); err != nil { + log.WithError(err).Errorf("failed to update feed: %s", feed.URL) } - }) - } + } + + c.Start() + + for { + <-ctx.Done() + return ctx.Err() + } + }) // Run web server srv := NewServer(cfg) @@ -167,7 +170,7 @@ func main() { } }) - if err := group.Wait(); err != nil && err != context.Canceled { + if err := group.Wait(); err != nil && (err != context.Canceled && err != http.ErrServerClosed) { log.WithError(err).Error("wait error") } diff --git a/go.mod b/go.mod index f23eb01..71663e3 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/jessevdk/go-flags v1.4.0 github.com/mxpv/podcast v0.0.0-20170823220358-fe328ad87d18 github.com/pkg/errors v0.8.1 + github.com/robfig/cron/v3 v3.0.1 github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c github.com/sirupsen/logrus v1.2.0 github.com/stretchr/testify v1.3.0 diff --git a/go.sum b/go.sum index a0a6ddd..df61d66 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c h1:KhHx/Ta3c9C1gcSo5UhDeo/D4JnhnxJTrlcOEOFiMfY= github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c/go.mod h1:10FeaKUMy5t3KLsYfy54dFrq0rpwcfyKkKcF7vRGIRY= diff --git a/pkg/config/config.go b/pkg/config/config.go index 1f3c4fb..2deaaee 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -25,6 +25,9 @@ type Feed struct { // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". // NOTE: too often update check might drain your API token. UpdatePeriod Duration `toml:"update_period"` + // Cron expression format is how often to check update + // NOTE: too often update check might drain your API token. + CronSchedule string `toml:"cron_schedule"` // Quality to use for this feed Quality model.Quality `toml:"quality"` // Maximum height of video