From 2bd28b4e85b5b5716ca0c1d0f8febaf8331e0049 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Mon, 21 Aug 2017 23:52:02 -0700 Subject: [PATCH] Minify scripts, update docker container --- .dockerignore | 5 ++ .gitignore | 3 +- Dockerfile | 13 ++++- README.md | 17 ++++++ gulpfile.js | 48 +++++++++++++++++ package.json | 32 ++++++++++++ pkg/config/config.go | 4 ++ pkg/config/config_test.go | 8 +++ pkg/server/server.go | 15 ++---- ytdl/ytdl.py | 106 -------------------------------------- 10 files changed, 133 insertions(+), 118 deletions(-) create mode 100644 .dockerignore create mode 100644 README.md create mode 100644 gulpfile.js create mode 100644 package.json delete mode 100644 ytdl/ytdl.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..be004c8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +node_modules/ +vendor/ + +package-lock.json +Gopkg.lock \ No newline at end of file diff --git a/.gitignore b/.gitignore index 19b829b..1d7a2eb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ Gopkg.lock .idea/ node_modules/ -package-lock.json \ No newline at end of file +package-lock.json +dist/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 45b263e..9e966b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,13 @@ +FROM node:latest AS gulp +WORKDIR /app +COPY . . +RUN npm install +RUN npm link gulp +RUN gulp patch + FROM golang:1.8 AS build WORKDIR /go/src/github.com/mxpv/podsync -COPY . . +COPY --from=gulp /app . ENV GOOS=linux ENV GOARCH=amd64 ENV CGO_ENABLED=0 @@ -11,5 +18,9 @@ RUN go install -v ./cmd/app FROM alpine RUN apk --update --no-cache add ca-certificates WORKDIR /app/ +COPY --from=gulp /app/templates ./templates +COPY --from=gulp /app/dist ./assets COPY --from=build /go/bin/app . +ENV ASSETS_PATH /app/assets +ENV TEMPLATES_PATH /app/templates ENTRYPOINT ["/app/app"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ec507ac --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +## Handling static files + +`ASSETS_PATH` should point to a directory with static files. +For debugging just 'Copy Path' to `assets` directory. + +For production run `gulp patch`. +Gulp will generate `dist` directory with minified files and update templates to include these files. + +`TEMPLATES_PATH` should just point to `templates` directory. + +Docker will run `gulp` and include `dist` and `templates` directories during build as well as specify `ASSETS_PATH` and `TEMPLATES_PATH` environment variables. + +## Patreon + +In order to login via Patreon the following variables should be configured: +- `PATREON_REDIRECT_URL` should point to `http://yout_host_here/patreon` +- `PATREON_CLIENT_ID` and `PATREON_SECRET` should be copied from https://www.patreon.com/platform/documentation/clients \ No newline at end of file diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000..8114d58 --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,48 @@ +var gulp = require('gulp'), + del = require('del'), + path = require('path'), + uglify = require('gulp-uglify'), + rev = require('gulp-rev'), + revreplace = require('gulp-rev-replace'), + cleancss = require('gulp-clean-css'), + autoprefixer = require('gulp-autoprefixer'), + size = require('gulp-size'), + gulpif = require('gulp-if'), + imagemin = require('gulp-imagemin'); + +abs = path.join(process.cwd(), 'assets'); + +gulp.task('clean', function () { + return del(['./dist/**/*']) +}); + +// Minify images and output to ./dist folder +gulp.task('img', ['clean'], function() { + return gulp.src('./assets/**/*.{png,ico}') + .pipe(imagemin()) + .pipe(size()) + .pipe(gulp.dest('./dist')) +}); + +// Minify scripts, build manifest.json and output to ./dist folder +gulp.task('js+css', ['clean', 'img'], function() { + return gulp.src(['./assets/js/**/*.js', './assets/css/**/*.css'], {base: abs}) + .pipe(gulpif(/js$/, uglify())) + .pipe(gulpif(/css$/, autoprefixer())) + .pipe(gulpif(/css$/, cleancss())) + .pipe(rev()) + .pipe(size()) + .pipe(gulp.dest('./dist')) + .pipe(rev.manifest('manifest.json', {merge: true})) + .pipe(gulp.dest('./dist')); +}); + +// Rewrite occurrences of scripts in template files +gulp.task('patch', ['js+css'], function() { + var manifest = gulp.src('./dist/manifest.json'); + return gulp.src('./templates/index.html') + .pipe(revreplace({manifest: manifest})) + .pipe(gulp.dest('./templates/')) +}); + +gulp.task('default', ['js+css']); \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..62422ff --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "name": "podsync", + "version": "1.0.0", + "description": "", + "main": "gulpfile.js", + "dependencies": {}, + "devDependencies": { + "del": "^3.0.0", + "gulp": "^3.9.1", + "gulp-autoprefixer": "^4.0.0", + "gulp-clean-css": "^3.7.0", + "gulp-if": "^2.0.2", + "gulp-imagemin": "^3.3.0", + "gulp-rev": "^8.0.0", + "gulp-rev-replace": "^0.4.3", + "gulp-size": "^2.1.0", + "gulp-uglify": "^3.0.0" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/mxpv/Podsync.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/mxpv/Podsync/issues" + }, + "homepage": "https://github.com/mxpv/Podsync#readme" +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 78faff0..8719a55 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,6 +17,8 @@ type AppConfig struct { PostgresConnectionURL string `yaml:"postgresConnectionUrl"` RedisURL string `yaml:"redisUrl"` CookieSecret string `yaml:"cookieSecret"` + AssetsPath string `yaml:"assetsPath"` + TemplatesPath string `yaml:"templatesPath"` } func ReadConfiguration() (cfg *AppConfig, err error) { @@ -39,6 +41,8 @@ func ReadConfiguration() (cfg *AppConfig, err error) { "postgresConnectionUrl": "POSTGRES_CONNECTION_URL", "redisUrl": "REDIS_CONNECTION_URL", "cookieSecret": "COOKIE_SECRET", + "assetsPath": "ASSETS_PATH", + "templatesPath": "TEMPLATES_PATH", } for k, v := range envmap { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index e4d1dd5..437a495 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -17,6 +17,8 @@ patreonSecret: "4" postgresConnectionUrl: "5" cookieSecret: "6" patreonRedirectUrl: "7" +assetsPath: "8" +templatesPath: "9" ` func TestReadYaml(t *testing.T) { @@ -36,6 +38,8 @@ func TestReadYaml(t *testing.T) { require.Equal(t, "5", cfg.PostgresConnectionURL) require.Equal(t, "6", cfg.CookieSecret) require.Equal(t, "7", cfg.PatreonRedirectURL) + require.Equal(t, "8", cfg.AssetsPath) + require.Equal(t, "9", cfg.TemplatesPath) } func TestReadEnv(t *testing.T) { @@ -49,6 +53,8 @@ func TestReadEnv(t *testing.T) { os.Setenv("POSTGRES_CONNECTION_URL", "55") os.Setenv("COOKIE_SECRET", "66") os.Setenv("PATREON_REDIRECT_URL", "77") + os.Setenv("ASSETS_PATH", "88") + os.Setenv("TEMPLATES_PATH", "99") cfg, err := ReadConfiguration() require.NoError(t, err) @@ -60,4 +66,6 @@ func TestReadEnv(t *testing.T) { require.Equal(t, "55", cfg.PostgresConnectionURL) require.Equal(t, "66", cfg.CookieSecret) require.Equal(t, "77", cfg.PatreonRedirectURL) + require.Equal(t, "88", cfg.AssetsPath) + require.Equal(t, "99", cfg.TemplatesPath) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 76b660e..a18ad7f 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -4,7 +4,6 @@ import ( "crypto/rand" "encoding/base64" "encoding/json" - "go/build" "log" "net/http" "path" @@ -39,9 +38,11 @@ func MakeHandlers(feed feed, cfg *config.AppConfig) http.Handler { // Static files + HTML - if cfg.PatreonRedirectURL == "" { - cfg.PatreonRedirectURL = "http://localhost:8080/patreon" - } + log.Printf("using assets path: %s", cfg.AssetsPath) + r.Static("/assets", cfg.AssetsPath) + + log.Printf("using templates path: %s", cfg.TemplatesPath) + r.LoadHTMLGlob(path.Join(cfg.TemplatesPath, "*.html")) conf := &oauth2.Config{ ClientID: cfg.PatreonClientId, @@ -54,12 +55,6 @@ func MakeHandlers(feed feed, cfg *config.AppConfig) http.Handler { }, } - rootDir := path.Join(build.Default.GOPATH, "src/github.com/mxpv/podsync") - log.Printf("Using root directory: %s", rootDir) - - r.Static("/assets", path.Join(rootDir, "assets")) - r.LoadHTMLGlob(path.Join(rootDir, "templates/*.html")) - r.GET("/", func(c *gin.Context) { s := sessions.Default(c) diff --git a/ytdl/ytdl.py b/ytdl/ytdl.py deleted file mode 100644 index 5f87294..0000000 --- a/ytdl/ytdl.py +++ /dev/null @@ -1,106 +0,0 @@ -import youtube_dl -import redis -import os -from youtube_dl.utils import DownloadError -from sanic import Sanic -from sanic.exceptions import InvalidUsage, NotFound -from sanic.response import text, redirect -from datetime import timedelta - -app = Sanic() - -db = redis.from_url(os.getenv('REDIS_CONNECTION_STRING', 'redis://localhost:6379')) -db.ping() - -opts = { - 'quiet': True, - 'no_warnings': True, - 'forceurl': True, - 'simulate': True, - 'skip_download': True, - 'call_home': False, - 'nocheckcertificate': True -} - -url_formats = { - 'youtube': 'https://youtube.com/watch?v={}', - 'vimeo': 'https://vimeo.com/{}', -} - - -@app.route('/download//', methods=['GET']) -async def download(request, feed_id, video_id): - if not feed_id: - raise InvalidUsage('Invalid feed id') - - # Remote extension and check if video id is ok - video_id = os.path.splitext(video_id)[0] - if not video_id: - raise InvalidUsage('Invalid video id') - - # Query redis - data = db.hgetall(feed_id) - if not data: - raise NotFound('Feed not found') - - # Delete this feed if no requests within 90 days - db.expire(feed_id, timedelta(days=90)) - - entries = {k.decode().lower(): v.decode().lower() for k, v in data.items()} - - # Build URL - provider = entries.get('provider') - tpl = url_formats[provider] - if not tpl: - raise InvalidUsage('Invalid feed') - - url = tpl.format(video_id) - quality = entries.get('quality') - - try: - redirect_url = _resolve(url, quality) - return redirect(redirect_url) - except DownloadError as e: - msg = str(e) - return text(msg, status=511) - - -def _resolve(url, quality): - if not url: - raise InvalidUsage('Invalid URL') - - if not quality: - quality = 'videohigh' - - try: - with youtube_dl.YoutubeDL(opts) as ytdl: - info = ytdl.extract_info(url, download=False) - return _choose_url(info, quality) - except DownloadError: - raise - except Exception as e: - print(e) - raise - - -def _choose_url(info, quality): - is_video = quality == 'videohigh' or quality == 'videolow' - - # Filter formats by file extension - ext = 'mp4' if is_video else 'm4a' - fmt_list = [x for x in info['formats'] if x['ext'] == ext and x['acodec'] != 'none'] - if not len(fmt_list): - return info['url'] - - # Sort list by field (width for videos, file size for audio) - sort_field = 'width' if is_video else 'filesize' - ordered = sorted(fmt_list, key=lambda x: x[sort_field], reverse=True) - - # Choose an item depending on quality - is_high_quality = quality == 'videohigh' or quality == 'audiohigh' - item = ordered[0] if is_high_quality else ordered[-1] - return item['url'] - - -if __name__ == '__main__': - app.run(host='0.0.0.0', port=5002, workers=32)