mirror/gohugoio-hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-05-11 05:54:58 +00:00
Code Issues Releases Activity

Add a consolidated file cache

Browse Source 
This commits reworks how file caching is performed in Hugo. Now there is only one way, and it can be configured.

This is the default configuration:

```toml
[caches]
[caches.getjson]
dir = ":cacheDir"
maxAge = -1
[caches.getcsv]
dir = ":cacheDir"
maxAge = -1
[caches.images]
dir = ":resourceDir/_gen"
maxAge = -1
[caches.assets]
dir = ":resourceDir/_gen"
maxAge = -1
```

You can override any of these cache setting in your own `config.toml`.

The placeholders explained:

`:cacheDir`: This is the value of the `cacheDir` config option if set (can also be set via OS env variable `HUGO_CACHEDIR`). It will fall back to `/opt/build/cache/hugo_cache/` on Netlify, or a `hugo_cache` directory below the OS temp dir for the others.
`:resourceDir`: This is the value of the `resourceDir` config option.

`maxAge` is the time in seconds before a cache entry will be evicted, -1 means forever and 0 effectively turns that particular cache off.

This means that if you run your builds on Netlify, all caches configured with `:cacheDir` will be saved and restored on the next build. For other CI vendors, please read their documentation. For an CircleCI example, see 6c3960a8f4/.circleci/config.yml

Fixes #5404
This commit is contained in:
Bjørn Erik Pedersen
2018-11-08 10:24:13 +01:00
parent 7d78a2afd3
commit f7aeaa6129
26 changed files with 1192 additions and 543 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
hugolib/prune_resources.go
View File

@@ -14,7 +14,6 @@
package hugolib
import (
"fmt"
"io"
"os"
"strings"
@@ -27,21 +26,12 @@ import (
// GC requires a build first.
func (h *HugoSites) GC() (int, error) {
s := h.Sites[0]
fs := h.PathSpec.BaseFs.Resources.Fs
assetsCacheFs := h.Deps.FileCaches.AssetsCache().Fs
imageCacheFs := h.Deps.FileCaches.ImageCache().Fs
imageCacheDir := s.ResourceSpec.GenImagePath
if len(imageCacheDir) < 10 {
panic("invalid image cache")
}
assetsCacheDir := s.ResourceSpec.GenAssetsPath
if len(assetsCacheDir) < 10 {
panic("invalid assets cache")
}
isImageInUse := func(filename string) bool {
key := strings.TrimPrefix(filename, imageCacheDir)
isImageInUse := func(name string) bool {
for _, site := range h.Sites {
if site.ResourceSpec.IsInImageCache(key) {
if site.ResourceSpec.IsInImageCache(name) {
return true
}
}
@@ -49,10 +39,9 @@ func (h *HugoSites) GC() (int, error) {
return false
}
isAssetInUse := func(filename string) bool {
key := strings.TrimPrefix(filename, assetsCacheDir)
isAssetInUse := func(name string) bool {
// These assets are stored in tuplets with an added extension to the key.
key = strings.TrimSuffix(key, helpers.Ext(key))
key := strings.TrimSuffix(name, helpers.Ext(name))
for _, site := range h.Sites {
if site.ResourceSpec.ResourceCache.Contains(key) {
return true
@@ -62,17 +51,13 @@ func (h *HugoSites) GC() (int, error) {
return false
}
walker := func(dirname string, inUse func(filename string) bool) (int, error) {
walker := func(fs afero.Fs, dirname string, inUse func(filename string) bool) (int, error) {
counter := 0
err := afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error {
if info == nil {
return nil
}
if !strings.HasPrefix(path, dirname) {
return fmt.Errorf("Invalid state, walk outside of resource dir: %q", path)
}
if info.IsDir() {
f, err := fs.Open(path)
if err != nil {
@@ -103,8 +88,8 @@ func (h *HugoSites) GC() (int, error) {
return counter, err
}
imageCounter, err1 := walker(imageCacheDir, isImageInUse)
assetsCounter, err2 := walker(assetsCacheDir, isAssetInUse)
imageCounter, err1 := walker(imageCacheFs, "", isImageInUse)
assetsCounter, err2 := walker(assetsCacheFs, "", isAssetInUse)
totalCount := imageCounter + assetsCounter
if err1 != nil {