New feature: require_glob() (similar to require() but supports globs) (#804)

* Initial implementation of findFiles/globe/glob

* Fixed path, some small improvements

* filepath.Dir() calls Clean() automatically anyway

* Relative path support (like require()), renamed func

* Check file ext prefix, further comments, var renaming

* Updated static.go after merge

* Added doc for glob()

* Tiny adjustment of description of glob()

* Updated docs for possible pattern

* Reworked glob, added public-facing require_glob()

* Updated docs with examples

* Updated static.go

* go generate

pkg/js/js.go

@@ -3,12 +3,12 @@ package js
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"path/filepath"
-	"strings"
-
 	"github.com/robertkrimen/otto"              // load underscore js into vm by default
 	_ "github.com/robertkrimen/otto/underscore" // required by otto
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/StackExchange/dnscontrol/v3/models"
 	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
@@ -31,12 +31,13 @@ func ExecuteJavascript(file string, devMode bool) (*models.DNSConfig, error) {
 	}
 
 	// Record the directory path leading up to this file.
-	currentDirectory = filepath.Clean(filepath.Dir(file))
+	currentDirectory = filepath.Dir(file)
 
 	vm := otto.New()
 
 	vm.Set("require", require)
 	vm.Set("REV", reverse)
+	vm.Set("glob", listFiles) // used for require_glob()
 
 	helperJs := GetHelpers(devMode)
 	// run helper script to prime vm and initialize variables
@@ -87,10 +88,11 @@ func require(call otto.FunctionCall) otto.Value {
 	// Record the old currentDirectory so that we can return there.
 	currentDirectoryOld := currentDirectory
 	// Record the directory path leading up to the file we're about to require.
-	currentDirectory = filepath.Clean(filepath.Dir(cleanFile))
+	currentDirectory = filepath.Dir(cleanFile)
 
 	printer.Debugf("requiring: %s (%s)\n", file, relFile)
-	data, err := ioutil.ReadFile(relFile)
+	// quick fix, by replacing to linux slashes, to make it work with windows paths too.
+	data, err := ioutil.ReadFile(filepath.ToSlash(relFile))
 
 	if err != nil {
 		throw(call.Otto, err.Error())
@@ -116,6 +118,94 @@ func require(call otto.FunctionCall) otto.Value {
 	return value
 }
 
+func listFiles(call otto.FunctionCall) otto.Value {
+	// Check amount of arguments provided
+	if ! (len(call.ArgumentList) >= 1 && len(call.ArgumentList) <= 3) {
+		throw(call.Otto, "glob requires at least one argument: folder (string). " +
+			"Optional: recursive (bool) [true], fileExtension (string) [.js]")
+	}
+
+	// Check if provided parameters are valid
+	// First: Let's check dir.
+	if !(call.Argument(0).IsDefined() && call.Argument(0).IsString() &&
+		len(call.Argument(0).String()) > 0) {
+		throw(call.Otto, "glob: first argument needs to be a path, provided as string.")
+	}
+	dir := call.Argument(0).String() // Path where to start listing
+	printer.Debugf("listFiles: cd: %s, user: %s \n", currentDirectory, dir)
+	// now we always prepend the current directory we're working in, which is being set within
+	// the func ExecuteJavascript() above. So when require("domains/load_all.js") is being used,
+	// where glob("customer1/") is being used, we basically search for files in domains/customer1/.
+	dir = filepath.ToSlash(filepath.Join(currentDirectory, dir))
+
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		throw(call.Otto, "glob: provided path does not exist.")
+	}
+
+	// Second: Recursive?
+	var recursive bool = true;
+	if call.Argument(1).IsDefined() && ! call.Argument(1).IsNull() {
+		if call.Argument(1).IsBoolean() {
+			recursive, _ = call.Argument(1).ToBoolean() // If it should be recursive
+		} else {
+			throw(call.Otto, "glob: second argument, if recursive, needs to be bool.")
+		}
+	}
+
+	// Third: File extension filter.
+	var fileExtension string = ".js";
+	if call.Argument(2).IsDefined() && ! call.Argument(2).IsNull() {
+		if call.Argument(2).IsString() {
+			fileExtension = call.Argument(2).String() // Which file extension to filter for.
+			if ! strings.HasPrefix(fileExtension, ".") {
+				// If it doesn't start with a dot, probably user forgot it and we do it instead.
+				fileExtension = "." + fileExtension
+			}
+		} else {
+			throw(call.Otto, "glob: third argument, file extension, needs to be a string. * for no filter.")
+		}
+	}
+
+	// Now we're doing the actual work: Listing files.
+	// Folders are ending with a slash. Can be identified later on from the user with JavaScript.
+	// Additionally, when more smart logic required, user can use regex in JS.
+	files := make([]string, 0) // init files list
+	dirClean := filepath.Clean(dir) // let's clean it here once, instead of over-and-over again within loop
+	err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+		// quick fix to get it working on windows, as it returns paths with double-backslash, what usually
+		// require() doesn't seem to handle well. For the sake of compatibility (and because slash looks nicer),
+		// we simply replace "\\" to "/" using filepath.ToSlash()..
+		path = filepath.ToSlash(filepath.Clean(path)) // convert to slashes for directories
+		if ! recursive && fi.IsDir() {
+			// If recursive is disabled, it is a dir what we're processing, and the path is different
+			// than specified, we're apparently in a different folder. Therefore: Skip it.
+			// So: Why this way? Because Walk() is always recursive and otherwise would require a complete
+			// different function to handle this scenario. This way it's easier to maintain.
+			if path != dirClean {
+				return filepath.SkipDir
+			}
+		}
+		if fileExtension != "*" && fileExtension != filepath.Ext(path) {
+			// ONLY skip, when the file extension is NOT matching, or when filter is NOT disabled.
+			return nil
+		}
+		//dirPath := filepath.ToSlash(filepath.Dir(path)) + "/"
+		files = append(files, path)
+		return err
+	})
+	if err != nil {
+		throw(call.Otto, fmt.Sprintf("dirwalk failed: %v", err.Error()))
+	}
+
+	// let's pass the data back to the JS engine.
+	value, err := call.Otto.ToValue(files)
+	if err != nil {
+		throw(call.Otto, fmt.Sprintf("converting value failed: %v", err.Error()))
+	}
+
+	return value
+}
+
 func throw(vm *otto.Otto, str string) {
 	panic(vm.MakeCustomError("Error", str))
 }