diff --git a/builtin.c b/builtin.c index 1b1cd65a..3a7a89c7 100644 --- a/builtin.c +++ b/builtin.c @@ -824,6 +824,21 @@ static jv f_env(jq_state *jq, jv input) { return env; } +static jv f_get_search_list(jq_state *jq, jv input) { + jv_free(input); + return jq_get_lib_dirs(jq); +} + +static jv f_get_prog_origin(jq_state *jq, jv input) { + jv_free(input); + return jq_get_prog_origin(jq); +} + +static jv f_get_jq_origin(jq_state *jq, jv input) { + jv_free(input); + return jq_get_jq_origin(jq); +} + static jv f_string_split(jq_state *jq, jv a, jv b) { if (jv_get_kind(a) != JV_KIND_STRING || jv_get_kind(b) != JV_KIND_STRING) { jv_free(a); @@ -937,6 +952,9 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_error, "error", 2}, {(cfunction_ptr)f_format, "format", 2}, {(cfunction_ptr)f_env, "env", 1}, + {(cfunction_ptr)f_get_search_list, "get_search_list", 1}, + {(cfunction_ptr)f_get_prog_origin, "get_prog_origin", 1}, + {(cfunction_ptr)f_get_jq_origin, "get_jq_origin", 1}, {(cfunction_ptr)f_match, "_match_impl", 4}, {(cfunction_ptr)f_modulemeta, "modulemeta", 1}, {(cfunction_ptr)f_input, "_input", 1}, diff --git a/bytecode.c b/bytecode.c index fed86f2b..14e413a2 100644 --- a/bytecode.c +++ b/bytecode.c @@ -9,6 +9,7 @@ #define NONE 0, 1 #define CONSTANT OP_HAS_CONSTANT, 2 #define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3 +#define GLOBAL (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4 #define BRANCH OP_HAS_BRANCH, 2 #define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3 #define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4 diff --git a/compile.c b/compile.c index 431f66eb..3aad4ab3 100644 --- a/compile.c +++ b/compile.c @@ -148,6 +148,15 @@ block gen_const(jv constant) { return inst_block(i); } +block gen_const_global(jv constant, const char *name) { + assert((opcode_describe(STORE_GLOBAL)->flags & (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING)) == + (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING)); + inst* i = inst_new(STORE_GLOBAL); + i->imm.constant = constant; + i->symbol = strdup(name); + return inst_block(i); +} + int block_is_const(block b) { return (block_is_single(b) && b.first->op == LOADK); } @@ -239,6 +248,10 @@ int block_has_only_binders_and_imports(block binders, int bindflags) { return 1; } +static int inst_is_binder(inst *i, int bindflags) { + return !((opcode_describe(i->op)->flags & bindflags) != bindflags && i->op != MODULEMETA); +} + int block_has_only_binders(block binders, int bindflags) { bindflags |= OP_HAS_BINDING; bindflags &= ~OP_BIND_WILDCARD; @@ -311,10 +324,6 @@ static int block_bind_subblock(block binder, block body, int bindflags, int brea ((bindflags & OP_BIND_WILDCARD) && i->symbol[0] == '*' && break_distance <= 3 && (i->symbol[1] == '1' + break_distance) && i->symbol[2] == '\0'))) { - if (bindflags & OP_BIND_WILDCARD) { - jv_mem_free(i->symbol); - i->symbol = strdup(binder.first->symbol); - } // bind this instruction if (i->op == CALL_JQ && i->nactuals == -1) i->nactuals = block_count_actuals(i->arglist); @@ -353,7 +362,6 @@ block block_bind(block binder, block body, int bindflags) { } block block_bind_library(block binder, block body, int bindflags, const char* libname) { - assert(block_has_only_binders(binder, bindflags)); bindflags |= OP_HAS_BINDING; int nrefs = 0; int matchlen = strlen(libname); @@ -363,13 +371,21 @@ block block_bind_library(block binder, block body, int bindflags, const char* li strcpy(matchname+matchlen,"::"); matchlen += 2; } + assert(block_has_only_binders(binder, bindflags)); for (inst *curr = binder.first; curr; curr = curr->next) { + int bindflags2 = bindflags; char* cname = curr->symbol; char* tname = malloc(strlen(curr->symbol)+matchlen+1); strcpy(tname, matchname); strcpy(tname+matchlen,cname); + + // Ew + if ((opcode_describe(curr->op)->flags & (OP_HAS_VARIABLE | OP_HAS_CONSTANT))) + bindflags2 = OP_HAS_VARIABLE | OP_HAS_BINDING; + + // This mutation is ugly, even if we undo it curr->symbol = tname; - nrefs += block_bind_subblock(inst_block(curr), body, bindflags, 0); + nrefs += block_bind_subblock(inst_block(curr), body, bindflags2, 0); curr->symbol = cname; free(tname); } @@ -440,15 +456,13 @@ jv block_take_imports(block* body) { jv imports = jv_array(); inst* top = NULL; - if (body->first->op == TOP) { + if (body->first && body->first->op == TOP) { top = block_take(body); } while (body->first && (body->first->op == MODULEMETA || body->first->op == DEPS)) { inst* dep = block_take(body); if (dep->op == DEPS) { - jv opts = jv_copy(dep->imm.constant); - opts = jv_object_set(opts,jv_string("name"),jv_string(dep->symbol)); - imports = jv_array_append(imports, opts); + imports = jv_array_append(imports, jv_copy(dep->imm.constant)); } inst_free(dep); } @@ -458,13 +472,11 @@ jv block_take_imports(block* body) { return imports; } -block gen_module(const char* name, block metadata) { +block gen_module(block metadata) { inst* i = inst_new(MODULEMETA); - i->symbol = strdup(name); i->imm.constant = block_const(metadata); if (jv_get_kind(i->imm.constant) != JV_KIND_OBJECT) i->imm.constant = jv_object_set(jv_object(), jv_string("metadata"), i->imm.constant); - i->imm.constant = jv_object_set(i->imm.constant, jv_string("name"), jv_string(name)); block_free(metadata); return inst_block(i); } @@ -475,17 +487,17 @@ jv block_module_meta(block b) { return jv_null(); } -block gen_import(const char* name, block metadata, const char* as) { +block gen_import(const char* name, block metadata, const char* as, int is_data) { assert(metadata.first == NULL || block_is_const(metadata)); inst* i = inst_new(DEPS); - i->symbol = strdup(name); jv meta; if (block_is_const(metadata)) meta = block_const(metadata); else meta = jv_object(); - if (as) - meta = jv_object_set(meta, jv_string("as"), jv_string(as)); + meta = jv_object_set(meta, jv_string("as"), jv_string(as)); + meta = jv_object_set(meta, jv_string("is_data"), is_data ? jv_true() : jv_false()); + meta = jv_object_set(meta, jv_string("relpath"), jv_string(name)); i->imm.constant = meta; block_free(metadata); return inst_block(i); @@ -797,7 +809,11 @@ block gen_or(block a, block b) { } block gen_var_binding(block var, const char* name, block body) { - return BLOCK(gen_op_simple(DUP), var, + // var bindings can be added after coding the program; leave the TOP first. + block top = gen_noop(); + if (body.first && body.first->op == TOP) + top = inst_block(block_take(&body)); + return BLOCK(top, gen_op_simple(DUP), var, block_bind(gen_op_unbound(STOREV, name), body, OP_HAS_VARIABLE)); } @@ -1084,6 +1100,13 @@ static int compile(struct bytecode* bc, block b) { code[pos++] = nesting_level(bc, arg->bound_by); code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE; } + } else if ((op->flags & OP_HAS_CONSTANT) && (op->flags & OP_HAS_VARIABLE)) { + // STORE_GLOBAL: constant global, basically + code[pos++] = jv_array_length(jv_copy(constant_pool)); + constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant)); + code[pos++] = nesting_level(bc, curr->bound_by); + uint16_t var = (uint16_t)curr->bound_by->imm.intval; + code[pos++] = var; } else if (op->flags & OP_HAS_CONSTANT) { code[pos++] = jv_array_length(jv_copy(constant_pool)); constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant)); diff --git a/compile.h b/compile.h index 25544e1e..45012847 100644 --- a/compile.h +++ b/compile.h @@ -20,6 +20,7 @@ block gen_noop(); int block_is_noop(block b); block gen_op_simple(opcode op); block gen_const(jv constant); +block gen_const_global(jv constant, const char *name); int block_is_const(block b); jv_kind block_const_kind(block b); jv block_const(block b); @@ -28,9 +29,9 @@ block gen_op_unbound(opcode op, const char* name); block gen_op_bound(opcode op, block binder); block gen_op_var_fresh(opcode op, const char* name); -block gen_module(const char* name, block metadata); +block gen_module(block metadata); jv block_module_meta(block b); -block gen_import(const char* name, block metadata, const char *as); +block gen_import(const char* name, block metadata, const char *as, int is_data); block gen_function(const char* name, block formals, block body); block gen_param_regular(const char* name); block gen_param(const char* name); diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 03553716..2c0ef071 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -185,8 +185,9 @@ sections: * `-Ldirectory` / `-L directory`: - Add `directory` to the search list for modules. See the section - on modules below. + Prepend `directory` to the search list for modules. If this + option is used then no builtin search list is used. See the + section on modules below. * `-e` / `--exit-status`: @@ -2474,9 +2475,7 @@ sections: body: | jq has a library/module system. Modules are files whose names end - in `.jq`. Modules should (but don't have to) start with a - `module` directive. Programs and modules must `import` their - dependencies, if any. + in `.jq`. Modules imported by a program are searched for in a default search path (see below). The `import` directive allows the importer to @@ -2498,29 +2497,20 @@ sections: the default is appended. The default search path is the search path given to the `-L` - command-line option, else the "$JQ_LIBRARY_PATH", if set in the - environment, else `["~/.jq", "$ORIGIN/../lib/jq"]` (on Unix) or - `["~/.jq", "$ORIGIN/lib"]` (on Windows). - - The `-L` argument's and "$JQ_LIBRARY_PATH" environment variable's - string values are split on ":" on Unix, or ";" on Windows. + command-line option, else `["~/.jq", "$ORIGIN/../lib/jq", + "$ORIGIN/../lib"]`. Null and empty string path elements terminate search path processing. - Modules may be organized into a hierarchy, with name components - separated by double-colons ("::"). For example: "foo::bar". Each - component in the name corresponds to a directory on the - filesystem. - - A module named "foo::bar" would be searched for in "foo/bar.jq" - and "foo/bar/bar.jq" in the given search path. This is intended to - allow modules to be placed in a directory along with, for example, - version control files, README files, and so on, but also to allow - for single-file modules. + A dependency with relative path "foo/bar" would be searched for in + "foo/bar.jq" and "foo/bar/bar.jq" in the given search path. This + is intended to allow modules to be placed in a directory along + with, for example, version control files, README files, and so on, + but also to allow for single-file modules. Consecutive components with the same name are not allowed to avoid - ambiguities (e.g., "foo::foo"). + ambiguities (e.g., "foo/foo"). For example, with `-L$HOME/.jq` a module `foo` can be found in `$HOME/.jq/foo.jq` and `$HOME/.jq/foo/foo.jq`. @@ -2528,38 +2518,78 @@ sections: If "$HOME/.jq" is a file, it is sourced into the main program. entries: - - title: "`module NAME {};`" + - title: "`import RelativePathString as NAME;`" body: | - Defines a module named `NAME` with the given metadata. If - given it must be the first thing in the module. Programs also - may use this directive. + Imports a module found at the given path relative to a + directory in a search path. The module's symbols are prefixed + with "NAME::". A ".jq" suffix will be added to the relative + path string. - The metadata must be a constant jq expression. It should be - an object with keys like "version", "repo", "URI", and so on. - At this time jq doesn't use this metadata, but it is made - available to users via the `modulemeta` builtin. - - - title: "`import NAME {};` `import NAME {} as NAME;`" + - title: "`import RelativePathString as NAME {};`" body: | - Imports a module named `NAME`. If the second form is used - then the module's symbols are prefixed with "NAME::". + Imports a module found at the given path relative to a + directory in a search path. The module's symbols are prefixed + with "NAME::". A ".jq" suffix will be added to the relative + path string. The metadata must be a constant jq expression. It should be - an object with keys like "version", "repo", "URI", and so on. + an object with keys like "homepage" and so on. At this time + jq only uses the "search" key/value of the metadate. The + metadata is also made available to users via the `modulemeta` + builtin. The "search" key in the metadata, if present, should have a string or array value (array of strings); this is the search path to be prefixed to the top-level search path. + - title: "`import RelativePathString as NAME;`" + body: | + + Imports a JSON file found at the given path relative to a + directory in a search path. The file's data will be available + as `$NAME::NAME`. A ".json" suffix will be added to the + relative path string. + + - title: "`import RelativePathString as $NAME {};`" + body: | + + Imports a JSON file found at the given path relative to a + directory in a search path. The file's data will be available + as `$NAME::NAME`. A ".json" suffix will be added to the + relative path string. + + The metadata must be a constant jq expression. It should be + an object with keys like "homepage" and so on. At this time + jq only uses the "search" key/value of the metadate. The + metadata is also made available to users via the `modulemeta` + builtin. + + The "search" key in the metadata, if present, should have a + string or array value (array of strings); this is the search + path to be prefixed to the top-level search path. + + - title: "`module {};`" + body: | + + This directive is entirely optional. It's not required for + proper operation. It serves only the purpose of providing + metadata that can be read with the `modulemeta` builtin. + + The metadata must be a constant jq expression. It should be + an object with keys like "homepage". At this time jq doesn't + use this metadata, but it is made available to users via the + `modulemeta` builtin. + - title: "`modulemeta`" body: | - Takes a module name as input and outputs the module's - metadata, with the module's imports (including metadata) as - the "deps" key. + Takes a module name as input and outputs the module's metadata + as an object, with the module's imports (including metadata) + as an array value for the "deps" key. Programs can use this to query a module's metadata, which they could then use to, for example, search for, download, and install missing dependencies. + diff --git a/execute.c b/execute.c index 3be6abf3..b760a7d0 100644 --- a/execute.c +++ b/execute.c @@ -508,6 +508,25 @@ jv jq_next(jq_state *jq) { break; } + case STORE_GLOBAL: { + // Get the constant + jv val = jv_array_get(jv_copy(frame_current(jq)->bc->constants), *pc++); + assert(jv_is_valid(val)); + + // Store the var + uint16_t level = *pc++; + uint16_t v = *pc++; + jv* var = frame_local_var(jq, v, level); + if (jq->debug_trace_enabled) { + printf("V%d = ", v); + jv_dump(jv_copy(val), 0); + printf(" (%d)\n", jv_get_refcnt(val)); + } + jv_free(*var); + *var = val; + break; + } + case PATH_BEGIN: { jv v = stack_pop(jq); stack_push(jq, jq->path); diff --git a/linker.c b/linker.c index e6a70569..d94f50e3 100644 --- a/linker.c +++ b/linker.c @@ -21,33 +21,71 @@ struct lib_loading_state { block *defs; uint64_t ct; }; -static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_loading_state *lib_state); +static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, + const char *as, block *out_block, + struct lib_loading_state *lib_state); + // Given a lib_path to search first, creates a chain of search paths // in the following order: // 1. lib_path -// 2. -L paths passed in on the command line (from jq_state*) -// 3. JQ_LIBRARY_PATH environment variable -jv build_lib_search_chain(jq_state *jq, jv search_path) { +// 2. -L paths passed in on the command line (from jq_state*) or builtin list +static jv build_lib_search_chain(jq_state *jq, jv search_path, jv jq_origin, jv lib_origin) { assert(jv_get_kind(search_path) == JV_KIND_ARRAY); jv expanded = jv_array(); + jv expanded_elt; jv_array_foreach(search_path, i, path) { - path = expand_path(path); - if (jv_is_valid(path)) - expanded = jv_array_append(expanded, expand_path(path)); - else - jv_free(path); + if (strcmp(".",jv_string_value(path)) == 0) { + expanded_elt = jv_copy(path); + } else if (jv_get_kind(lib_origin) == JV_KIND_STRING && + strncmp("./", jv_string_value(path),sizeof("./")-1) == 0) { + expanded_elt = jv_string_fmt("%s/%s", + jv_string_value(lib_origin), + jv_string_value(path) + sizeof ("./") - 1); + } else if (strncmp("$ORIGIN/",jv_string_value(path),sizeof("$ORIGIN/")-1) == 0) { + expanded_elt = jv_string_fmt("%s/%s", + jv_string_value(jq_origin), + jv_string_value(path) + sizeof ("$ORIGIN/") - 1); + } else { + expanded_elt = expand_path(path); + if (!jv_is_valid(expanded_elt)) { + jv_free(search_path); + jv_free(expanded); + jv_free(path); + return expanded_elt; + } + path = jv_invalid(); + } + expanded = jv_array_append(expanded, expanded_elt); + jv_free(path); } + jv_free(jq_origin); + jv_free(lib_origin); jv_free(search_path); - expanded = jv_array_concat(expanded, jq_get_lib_dirs(jq)); return expanded; } -static jv name2relpath(jv name) { - jv components = jv_string_split(jv_copy(name), jv_string("::")); +// Doesn't actually check that name not be an absolute path; we could +// just always prepend "./"! +static jv validate_relpath(jv name) { + const char *s = jv_string_value(name); + if (strchr(s, '\\')) { + jv res = jv_invalid_with_msg(jv_string_fmt("Modules must be named by relative paths using '/', not '\\' (%s)", s)); + jv_free(name); + return res; + } + jv components = jv_string_split(jv_copy(name), jv_string("/")); jv rp = jv_array_get(jv_copy(components), 0); components = jv_array_slice(components, 1, jv_array_length(jv_copy(components))); jv_array_foreach(components, i, x) { + if (!strcmp(jv_string_value(x), "..")) { + jv_free(x); + jv_free(rp); + jv_free(components); + jv res = jv_invalid_with_msg(jv_string_fmt("Relative paths to modules may not traverse to parent directories (%s)", s)); + jv_free(name); + return res; + } if (i > 0 && jv_equal(jv_copy(x), jv_array_get(jv_copy(components), i - 1))) { jv_free(x); jv_free(rp); @@ -64,22 +102,33 @@ static jv name2relpath(jv name) { return rp; } -static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) { - assert(jv_get_kind(lib_search_path) == JV_KIND_ARRAY); - assert(jv_get_kind(lib_name) == JV_KIND_STRING); +// Assumes name has been validated +static jv jv_basename(jv name) { + const char *s = jv_string_value(name); + const char *p = strrchr(s, '/'); + if (!p) + return name; + jv res = jv_string_fmt("%s", p); + jv_free(name); + return res; +} - jv rel_path = name2relpath(jv_copy(lib_name)); - if (!jv_is_valid(rel_path)) { - jv_free(lib_name); - return rel_path; - } +// Asummes validated relative path to module +static jv find_lib(jq_state *jq, jv rel_path, jv search, const char *suffix, jv jq_origin, jv lib_origin) { + if (jv_get_kind(search) != JV_KIND_ARRAY) + return jv_invalid_with_msg(jv_string_fmt("Module search path must be an array")); + if (jv_get_kind(rel_path) != JV_KIND_STRING) + return jv_invalid_with_msg(jv_string_fmt("Module path must be a string")); struct stat st; int ret; - jv lib_search_paths = build_lib_search_chain(jq, lib_search_path); + // Ideally we should cache this somewhere + search = build_lib_search_chain(jq, search, jq_origin, lib_origin); - jv_array_foreach(lib_search_paths, i, spath) { + jv bname = jv_basename(jv_copy(rel_path)); + + jv_array_foreach(search, i, spath) { if (jv_get_kind(spath) == JV_KIND_NULL) { jv_free(spath); break; @@ -89,43 +138,44 @@ static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) { jv_free(spath); continue; /* XXX report non-strings in search path?? */ } - jv testpath = jq_realpath(jv_string_fmt("%s/%s.jq", + // Try .../module/last/component.jq + jv testpath = jq_realpath(jv_string_fmt("%s/%s%s", jv_string_value(spath), - jv_string_value(rel_path))); + jv_string_value(rel_path), + suffix)); ret = stat(jv_string_value(testpath),&st); if (ret == -1 && errno == ENOENT) { jv_free(testpath); - testpath = jq_realpath(jv_string_fmt("%s/%s/%s.jq", + // Try .../module/last/component/component.jq + testpath = jq_realpath(jv_string_fmt("%s/%s/%s%s", jv_string_value(spath), jv_string_value(rel_path), - jv_string_value(lib_name))); + jv_string_value(bname), + suffix)); ret = stat(jv_string_value(testpath),&st); } if (ret == 0) { - jv_free(spath); jv_free(rel_path); - jv_free(lib_name); - jv_free(lib_search_paths); + jv_free(search); + jv_free(bname); + jv_free(spath); return testpath; } jv_free(testpath); jv_free(spath); } - jv output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", jv_string_value(lib_name))); + jv output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", jv_string_value(rel_path))); jv_free(rel_path); - jv_free(lib_name); - jv_free(lib_search_paths); + jv_free(search); + jv_free(bname); return output; } -static int version_matches(jq_state *jq, block importer, block module) { - return 1; -} - -static jv default_search(jv value) { +static jv default_search(jq_state *jq, jv value) { if (!jv_is_valid(value)) { + // dependent didn't say; prepend . to system search path listj jv_free(value); - return JV_ARRAY(jv_string("."), jv_string("$ORIGIN")); + return jv_array_concat(JV_ARRAY(jv_string(".")), jq_get_lib_dirs(jq)); } if (jv_get_kind(value) != JV_KIND_ARRAY) return JV_ARRAY(value); @@ -139,38 +189,23 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block int nerrors = 0; jv_array_foreach(deps, i, dep) { - jv name = jv_object_get(jv_copy(dep), jv_string("name")); + int is_data = jv_get_kind(jv_object_get(jv_copy(dep), jv_string("is_data"))) == JV_KIND_TRUE; + int raw = 0; + jv v = jv_object_get(jv_copy(dep), jv_string("raw")); + if (jv_get_kind(v) == JV_KIND_TRUE) + raw = 1; + jv_free(v); + jv relpath = validate_relpath(jv_object_get(jv_copy(dep), jv_string("relpath"))); jv as = jv_object_get(jv_copy(dep), jv_string("as")); - if (!jv_is_valid(as)) { - jv_free(as); - as = jv_string(""); - } - jv search = default_search(jv_object_get(dep, jv_string("search"))); - jv_array_foreach(search, k, search_elt) { - if (strcmp(".",jv_string_value(search_elt)) == 0) { - jv tsearch = jv_copy(lib_origin); - jv_free(search_elt); - search = jv_array_set(search, k, tsearch); - } else if (strncmp("./",jv_string_value(search_elt),sizeof("./")-1) == 0) { - jv tsearch = jv_string_fmt("%s/%s", - jv_string_value(lib_origin), - jv_string_value(search_elt) + sizeof ("./") - 1); - jv_free(search_elt); - search = jv_array_set(search, k, tsearch); - } else if (strncmp("$ORIGIN/",jv_string_value(search_elt),sizeof("$ORIGIN/")-1) == 0) { - jv tsearch = jv_string_fmt("%s/%s", - jv_string_value(jq_origin), - jv_string_value(search_elt) + sizeof ("$ORIGIN/") - 1); - jv_free(search_elt); - search = jv_array_set(search, k, tsearch); - } else { - jv_free(search_elt); - } - } - jv lib_path = find_lib(jq, name, search); + assert(jv_is_valid(as) && jv_get_kind(as) == JV_KIND_STRING); + jv search = default_search(jq, jv_object_get(dep, jv_string("search"))); + // dep is now freed; do not reuse + + // find_lib does a lot of work that could be cached... + jv resolved = find_lib(jq, relpath, search, is_data ? ".json" : ".jq", jv_copy(jq_origin), jv_copy(lib_origin)); // XXX ...move the rest of this into a callback. - if (!jv_is_valid(lib_path)) { - jv emsg = jv_invalid_get_msg(lib_path); + if (!jv_is_valid(resolved)) { + jv emsg = jv_invalid_get_msg(resolved); jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg))); jv_free(emsg); jv_free(as); @@ -181,26 +216,20 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block } uint64_t state_idx = 0; for (; state_idx < lib_state->ct; ++state_idx) { - if (strcmp(lib_state->names[state_idx],jv_string_value(lib_path)) == 0) + if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0) break; } if (state_idx < lib_state->ct) { // Found - // XXX Check version matching here! - if (version_matches(jq, bk, lib_state->defs[state_idx])) - bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, jv_string_value(as)); - else - // XXX Would be nice to have the dependent's name here too - jq_report_error(jq, jv_string_fmt("jq: error: version mismatch for %s", jv_string_value(name))); - jv_free(lib_path); + jv_free(resolved); + // Bind the library to the program + bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, jv_string_value(as)); } else { // Not found. Add it to the table before binding. block dep_def_block = gen_noop(); - nerrors += load_library(jq, lib_path, &dep_def_block, lib_state); + nerrors += load_library(jq, resolved, is_data, raw, jv_string_value(as), &dep_def_block, lib_state); + // resolved has been freed if (nerrors == 0) { - // XXX Check version matching here! - if (version_matches(jq, bk, dep_def_block)) - bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, jv_string_value(as)); - else - jq_report_error(jq, jv_string_fmt("jq: error: version mismatch for %s", jv_string_value(name))); + // Bind the library to the program + bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, jv_string_value(as)); } } jv_free(as); @@ -213,30 +242,48 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block // Loads the library at lib_path into lib_state, putting the library's defs // into *out_block -static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_loading_state *lib_state) { +static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, const char *as, block *out_block, struct lib_loading_state *lib_state) { int nerrors = 0; - struct locfile* src; + struct locfile* src = NULL; block program; - jv data = jv_load_file(jv_string_value(lib_path), 1); + jv data; + if (is_data && !raw) + data = jv_load_file(jv_string_value(lib_path), 0); + else + data = jv_load_file(jv_string_value(lib_path), 1); int state_idx; - if (jv_is_valid(data)) { + if (!jv_is_valid(data)) { + if (jv_invalid_has_msg(jv_copy(data))) + data = jv_invalid_get_msg(data); + else + data = jv_string("unknown error"); + jq_report_error(jq, jv_string_fmt("jq: error loading data file %s: %s\n", jv_string_value(lib_path), jv_string_value(data))); + nerrors++; + goto out; + } else if (is_data) { + // import "foo" as $bar; + program = gen_const_global(jv_copy(data), as); + } else { + // import "foo" as bar; src = locfile_init(jq, jv_string_value(data), jv_string_length_bytes(jv_copy(data))); nerrors += jq_parse_library(src, &program); if (nerrors == 0) { - state_idx = lib_state->ct++; - lib_state->names = realloc(lib_state->names, lib_state->ct * sizeof(const char *)); - lib_state->defs = realloc(lib_state->defs, lib_state->ct * sizeof(block)); - lib_state->names[state_idx] = strdup(jv_string_value(lib_path)); - lib_state->defs[state_idx] = program; char *lib_origin = strdup(jv_string_value(lib_path)); nerrors += process_dependencies(jq, jq_get_jq_origin(jq), jv_string(dirname(lib_origin)), - &lib_state->defs[state_idx], lib_state); + &program, lib_state); free(lib_origin); - *out_block = lib_state->defs[state_idx]; } - locfile_free(src); } + state_idx = lib_state->ct++; + lib_state->names = realloc(lib_state->names, lib_state->ct * sizeof(const char *)); + lib_state->defs = realloc(lib_state->defs, lib_state->ct * sizeof(block)); + lib_state->names[state_idx] = strdup(jv_string_value(lib_path)); + lib_state->defs[state_idx] = program; + *out_block = program; + if (src) + locfile_free(src); +out: jv_free(lib_path); jv_free(data); return nerrors; @@ -244,8 +291,11 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ // FIXME It'd be nice to have an option to search the same search path // as we do in process_dependencies. -jv load_module_meta(jq_state *jq, jv modname) { - jv lib_path = find_lib(jq, modname, jv_array()); +jv load_module_meta(jq_state *jq, jv mod_relpath) { + // We can't know the caller's origin; we could though, if it was passed in + jv lib_path = find_lib(jq, validate_relpath(mod_relpath), jq_get_lib_dirs(jq), ".jq", jq_get_jq_origin(jq), jv_null()); + if (!jv_is_valid(lib_path)) + return lib_path; jv meta = jv_null(); jv data = jv_load_file(jv_string_value(lib_path), 1); if (jv_is_valid(data)) { @@ -278,7 +328,7 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) { block libs = gen_noop(); for (uint64_t i = 0; i < lib_state.ct; ++i) { free(lib_state.names[i]); - if (nerrors == 0) + if (nerrors == 0 && !block_is_const(lib_state.defs[i])) libs = block_join(libs, lib_state.defs[i]); else block_free(lib_state.defs[i]); diff --git a/main.c b/main.c index 4457c58c..65c4ced4 100644 --- a/main.c +++ b/main.c @@ -444,23 +444,11 @@ int main(int argc, char* argv[]) { if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR; if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR; - char *penv = getenv("JQ_LIBRARY_PATH"); - if (penv && jv_get_kind(lib_search_paths) == JV_KIND_NULL) { - // Use $JQ_LIBRARY_PATH -#ifdef WIN32 -#define PATH_ENV_SEPARATOR ";" -#else -#define PATH_ENV_SEPARATOR ":" -#endif - lib_search_paths = jv_array_concat(lib_search_paths,jv_string_split(jv_string(penv),jv_string(PATH_ENV_SEPARATOR))); -#undef PATH_ENV_SEPARATOR - } else if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) { - // Use compiled-in default JQ_LIBRARY_PATH -#ifdef WIN32 - lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/lib")); -#else - lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/../lib/jq")); -#endif + if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) { + // Default search path list + lib_search_paths = JV_ARRAY(jv_string("~/.jq"), + jv_string("$ORIGIN/../lib/jq"), + jv_string("$ORIGIN/lib")); } jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths); diff --git a/opcode_list.h b/opcode_list.h index bdc4f48e..e38d6843 100644 --- a/opcode_list.h +++ b/opcode_list.h @@ -6,6 +6,7 @@ OP(POP, NONE, 1, 0) OP(LOADV, VARIABLE, 1, 1) OP(LOADVN, VARIABLE, 1, 1) OP(STOREV, VARIABLE, 1, 0) +OP(STORE_GLOBAL, GLOBAL, 0, 0) OP(INDEX, NONE, 2, 1) OP(INDEX_OPT, NONE, 2, 1) OP(EACH, NONE, 1, 1) diff --git a/parser.y b/parser.y index 72aad3ae..d551c0e3 100644 --- a/parser.y +++ b/parser.y @@ -280,14 +280,13 @@ Module: %empty { $$ = gen_noop(); } | -"module" IDENT Exp ';' { - if (!block_is_const($3)) { +"module" Exp ';' { + if (!block_is_const($2)) { FAIL(@$, "Module metadata must be constant."); $$ = gen_noop(); } else { - $$ = gen_module(jv_string_value($2), $3); + $$ = gen_module($2); } - jv_free($2); } Imports: @@ -479,33 +478,45 @@ Term { } Import: -"import" IDENT ';' { - $$ = gen_import(jv_string_value($2), gen_noop(), NULL); - jv_free($2); +"import" String "as" '$' IDENT ';' { + jv v = block_const($2); + // XXX Make gen_import take only blocks and the int is_data so we + // don't have to free so much stuff here + $$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($5), 1); + block_free($2); + jv_free($5); + jv_free(v); } | -"import" IDENT Exp ';' { - if (!block_is_const($3)) { - FAIL(@$, "Module metadata must be constant."); - $$ = gen_noop(); - } else { - $$ = gen_import(jv_string_value($2), $3, NULL); - } - jv_free($2); -} | -"import" IDENT "as" IDENT ';' { - $$ = gen_import(jv_string_value($2), gen_noop(), jv_string_value($4)); - jv_free($2); +"import" String "as" IDENT ';' { + jv v = block_const($2); + $$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($4), 0); + block_free($2); jv_free($4); + jv_free(v); } | -"import" IDENT "as" IDENT Exp ';' { +"import" String "as" IDENT Exp ';' { if (!block_is_const($5)) { FAIL(@$, "Module metadata must be constant."); $$ = gen_noop(); } else { - $$ = gen_import(jv_string_value($2), $5, jv_string_value($4)); + jv v = block_const($2); + $$ = gen_import(jv_string_value(v), $5, jv_string_value($4), 0); + jv_free(v); } - jv_free($2); + block_free($2); jv_free($4); +} | +"import" String "as" '$' IDENT Exp ';' { + if (!block_is_const($6)) { + FAIL(@$, "Module metadata must be constant."); + $$ = gen_noop(); + } else { + jv v = block_const($2); + $$ = gen_import(jv_string_value(v), $6, jv_string_value($5), 1); + jv_free(v); + } + block_free($2); + jv_free($5); } FuncDef: diff --git a/tests/modules/a.jq b/tests/modules/a.jq index 1bf8d069..01e3c1bb 100644 --- a/tests/modules/a.jq +++ b/tests/modules/a.jq @@ -1,2 +1,2 @@ -module a {version:1.7}; +module {version:1.7}; def a: "a"; diff --git a/tests/modules/c/c.jq b/tests/modules/c/c.jq index d5000067..4a6a8ed3 100644 --- a/tests/modules/c/c.jq +++ b/tests/modules/c/c.jq @@ -1,12 +1,14 @@ -module c {whatever:null}; -import a as foo; -import d as d {search:"./"}; -import d {search:"./"}; -import e as e {search:"./../lib/jq"}; -import f as f {search:"./../lib/jq"}; +module {whatever:null}; +import "a" as foo; +import "d" as d {search:"./"}; +import "d" as d2{search:"./"}; +import "e" as e {search:"./../lib/jq"}; +import "f" as f {search:"./../lib/jq"}; +import "data" as $d; def a: 0; def c: - if meh != d::meh then error("import into namespace doesn't work") + if $d::d[0] != {this:"is a test",that:"is too"} then error("data import is busted") + elif d2::meh != d::meh then error("import twice doesn't work") elif foo::a != "a" then error("foo::a didn't work as expected") elif d::meh != "meh" then error("d::meh didn't work as expected") elif e::bah != "bah" then error("e::bah didn't work as expected") diff --git a/tests/run b/tests/run index a7afc70a..bd227059 100755 --- a/tests/run +++ b/tests/run @@ -159,7 +159,7 @@ if which seq > /dev/null 2>&1; then fi dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null | - $VALGRIND ./jq -cn --stream -L "$mods" 'import streaming; tovalues(inputs)' > $d/out1 2>$d/err || true + $VALGRIND ./jq -cn --stream -L "$mods" 'import "streaming" as streaming; streaming::tovalues(inputs)' > $d/out1 2>$d/err || true if [ -n "$VALGRIND" ]; then grep '^==[0-9][0-9]*== ERROR SUMMARY: 0 errors' $d/err > /dev/null else @@ -200,22 +200,22 @@ if [ `HOME="$mods" $VALGRIND $Q ./jq --debug-dump-disasm -n fg | grep '^[a-z]' | exit 1 fi -if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import a as foo; import b as bar; import a; def fooa: foo::a; [fooa, bar::a, bar::b, foo::a, a] | . == ["a","b","c","a","a"]' > /dev/null; then +if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import "a" as foo; import "b" as bar; def fooa: foo::a; [fooa, bar::a, bar::b, foo::a] | . == ["a","b","c","a"]' > /dev/null; then echo "Module system appears to be broken" 1>&2 exit 1 fi -if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import c as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then +if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import "c" as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then echo "Module system appears to be broken" 1>&2 exit 1 fi -if [ "`$VALGRIND $Q ./jq -cner -L "$mods" '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"./","as":"d","name":"d"},{"search":"./","name":"d"},{"search":"./../lib/jq","as":"e","name":"e"},{"search":"./../lib/jq","as":"f","name":"f"}]}' ]; then +if [ "`$VALGRIND $Q ./jq -cner -L "$mods" '\"c\" | modulemeta'`" != '{"whatever":null,"deps":[{"as":"foo","is_data":false,"relpath":"a"},{"search":"./","as":"d","is_data":false,"relpath":"d"},{"search":"./","as":"d2","is_data":false,"relpath":"d"},{"search":"./../lib/jq","as":"e","is_data":false,"relpath":"e"},{"search":"./../lib/jq","as":"f","is_data":false,"relpath":"f"},{"as":"d","is_data":true,"relpath":"data"}]}' ]; then echo "modulemeta builtin appears to be broken" 1>&2 exit 1 fi -if $VALGRIND ./jq -ner -L "$mods" 'import syntaxerror; .' > $d/out 2>&1; then +if $VALGRIND ./jq -ner -L "$mods" 'import "syntaxerror" as e; .' > $d/out 2>&1; then echo "Module system appears to be broken" 1>&2 exit 1 fi