mirror/stedolan-jq
1
0
Fork 0
mirror of https://github.com/stedolan/jq.git synced 2024-05-11 05:55:39 +00:00
Code Issues Releases Activity

Add -I / --online-input for huge top-level arrays

Browse Source
This commit is contained in:
Nicolas Williams
2013-11-29 17:41:04 -06:00
parent 4e07eac356
commit 77936a594d
4 changed files with 31 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
jv.h
View File

@@ -130,6 +130,9 @@ void jv_nomem_handler(jv_nomem_handler_f, void *);
jv jv_load_file(const char *, int);
typedef enum {
JV_PARSE_EXPLODE_TOPLEVEL_ARRAY = 1
} jv_parser_flags;
struct jv_parser;
struct jv_parser* jv_parser_new();
void jv_parser_set_buf(struct jv_parser*, const char*, int, int);

2
jv_file.c
View File

@@ -18,7 +18,7 @@ jv jv_load_file(const char* filename, int raw) {
data = jv_string("");
} else {
data = jv_array();
parser = jv_parser_new();
parser = jv_parser_new(0);
}
while (!feof(file) && !ferror(file)) {
char buf[4096];

24
jv_parse.c
View File

@@ -29,6 +29,8 @@ struct jv_parser {
int stacklen;
jv next;
jv_parser_flags flags;
char* tokenbuf;
int tokenpos;
int tokenlen;
@@ -45,7 +47,8 @@ struct jv_parser {
};
static void parser_init(struct jv_parser* p) {
static void parser_init(struct jv_parser* p, jv_parser_flags flags) {
p->flags = flags;
p->stack = 0;
p->stacklen = p->stackpos = 0;
p->next = jv_invalid();
@@ -110,6 +113,9 @@ static pfunc token(struct jv_parser* p, char ch) {
break;
case ',':
if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) &&
jv_get_kind(p->stack[0]) == JV_KIND_ARRAY)
return 0;
if (!jv_is_valid(p->next))
return "Expected value before ','";
if (p->stackpos == 0)
@@ -133,16 +139,22 @@ static pfunc token(struct jv_parser* p, char ch) {
if (p->stackpos == 0 || jv_get_kind(p->stack[p->stackpos-1]) != JV_KIND_ARRAY)
return "Unmatched ']'";
if (jv_is_valid(p->next)) {
if (p->stackpos != 1 || !(p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) {
p->stack[p->stackpos-1] = jv_array_append(p->stack[p->stackpos-1], p->next);
p->next = jv_invalid();
}
} else {
if (jv_array_length(jv_copy(p->stack[p->stackpos-1])) != 0) {
// this case hits on input like [1,2,3,]
return "Expected another array element";
}
}
if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) {
jv_free(p->stack[--p->stackpos]);
} else {
jv_free(p->next);
p->next = p->stack[--p->stackpos];
}
break;
case '}':
@@ -315,7 +327,9 @@ static chclass classify(char c) {
static const presult OK = "output produced";
static int check_done(struct jv_parser* p, jv* out) {
if (p->stackpos == 0 && jv_is_valid(p->next)) {
if ((p->stackpos == 0 && jv_is_valid(p->next)) ||
(p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) &&
jv_get_kind(p->stack[0]) == JV_KIND_ARRAY && jv_is_valid(p->next))) {
*out = p->next;
p->next = jv_invalid();
return 1;
@@ -370,9 +384,9 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
return answer;
}
struct jv_parser* jv_parser_new() {
struct jv_parser* jv_parser_new(jv_parser_flags flags) {
struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser));
parser_init(p);
parser_init(p, flags);
return p;
}
@@ -444,7 +458,7 @@ jv jv_parser_next(struct jv_parser* p) {
jv jv_parse_sized(const char* string, int length) {
struct jv_parser parser;
parser_init(&parser);
parser_init(&parser, 0);
jv_parser_set_buf(&parser, string, length, 0);
jv value = jv_parser_next(&parser);
if (jv_is_valid(value)) {

5
main.c
View File

@@ -159,6 +159,7 @@ int main(int argc, char* argv[]) {
ninput_files = 0;
int further_args_are_files = 0;
int jq_flags = 0;
jv_parser_flags parser_flags = 0;
jv program_arguments = jv_array();
for (int i=1; i<argc; i++) {
if (further_args_are_files) {
@@ -196,6 +197,8 @@ int main(int argc, char* argv[]) {
options |= FROM_FILE;
} else if (isoption(argv[i], 'e', "exit-status")) {
options |= EXIT_STATUS;
} else if (isoption(argv[i], 'I', "online-input")) {
parser_flags = JV_PARSE_EXPLODE_TOPLEVEL_ARRAY;
} else if (isoption(argv[i], 0, "arg")) {
if (i >= argc - 2) {
fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
@@ -279,7 +282,7 @@ int main(int argc, char* argv[]) {
slurped = jv_array();
}
}
struct jv_parser* parser = jv_parser_new();
struct jv_parser* parser = jv_parser_new(parser_flags);
char buf[4096];
while (read_more(buf, sizeof(buf))) {
if (options & RAW_INPUT) {