Add --disable-maintainer-mode; make bison optional

Also flex is now optional.

The outputs of flex and bison are now committed.  By default they get
built, but users who want to build from git can now

    ./configure --disable-maintainer-mode

to turn off the dependency on bison and flex.

Maintainers must, of course, commit the bison and/or flex outputs when
they make changes to parser.y and/or lexer.l, respectively.

.gitignore

@@ -16,12 +16,6 @@ jq.1
 # Something delightfully recursive happens otherwise
 docs/content/2.download/source/*
 
-# Autogenerated
-lexer.c
-lexer.h
-parser.c
-parser.h
-
 # Autotools junk
 .libs
 .deps

Makefile.am

@@ -20,10 +20,18 @@ ACLOCAL_AMFLAGS = -I config/m4
 
 # While there is some autoconf macro support for lex/flex, it doesn't support
 # header file creation so we'll use good old make
+if MAINTAINER_MODE
 BUILT_SOURCES = lexer.h lexer.c parser.h parser.c version.h
 lexer.c: lexer.l
 	$(AM_V_LEX) flex -o lexer.c --header-file=lexer.h $<
 lexer.h: lexer.c
+else
+BUILT_SOURCES = 
+.y.c:
+	$(AM_V_YACC) echo "NOT building parser.c!"
+.l.c:
+	$(AM_V_LEX) echo "NOT building lexer.c!"
+endif
 
 # Tell YACC (bison) autoconf macros that you want a header file created.
 # If the --warnings=all fails, you probably have an old version of bison

README.md

@@ -12,14 +12,27 @@ If you want to hack on jq, feel free, but be warned that its internals
 are not well-documented at the moment. Bring a hard hat and a
 shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki
 
+Source tarball and built executable releases can be found on the
+homepage and on the github release page, https://github.com/stedolan/jq/releases
+
 If you're building directly from the latest git, you'll need flex,
-bison, libtool, make, autoconf and libonig installed. To build, run:
+bison (3.0 or newer), libtool, make, and autoconf installed.  To get
+regexp support you'll also need to install Oniguruma (note that jq's
+tests require regexp support to pass).  To build, run:
 
     autoreconf -i   # if building from git
     ./configure
     make -j8
     make check
 
+To build without bison or flex, add `--disable-maintainer-mode` to the
+./configure invocation:
+
+    ./configure --disable-maintainer-mode
+
+(Developers must not use `--disable-maintainer-mode`, not when making
+changes to the jq parser and/or lexer.)
+
 To build a statically linked version of jq, run:
 
     make LDFLAGS=-all-static

configure.ac

@@ -12,6 +12,7 @@ AC_CONFIG_AUX_DIR([config])
 AM_INIT_AUTOMAKE([parallel-tests foreign -Wall])
 AM_SILENT_RULES([yes])
 AM_PROG_AR
+AM_MAINTAINER_MODE([enable])
 AC_PROG_CC
 AC_PROG_CC_STDC
 AC_PROG_CPP_WERROR
@@ -27,28 +28,20 @@ dnl AC_PROG_YACC macros...
 
 dnl check bison version
 
-if test "$YACC" != "bison -y"; then
+if test "$USE_MAINTAINER_MODE" = yes; then
-  AC_MSG_CHECKING([bison version])
+  if test "$YACC" != "bison -y"; then
-  AC_MSG_RESULT([not bison])
+    AC_MSG_CHECKING([bison version])
-else
+    AC_MSG_RESULT([not bison])
-  AX_PROG_BISON_VERSION([3],
+  else
-    [],
+    AX_PROG_BISON_VERSION([3],
-    [AC_MSG_ERROR([You need bison version 3.0 or greater.])])
+      [],
-  dnl These gross square brackets keep autotools from eating the perl script...
+      [AC_MSG_ERROR([You need bison version 3.0 or greater.])])
-#  yacc_version=[$($YACC --version 2>1 | perl -pe '($_)=/([0-9]+([.][0-9]+)+)/')]
+  fi
-#  AX_COMPARE_VERSION([$yacc_version], [gt], [3],
-#  [AC_MSG_RESULT([ok])],
-#  [AC_MSG_RESULT([no])
-#   AC_MSG_ERROR([You need bison version 3.0 or greater.])])
-fi
 
-dnl
+  AC_PROG_LEX
-dnl these program checks should probably be deleted
+  if test "x$LEX" != xflex; then
-dnl
+    LEX="$SHELL $missing_dir/missing flex"
-
+  fi
-AC_PROG_LEX
-if test "x$LEX" != xflex; then
-  LEX="$SHELL $missing_dir/missing flex"
 fi
 
 

lexer.h

@@ -0,0 +1,362 @@
+#ifndef jq_yyHEADER_H
+#define jq_yyHEADER_H 1
+#define jq_yyIN_HEADER 1
+
+#line 6 "lexer.h"
+
+#line 8 "lexer.h"
+
+#define  YY_INT_ALIGNED short int
+
+/* A lexical scanner generated by flex */
+
+#define FLEX_SCANNER
+#define YY_FLEX_MAJOR_VERSION 2
+#define YY_FLEX_MINOR_VERSION 5
+#define YY_FLEX_SUBMINOR_VERSION 35
+#if YY_FLEX_SUBMINOR_VERSION > 0
+#define FLEX_BETA
+#endif
+
+/* First, we deal with  platform-specific or compiler-specific issues. */
+
+/* begin standard C headers. */
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+
+/* end standard C headers. */
+
+/* flex integer type definitions */
+
+#ifndef FLEXINT_H
+#define FLEXINT_H
+
+/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */
+
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+
+/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
+ * if you want the limit (max/min) macros for int types. 
+ */
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS 1
+#endif
+
+#include <inttypes.h>
+typedef int8_t flex_int8_t;
+typedef uint8_t flex_uint8_t;
+typedef int16_t flex_int16_t;
+typedef uint16_t flex_uint16_t;
+typedef int32_t flex_int32_t;
+typedef uint32_t flex_uint32_t;
+#else
+typedef signed char flex_int8_t;
+typedef short int flex_int16_t;
+typedef int flex_int32_t;
+typedef unsigned char flex_uint8_t; 
+typedef unsigned short int flex_uint16_t;
+typedef unsigned int flex_uint32_t;
+
+/* Limits of integral types. */
+#ifndef INT8_MIN
+#define INT8_MIN               (-128)
+#endif
+#ifndef INT16_MIN
+#define INT16_MIN              (-32767-1)
+#endif
+#ifndef INT32_MIN
+#define INT32_MIN              (-2147483647-1)
+#endif
+#ifndef INT8_MAX
+#define INT8_MAX               (127)
+#endif
+#ifndef INT16_MAX
+#define INT16_MAX              (32767)
+#endif
+#ifndef INT32_MAX
+#define INT32_MAX              (2147483647)
+#endif
+#ifndef UINT8_MAX
+#define UINT8_MAX              (255U)
+#endif
+#ifndef UINT16_MAX
+#define UINT16_MAX             (65535U)
+#endif
+#ifndef UINT32_MAX
+#define UINT32_MAX             (4294967295U)
+#endif
+
+#endif /* ! C99 */
+
+#endif /* ! FLEXINT_H */
+
+#ifdef __cplusplus
+
+/* The "const" storage-class-modifier is valid. */
+#define YY_USE_CONST
+
+#else	/* ! __cplusplus */
+
+/* C99 requires __STDC__ to be defined as 1. */
+#if defined (__STDC__)
+
+#define YY_USE_CONST
+
+#endif	/* defined (__STDC__) */
+#endif	/* ! __cplusplus */
+
+#ifdef YY_USE_CONST
+#define yyconst const
+#else
+#define yyconst
+#endif
+
+/* An opaque pointer. */
+#ifndef YY_TYPEDEF_YY_SCANNER_T
+#define YY_TYPEDEF_YY_SCANNER_T
+typedef void* yyscan_t;
+#endif
+
+/* For convenience, these vars (plus the bison vars far below)
+   are macros in the reentrant scanner. */
+#define yyin yyg->yyin_r
+#define yyout yyg->yyout_r
+#define yyextra yyg->yyextra_r
+#define yyleng yyg->yyleng_r
+#define yytext yyg->yytext_r
+#define yylineno (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno)
+#define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
+#define yy_flex_debug yyg->yy_flex_debug_r
+
+/* Size of default input buffer. */
+#ifndef YY_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k.
+ * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case.
+ * Ditto for the __ia64__ case accordingly.
+ */
+#define YY_BUF_SIZE 32768
+#else
+#define YY_BUF_SIZE 16384
+#endif /* __ia64__ */
+#endif
+
+#ifndef YY_TYPEDEF_YY_BUFFER_STATE
+#define YY_TYPEDEF_YY_BUFFER_STATE
+typedef struct yy_buffer_state *YY_BUFFER_STATE;
+#endif
+
+#ifndef YY_TYPEDEF_YY_SIZE_T
+#define YY_TYPEDEF_YY_SIZE_T
+typedef size_t yy_size_t;
+#endif
+
+#ifndef YY_STRUCT_YY_BUFFER_STATE
+#define YY_STRUCT_YY_BUFFER_STATE
+struct yy_buffer_state
+	{
+	FILE *yy_input_file;
+
+	char *yy_ch_buf;		/* input buffer */
+	char *yy_buf_pos;		/* current position in input buffer */
+
+	/* Size of input buffer in bytes, not including room for EOB
+	 * characters.
+	 */
+	yy_size_t yy_buf_size;
+
+	/* Number of characters read into yy_ch_buf, not including EOB
+	 * characters.
+	 */
+	int yy_n_chars;
+
+	/* Whether we "own" the buffer - i.e., we know we created it,
+	 * and can realloc() it to grow it, and should free() it to
+	 * delete it.
+	 */
+	int yy_is_our_buffer;
+
+	/* Whether this is an "interactive" input source; if so, and
+	 * if we're using stdio for input, then we want to use getc()
+	 * instead of fread(), to make sure we stop fetching input after
+	 * each newline.
+	 */
+	int yy_is_interactive;
+
+	/* Whether we're considered to be at the beginning of a line.
+	 * If so, '^' rules will be active on the next match, otherwise
+	 * not.
+	 */
+	int yy_at_bol;
+
+    int yy_bs_lineno; /**< The line count. */
+    int yy_bs_column; /**< The column count. */
+    
+	/* Whether to try to fill the input buffer when we reach the
+	 * end of it.
+	 */
+	int yy_fill_buffer;
+
+	int yy_buffer_status;
+
+	};
+#endif /* !YY_STRUCT_YY_BUFFER_STATE */
+
+void jq_yyrestart (FILE *input_file ,yyscan_t yyscanner );
+void jq_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
+YY_BUFFER_STATE jq_yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner );
+void jq_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
+void jq_yy_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
+void jq_yypush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
+void jq_yypop_buffer_state (yyscan_t yyscanner );
+
+YY_BUFFER_STATE jq_yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner );
+YY_BUFFER_STATE jq_yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner );
+YY_BUFFER_STATE jq_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner );
+
+void *jq_yyalloc (yy_size_t ,yyscan_t yyscanner );
+void *jq_yyrealloc (void *,yy_size_t ,yyscan_t yyscanner );
+void jq_yyfree (void * ,yyscan_t yyscanner );
+
+/* Begin user sect3 */
+
+#define jq_yywrap(n) 1
+#define YY_SKIP_YYWRAP
+
+#define yytext_ptr yytext_r
+
+#ifdef YY_HEADER_EXPORT_START_CONDITIONS
+#define INITIAL 0
+#define IN_PAREN 1
+#define IN_BRACKET 2
+#define IN_BRACE 3
+#define IN_QQINTERP 4
+#define IN_QQSTRING 5
+
+#endif
+
+#ifndef YY_NO_UNISTD_H
+/* Special case for "unistd.h", since it is non-ANSI. We include it way
+ * down here because we want the user's section 1 to have been scanned first.
+ * The user has a chance to override it with an option.
+ */
+#include <unistd.h>
+#endif
+
+#define YY_EXTRA_TYPE int
+
+int jq_yylex_init (yyscan_t* scanner);
+
+int jq_yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner);
+
+/* Accessor methods to globals.
+   These are made visible to non-reentrant scanners for convenience. */
+
+int jq_yylex_destroy (yyscan_t yyscanner );
+
+int jq_yyget_debug (yyscan_t yyscanner );
+
+void jq_yyset_debug (int debug_flag ,yyscan_t yyscanner );
+
+YY_EXTRA_TYPE jq_yyget_extra (yyscan_t yyscanner );
+
+void jq_yyset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner );
+
+FILE *jq_yyget_in (yyscan_t yyscanner );
+
+void jq_yyset_in  (FILE * in_str ,yyscan_t yyscanner );
+
+FILE *jq_yyget_out (yyscan_t yyscanner );
+
+void jq_yyset_out  (FILE * out_str ,yyscan_t yyscanner );
+
+int jq_yyget_leng (yyscan_t yyscanner );
+
+char *jq_yyget_text (yyscan_t yyscanner );
+
+int jq_yyget_lineno (yyscan_t yyscanner );
+
+void jq_yyset_lineno (int line_number ,yyscan_t yyscanner );
+
+YYSTYPE * jq_yyget_lval (yyscan_t yyscanner );
+
+void jq_yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
+
+       YYLTYPE *jq_yyget_lloc (yyscan_t yyscanner );
+    
+        void jq_yyset_lloc (YYLTYPE * yylloc_param ,yyscan_t yyscanner );
+    
+/* Macros after this point can all be overridden by user definitions in
+ * section 1.
+ */
+
+#ifndef YY_SKIP_YYWRAP
+#ifdef __cplusplus
+extern "C" int jq_yywrap (yyscan_t yyscanner );
+#else
+extern int jq_yywrap (yyscan_t yyscanner );
+#endif
+#endif
+
+#ifndef yytext_ptr
+static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner);
+#endif
+
+#ifdef YY_NEED_STRLEN
+static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
+#endif
+
+#ifndef YY_NO_INPUT
+
+#endif
+
+/* Amount of stuff to slurp up with each read. */
+#ifndef YY_READ_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k */
+#define YY_READ_BUF_SIZE 16384
+#else
+#define YY_READ_BUF_SIZE 8192
+#endif /* __ia64__ */
+#endif
+
+/* Number of entries by which start-condition stack grows. */
+#ifndef YY_START_STACK_INCR
+#define YY_START_STACK_INCR 25
+#endif
+
+/* Default declaration of generated scanner - a define so the user can
+ * easily add parameters.
+ */
+#ifndef YY_DECL
+#define YY_DECL_IS_OURS 1
+
+extern int jq_yylex \
+               (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner);
+
+#define YY_DECL int jq_yylex \
+               (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner)
+#endif /* !YY_DECL */
+
+/* yy_get_previous_state - get the state just before the EOB char was reached */
+
+#undef YY_NEW_FILE
+#undef YY_FLUSH_BUFFER
+#undef yy_set_bol
+#undef yy_new_buffer
+#undef yy_set_interactive
+#undef YY_DO_BEFORE_ACTION
+
+#ifdef YY_DECL_IS_OURS
+#undef YY_DECL_IS_OURS
+#undef YY_DECL
+#endif
+
+#line 129 "lexer.l"
+
+
+#line 361 "lexer.h"
+#undef jq_yyIN_HEADER
+#endif /* jq_yyHEADER_H */

parser.h

@@ -0,0 +1,190 @@
+/* A Bison parser, made by GNU Bison 3.0.2.  */
+
+/* Bison interface for Yacc-like parsers in C
+
+   Copyright (C) 1984, 1989-1990, 2000-2013 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* As a special exception, you may create a larger work that contains
+   part or all of the Bison parser skeleton and distribute that work
+   under terms of your choice, so long as that work isn't itself a
+   parser generator using the skeleton or a modified version thereof
+   as a parser skeleton.  Alternatively, if you modify or redistribute
+   the parser skeleton itself, you may (at your option) remove this
+   special exception, which will cause the skeleton and the resulting
+   Bison output files to be licensed under the GNU General Public
+   License without this special exception.
+
+   This special exception was added by the Free Software Foundation in
+   version 2.2 of Bison.  */
+
+#ifndef YY_YY_PARSER_H_INCLUDED
+# define YY_YY_PARSER_H_INCLUDED
+/* Debug traces.  */
+#ifndef YYDEBUG
+# define YYDEBUG 0
+#endif
+#if YYDEBUG
+extern int yydebug;
+#endif
+/* "%code requires" blocks.  */
+#line 10 "parser.y" /* yacc.c:1909  */
+
+#include "locfile.h"
+struct lexer_param;
+
+#define YYLTYPE location
+#define YYLLOC_DEFAULT(Loc, Rhs, N)             \
+  do {                                          \
+    if (N) {                                    \
+      (Loc).start = YYRHSLOC(Rhs, 1).start;     \
+      (Loc).end = YYRHSLOC(Rhs, N).end;         \
+    } else {                                    \
+      (Loc).start = YYRHSLOC(Rhs, 0).end;       \
+      (Loc).end = YYRHSLOC(Rhs, 0).end;         \
+    }                                           \
+  } while (0)
+ 
+
+#line 62 "parser.h" /* yacc.c:1909  */
+
+/* Token type.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+  enum yytokentype
+  {
+    INVALID_CHARACTER = 258,
+    IDENT = 259,
+    FIELD = 260,
+    LITERAL = 261,
+    FORMAT = 262,
+    REC = 263,
+    SETMOD = 264,
+    EQ = 265,
+    NEQ = 266,
+    DEFINEDOR = 267,
+    AS = 268,
+    DEF = 269,
+    MODULE = 270,
+    IMPORT = 271,
+    IF = 272,
+    THEN = 273,
+    ELSE = 274,
+    ELSE_IF = 275,
+    REDUCE = 276,
+    FOREACH = 277,
+    END = 278,
+    AND = 279,
+    OR = 280,
+    TRY = 281,
+    CATCH = 282,
+    LABEL = 283,
+    BREAK = 284,
+    BREAK2 = 285,
+    BREAK3 = 286,
+    SETPIPE = 287,
+    SETPLUS = 288,
+    SETMINUS = 289,
+    SETMULT = 290,
+    SETDIV = 291,
+    SETDEFINEDOR = 292,
+    LESSEQ = 293,
+    GREATEREQ = 294,
+    QQSTRING_START = 295,
+    QQSTRING_TEXT = 296,
+    QQSTRING_INTERP_START = 297,
+    QQSTRING_INTERP_END = 298,
+    QQSTRING_END = 299
+  };
+#endif
+/* Tokens.  */
+#define INVALID_CHARACTER 258
+#define IDENT 259
+#define FIELD 260
+#define LITERAL 261
+#define FORMAT 262
+#define REC 263
+#define SETMOD 264
+#define EQ 265
+#define NEQ 266
+#define DEFINEDOR 267
+#define AS 268
+#define DEF 269
+#define MODULE 270
+#define IMPORT 271
+#define IF 272
+#define THEN 273
+#define ELSE 274
+#define ELSE_IF 275
+#define REDUCE 276
+#define FOREACH 277
+#define END 278
+#define AND 279
+#define OR 280
+#define TRY 281
+#define CATCH 282
+#define LABEL 283
+#define BREAK 284
+#define BREAK2 285
+#define BREAK3 286
+#define SETPIPE 287
+#define SETPLUS 288
+#define SETMINUS 289
+#define SETMULT 290
+#define SETDIV 291
+#define SETDEFINEDOR 292
+#define LESSEQ 293
+#define GREATEREQ 294
+#define QQSTRING_START 295
+#define QQSTRING_TEXT 296
+#define QQSTRING_INTERP_START 297
+#define QQSTRING_INTERP_END 298
+#define QQSTRING_END 299
+
+/* Value type.  */
+#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
+typedef union YYSTYPE YYSTYPE;
+union YYSTYPE
+{
+#line 30 "parser.y" /* yacc.c:1909  */
+
+  jv literal;
+  block blk;
+
+#line 167 "parser.h" /* yacc.c:1909  */
+};
+# define YYSTYPE_IS_TRIVIAL 1
+# define YYSTYPE_IS_DECLARED 1
+#endif
+
+/* Location type.  */
+#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
+typedef struct YYLTYPE YYLTYPE;
+struct YYLTYPE
+{
+  int first_line;
+  int first_column;
+  int last_line;
+  int last_column;
+};
+# define YYLTYPE_IS_DECLARED 1
+# define YYLTYPE_IS_TRIVIAL 1
+#endif
+
+
+
+int yyparse (block* answer, int* errors, struct locfile* locations, struct lexer_param* lexer_param_ptr);
+
+#endif /* !YY_YY_PARSER_H_INCLUDED  */

setup.sh

@@ -13,7 +13,7 @@ if [ "clean" == "$1" ]; then
   find config -name m4 -prune -o -type f -print | xargs rm -f
   rm -rf tmp autom4te.cache
   rm -f INSTALL Makefile.in aclocal.m4 configure config.h.in ChangeLog
-  rm -f jv_utf8_tables.gen.h lexer.c lexer.h parser.c parser.h
+  rm -f jv_utf8_tables.gen.h
 elif [ "superclean" == "$1" ]; then
   # if autoconf errors during distcheck, it leaves files that need chmod'ing
   ver=$(scripts/version|tr -d '\n')