Split man.test to make tests pass without oniguruma (#2722)

.gitattributes

@@ -9,5 +9,6 @@ src/decNumber/** linguist-vendored
 src/lexer.[ch] linguist-generated=true
 src/parser.[ch] linguist-generated=true
 tests/man.test linguist-generated=true
+tests/manonig.test linguist-generated=true
 jq.1.prebuilt linguist-generated=true
 docs/Pipfile.lock linguist-generated=true

.github/workflows/manpage.yml

@@ -1,16 +1,18 @@
-name: Building man page and man.test
+name: Building man page, man.test, manonig.test
 on:
   push:
     paths:
       - '.github/workflows/manpage.yml'
       - 'docs/**'
       - 'tests/man.test'
+      - 'tests/manonig.test'
       - 'jq.1.prebuilt'
   pull_request:
     paths:
       - '.github/workflows/manpage.yml'
       - 'docs/**'
       - 'tests/man.test'
+      - 'tests/manonig.test'
       - 'jq.1.prebuilt'
 
 jobs:
@@ -33,14 +35,14 @@ jobs:
         run: |
           autoreconf -i
           ./configure --enable-docs
-      - name: Build man page and man.test
+      - name: Build man page, man.test, manonig.test
         run: |
           mv jq.1.prebuilt jq.1.old
-          rm -f tests/man.test
+          rm -f tests/man.test manonig.test
-          make jq.1.prebuilt tests/man.test
+          make jq.1.prebuilt tests/man.test tests/manonig.test
-      - name: Make sure that jq.1.prebuilt and man.test are up to date
+      - name: Make sure that jq.1.prebuilt, man.test, manonig.test are up to date
         run: |
-          git diff --exit-code tests/man.test
+          git diff --exit-code tests/man.test tests/manonig.test
           # skip build date in jq.1.prebuilt
           test -s jq.1.prebuilt
           diff -- <(tail -n +3 jq.1.old) <(tail -n +3 jq.1.prebuilt)

Makefile.am

@@ -134,7 +134,7 @@ endif
 
 ### Tests (make check)
 
-TESTS = tests/optionaltest tests/mantest tests/jqtest tests/onigtest tests/shtest tests/utf8test tests/base64test
+TESTS = tests/optionaltest tests/mantest tests/jqtest tests/shtest tests/utf8test tests/base64test
 AM_TESTS_ENVIRONMENT = JQ=$(abs_builddir)/jq NO_VALGRIND=$(NO_VALGRIND)
 
 # This is a magic make variable that causes it to treat tests/man.test as a
@@ -150,9 +150,9 @@ check_DATA = tests/man.test
 # Making changes to the manpage without having the python deps means your
 # tests won't run. If you aren't making changes to the examples, you probably
 # don't care. But if you are, then you need to run the tests anyway.
-tests/man.test: $(srcdir)/docs/content/manual/manual.yml
+tests/man.test tests/manonig.test: $(srcdir)/docs/content/manual/manual.yml
 if ENABLE_DOCS
-	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; $(PIPENV) run python build_mantests.py ) > $@
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; $(PIPENV) run python build_mantests.py )
 else
 	@echo Changes to the manual.yml require docs to be enabled to update the manual test.
 	@echo As a result, the manual test is out of date.
@@ -186,6 +186,10 @@ endif
 
 AM_CFLAGS += $(onig_CFLAGS)
 
+if WITH_ONIGURUMA
+TESTS += tests/onigtest tests/manonigtest
+endif
+
 ### Packaging
 
 docs/site.yml: configure.ac
@@ -216,9 +220,9 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         tests/modules/test_bind_order2.jq                               \
         tests/onig.supp tests/local.supp                                \
         tests/setup tests/torture/input0.json                           \
-        tests/optional.test tests/man.test tests/jq.test                \
+        tests/optional.test tests/man.test tests/manonig.test           \
-        tests/onig.test tests/base64.test tests/utf8-truncate.jq        \
+        tests/jq.test tests/onig.test tests/base64.test                 \
-        tests/jq-f-test.sh
+        tests/utf8-truncate.jq tests/jq-f-test.sh
 
 AM_DISTCHECK_CONFIGURE_FLAGS=--disable-maintainer-mode --with-oniguruma=builtin
 

README.md

@@ -20,8 +20,7 @@ homepage and on the github release page, https://github.com/jqlang/jq/releases
 If you're building directly from the latest git, you'll need flex,
 bison (3.0 or newer), libtool, make, automake, and autoconf installed.
 To get regexp support you'll also need to install Oniguruma or clone it as a
-git submodule as per the instructions below.
+git submodule as per the instructions below. To build, run:
-(note that jq's tests require regexp support to pass).  To build, run:
 
     git submodule update --init # if building from git to get oniguruma
     autoreconf -i               # if building from git

configure.ac

@@ -288,6 +288,7 @@ AC_SUBST(onig_CFLAGS)
 AC_SUBST(onig_LDFLAGS)
 
 AM_CONDITIONAL([BUILD_ONIGURUMA], [test "x$build_oniguruma" = xyes])
+AM_CONDITIONAL([WITH_ONIGURUMA], [test "x$with_oniguruma" != xno])
 AC_SUBST([BUNDLER], ["$bundle_cmd"])
 
 AC_CONFIG_MACRO_DIRS([config/m4 m4])

docs/README.md

@@ -5,22 +5,22 @@ The jq website, manpages and some of the tests are generated from this
 directory. The manual is a YAML file in `content/manual`.
 
 To build the documentation (including building the jq manpage), you'll
-need python3 and pipenv. You can install pipenv like so:
+need `python3` and `pipenv`. You can install `pipenv` like so:
 
     pip install pipenv
 
-Though, you may need to say pip3 instead, depending on your system. Once
+Though, you may need to say `pip3` instead, depending on your system. Once
-you have pipenv installed, you can install the dependencies by running
+you have `pipenv` installed, you can install the dependencies by running
-`pipenv install` from the `docs` directory.
+`pipenv sync` from the `docs` directory.
 
 Also, you may need to run `virtualenv -p /usr/bin/python3 venv/` and
-then `source venv/bin/activate`, and only then `pipenv install`.
+then `source venv/bin/activate`, and only then `pipenv sync`.
 
 Once this is done, rerun `./configure` in the jq root directory and then
 the Makefile will be able to generate the jq manpage.  You can also just
 run `pipenv run build_manpage.py` in the `docs` directory to build the
 `jq.1` page manually, and `pipenv run build_mantests.py` to build the
-contents of `tests/man.test`.
+contents of `tests/man.test` and `tests/manonig.test`.
 
 To build the website, run `pipenv run ./build_website.py` from inside
 the `docs` directory.

docs/build_mantests.py

@@ -1,13 +1,21 @@
 #!/usr/bin/env python3
 import yaml
+import re
 
-with open("content/manual/manual.yml") as f:
+regex_program_pattern = re.compile(
-    manual = yaml.safe_load(f)
+    r'\b(?:test|match|capture|scan|split|splits|sub|gsub)\s*\(')
+
+with open('content/manual/manual.yml') as source, \
+        open('../tests/man.test', 'w') as man, \
+        open('../tests/manonig.test', 'w') as manonig:
+    manual = yaml.safe_load(source)
     for section in manual.get('sections', []):
         for entry in section.get('entries', []):
             for example in entry.get('examples', []):
-                print(example.get('program', '').replace('\n', ' '))
+                program = example.get('program', '').replace('\n', ' ')
-                print(example.get('input', ''))
+                out = manonig if regex_program_pattern.search(program) else man
+                print(program, file=out)
+                print(example.get('input', ''), file=out)
                 for s in example.get('output', []):
-                    print(s)
+                    print(s, file=out)
-                print('')
+                print('', file=out)

tests/man.test

@@ -564,10 +564,6 @@ implode
 [65, 66, 67]
 "ABC"
 
-split(", ")
-"a, b,c,d, e, "
-["a","b,c,d","e",""]
-
 join(", ")
 ["a","b,c,d","e"]
 "a, b,c,d, e"
@@ -612,10 +608,6 @@ walk(if type == "array" then sort else . end)
 [[4, 1, 7], [8, 5, 2], [3, 6, 9]]
 [[1,4,7],[2,5,8],[3,6,9]]
 
-walk( if type == "object" then with_entries( .key |= sub( "^_+"; "") ) else . end )
-[ { "_a": { "__b": 2 } } ]
-[{"a":{"b":2}}]
-
 $ENV.PAGER
 null
 "less"
@@ -755,42 +747,6 @@ true
 ["1", "invalid", "3", 4]
 [1, 3, 4]
 
-test("foo")
-"foo"
-true
-
-.[] | test("a b c # spaces are ignored"; "ix")
-["xabcd", "ABC"]
-true
-true
-
-match("(abc)+"; "g")
-"abc abc"
-{"offset": 0, "length": 3, "string": "abc", "captures": [{"offset": 0, "length": 3, "string": "abc", "name": null}]}
-{"offset": 4, "length": 3, "string": "abc", "captures": [{"offset": 4, "length": 3, "string": "abc", "name": null}]}
-
-match("foo")
-"foo bar foo"
-{"offset": 0, "length": 3, "string": "foo", "captures": []}
-
-match(["foo", "ig"])
-"foo bar FOO"
-{"offset": 0, "length": 3, "string": "foo", "captures": []}
-{"offset": 8, "length": 3, "string": "FOO", "captures": []}
-
-match("foo (?<bar123>bar)? foo"; "ig")
-"foo bar foo foo  foo"
-{"offset": 0, "length": 11, "string": "foo bar foo", "captures": [{"offset": 4, "length": 3, "string": "bar", "name": "bar123"}]}
-{"offset": 12, "length": 8, "string": "foo  foo", "captures": [{"offset": -1, "length": 0, "string": null, "name": "bar123"}]}
-
-[ match("."; "g")] | length
-"abc"
-3
-
-capture("(?<a>[a-z]+)-(?<n>[0-9]+)")
-"xyzzy-14"
-{ "a": "xyzzy", "n": "14" }
-
 .bar as $x | .foo | . + $x
 {"foo":10, "bar":200}
 210

tests/manonig.test

@@ -0,0 +1,44 @@
+split(", ")
+"a, b,c,d, e, "
+["a","b,c,d","e",""]
+
+walk( if type == "object" then with_entries( .key |= sub( "^_+"; "") ) else . end )
+[ { "_a": { "__b": 2 } } ]
+[{"a":{"b":2}}]
+
+test("foo")
+"foo"
+true
+
+.[] | test("a b c # spaces are ignored"; "ix")
+["xabcd", "ABC"]
+true
+true
+
+match("(abc)+"; "g")
+"abc abc"
+{"offset": 0, "length": 3, "string": "abc", "captures": [{"offset": 0, "length": 3, "string": "abc", "name": null}]}
+{"offset": 4, "length": 3, "string": "abc", "captures": [{"offset": 4, "length": 3, "string": "abc", "name": null}]}
+
+match("foo")
+"foo bar foo"
+{"offset": 0, "length": 3, "string": "foo", "captures": []}
+
+match(["foo", "ig"])
+"foo bar FOO"
+{"offset": 0, "length": 3, "string": "foo", "captures": []}
+{"offset": 8, "length": 3, "string": "FOO", "captures": []}
+
+match("foo (?<bar123>bar)? foo"; "ig")
+"foo bar foo foo  foo"
+{"offset": 0, "length": 11, "string": "foo bar foo", "captures": [{"offset": 4, "length": 3, "string": "bar", "name": "bar123"}]}
+{"offset": 12, "length": 8, "string": "foo  foo", "captures": [{"offset": -1, "length": 0, "string": null, "name": "bar123"}]}
+
+[ match("."; "g")] | length
+"abc"
+3
+
+capture("(?<a>[a-z]+)-(?<n>[0-9]+)")
+"xyzzy-14"
+{ "a": "xyzzy", "n": "14" }
+

tests/manonigtest

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+. "${0%/*}/setup" "$@"
+
+$VALGRIND $Q $JQ -L "$mods" --run-tests $JQBASEDIR/tests/manonig.test

tests/mantest

@@ -3,4 +3,4 @@
 . "${0%/*}/setup" "$@"
 
 # We set PAGER because there's a mantest for `env` that uses it.
-env PAGER=less $VALGRIND $Q $JQ -L "$mods" --run-tests < $JQBASEDIR/tests/man.test
+env PAGER=less $VALGRIND $Q $JQ -L "$mods" --run-tests $JQBASEDIR/tests/man.test