Save literal value of the parsed number to preserve it for the output

Extend jv_number to use decNumber for storing number literals. Any math operations on the numbers will truncate them to double precision. Comparisons when both numbers are literal numbers will compare them without truncation. Delay conversion of numbers to doubles until a math operation is performed, to preserve precision. A literal jv_number will only need conversion to double once, and will reuse the resultant double on subsequent conversions. Outputting literal jv_numbers preserves the original precision. Add strong pthread requirement to manage contexts/allocations for converting numbers between their decNumber, string, and double formats.
2024-05-11 05:55:39 +00:00 · 2018-10-19 21:57:41 +03:00
parent b6be13d5de
commit cf4b48c7ba
21 changed files with 1374 additions and 755 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -50,3 +50,4 @@ tests/*.trs
 cscope.in.out
 cscope.out
 cscope.po.out
+jq.dSYM
--- a/Makefile.am
+++ b/Makefile.am
@@ -11,6 +11,8 @@ LIBJQ_SRC = src/builtin.c src/bytecode.c src/compile.c src/execute.c    \
        src/jq_test.c src/jv.c src/jv_alloc.c src/jv_aux.c              \
        src/jv_dtoa.c src/jv_file.c src/jv_parse.c src/jv_print.c       \
        src/jv_unicode.c src/linker.c src/locfile.c src/util.c          \
+        src/decNumber/decContext.c src/decNumber/decNumber.c            \
+        src/jv_dtoa_tsd.c                                               \
        ${LIBJQ_INCS}

 ### C build options
@@ -186,9 +188,13 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
        tests/modules/test_bind_order.jq                                \
        tests/modules/test_bind_order0.jq                               \
        tests/modules/test_bind_order1.jq                               \
-        tests/modules/test_bind_order2.jq tests/onig.supp               \
-        tests/onig.test tests/optional.test tests/setup                 \
-        tests/torture/input0.json tests/utf8-truncate.jq
+        tests/modules/test_bind_order2.jq                               \
+        tests/onig.supp tests/local.supp                                \
+        tests/onig.test tests/setup tests/torture/input0.json		\
+        tests/optional.test tests/optionaltest				\
+        tests/utf8-truncate.jq tests/utf8test				\
+        tests/base64.test tests/base64test 				\
+        tests/jq-f-test.sh tests/shtest

 # README.md is expected in Github projects, good stuff in it, so we'll
 # distribute it and install it with the package in the doc directory.
--- a/configure.ac
+++ b/configure.ac
@@ -136,17 +136,9 @@ AC_CHECK_MEMBER([struct tm.tm_gmtoff], [AC_DEFINE([HAVE_TM_TM_GMT_OFF],1,[Define
 AC_CHECK_MEMBER([struct tm.__tm_gmtoff], [AC_DEFINE([HAVE_TM___TM_GMT_OFF],1,[Define to 1 if the system has the __tm_gmt_off field in struct tm])],
                [], [[#include <time.h>]])

-AC_ARG_ENABLE([pthread-tls],
-              [AC_HELP_STRING([--enable-pthread-tls],
-                              [Enable use of pthread thread local storage])],
-              [],
-              [enable_pthread_tls=no])
-
-if test $enable_pthread_tls = yes; then
-    AC_FIND_FUNC([pthread_key_create], [pthread], [#include <pthread.h>], [NULL, NULL])
-    AC_FIND_FUNC([pthread_once], [pthread], [#include <pthread.h>], [NULL, NULL])
-    AC_FIND_FUNC([atexit], [pthread], [#include <stdlib.h>], [NULL])
-fi
+AC_FIND_FUNC([pthread_key_create], [pthread], [#include <pthread.h>], [NULL, NULL])
+AC_FIND_FUNC([pthread_once], [pthread], [#include <pthread.h>], [NULL, NULL])
+AC_FIND_FUNC([atexit], [pthread], [#include <stdlib.h>], [NULL])

 dnl libm math.h functions
 AC_CHECK_MATH_FUNC(acos)
--- a/docs/content/manual/v1.6/manual.yml
+++ b/docs/content/manual/v1.6/manual.yml
@@ -292,11 +292,37 @@ sections:
          program can be a useful way of formatting JSON output from,
          say, `curl`.

+          An important point about the identity filter is that it 
+          guarantees to preserve the literal decimal representation 
+          of values. This is particularly important when dealing with numbers 
+          which can't be losslessly converted to an IEEE754 double precision 
+          representation.
+
+          jq doesn't truncate the literal numbers to double unless there
+          is a need to make arithmetic operations with the number.
+          Comparisions are carried out over the untruncated big decimal 
+          representation of the number.
+
+          jq will also try to maintain the original decimal precision of the provided
+          number literal. See below for examples. 
+
        examples:
          - program: '.'
            input: '"Hello, world!"'
            output: ['"Hello, world!"']

+          - program: '. | tojson'
+            input: '12345678909876543212345'
+            output: ['"12345678909876543212345"']
+
+          - program: 'map([., . == 1]) | tojson'
+            input: '[1, 1.000, 1.0, 100e-2]'
+            output: ['"[[1,true],[1.000,true],[1.0,true],[1.00,true]]"']
+
+          - program: '. as $big | [$big, $big + 1] | map(. > 10000000000000000000000000000000)'
+            input: '10000000000000000000000000000001'
+            output: ['[true, false]']
+
      - title: "Object Identifier-Index: `.foo`, `.foo.bar`"
        body: |

@@ -512,6 +538,16 @@ sections:
      expression that takes an input, ignores it, and returns 42
      instead.

+      Numbers in jq are internally represented by their IEEE754 double 
+      precision approximation. Any arithmetic operation with numbers, 
+      whether they are literals or results of previous filters, will 
+      produce a double precision floating point result.
+
+      However, when parsing a literal jq will store the original literal
+      string. If no mutation is applied to this value then it will make
+      to the output in its original form, even if conversion to double 
+      would result in a loss. 
+
    entries:
      - title: "Array construction: `[]`"
        body: |
@@ -630,6 +666,18 @@ sections:
      try to add a string to an object you'll get an error message and
      no result.

+      Please note that all numbers are converted to IEEE754 double precision 
+      floating point representation. Arithmetic and logical operators are working
+      with these converted doubles. Results of all such operations are also limited 
+      to the double precision. 
+
+      The only exception to this behaviour of number is a snapshot of original number 
+      literal. When a number which originally was provided as a literal is never
+      mutated until the end of the program then it is printed to the output in its 
+      original literal form. This also includes cases when the original literal 
+      would be truncated when converted to the IEEE754 double precision floating point
+      number.
+
    entries:
      - title: "Addition: `+`"
        body: |
--- a/src/builtin.c
+++ b/src/builtin.c
@@ -90,8 +90,11 @@ static jv f_plus(jq_state *jq, jv input, jv a, jv b) {
    jv_free(b);
    return a;
  } else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    return jv_number(jv_number_value(a) +
+    jv r = jv_number(jv_number_value(a) +
                     jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
  } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
    return jv_string_concat(a, b);
  } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
@@ -274,7 +277,10 @@ static jv f_rtrimstr(jq_state *jq, jv input, jv right) {
 static jv f_minus(jq_state *jq, jv input, jv a, jv b) {
  jv_free(input);
  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    return jv_number(jv_number_value(a) - jv_number_value(b));
+    jv r = jv_number(jv_number_value(a) - jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
  } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
    jv out = jv_array();
    jv_array_foreach(a, i, x) {
@@ -302,7 +308,10 @@ static jv f_multiply(jq_state *jq, jv input, jv a, jv b) {
  jv_kind bk = jv_get_kind(b);
  jv_free(input);
  if (ak == JV_KIND_NUMBER && bk == JV_KIND_NUMBER) {
-    return jv_number(jv_number_value(a) * jv_number_value(b));
+    jv r = jv_number(jv_number_value(a) * jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
  } else if ((ak == JV_KIND_STRING && bk == JV_KIND_NUMBER) ||
             (ak == JV_KIND_NUMBER && bk == JV_KIND_STRING)) {
    jv str = a;
@@ -336,7 +345,10 @@ static jv f_divide(jq_state *jq, jv input, jv a, jv b) {
  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
    if (jv_number_value(b) == 0.0)
      return type_error2(a, b, "cannot be divided because the divisor is zero");
-    return jv_number(jv_number_value(a) / jv_number_value(b));
+    jv r = jv_number(jv_number_value(a) / jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
  } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
    return jv_string_split(a, b);
  } else {
@@ -349,7 +361,10 @@ static jv f_mod(jq_state *jq, jv input, jv a, jv b) {
  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
    if ((intmax_t)jv_number_value(b) == 0)
      return type_error2(a, b, "cannot be divided (remainder) because the divisor is zero");
-    return jv_number((intmax_t)jv_number_value(a) % (intmax_t)jv_number_value(b));
+    jv r = jv_number((intmax_t)jv_number_value(a) % (intmax_t)jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
  } else {
    return type_error2(a, b, "cannot be divided (remainder)");
  }
@@ -440,7 +455,9 @@ static jv f_length(jq_state *jq, jv input) {
  } else if (jv_get_kind(input) == JV_KIND_STRING) {
    return jv_number(jv_string_length_codepoints(input));
  } else if (jv_get_kind(input) == JV_KIND_NUMBER) {
-    return jv_number(fabs(jv_number_value(input)));
+    jv r = jv_number(fabs(jv_number_value(input)));
+    jv_free(input);
+    return r;
  } else if (jv_get_kind(input) == JV_KIND_NULL) {
    jv_free(input);
    return jv_number(0);
--- a/src/execute.c
+++ b/src/execute.c
@@ -509,21 +509,25 @@ jv jq_next(jq_state *jq) {
      uint16_t v = *pc++;
      jv* var = frame_local_var(jq, v, level);
      jv max = stack_pop(jq);
-      if (raising) goto do_backtrack;
+      if (raising) {
+        jv_free(max);
+        goto do_backtrack;
+      } 
      if (jv_get_kind(*var) != JV_KIND_NUMBER ||
          jv_get_kind(max) != JV_KIND_NUMBER) {
        set_error(jq, jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric")));
        jv_free(max);
        goto do_backtrack;
-      } else if (jv_number_value(jv_copy(*var)) >= jv_number_value(jv_copy(max))) {
+      } else if (jv_number_value(*var) >= jv_number_value(max)) {
        /* finished iterating */
+        jv_free(max);
        goto do_backtrack;
      } else {
-        jv curr = jv_copy(*var);
+        jv curr = *var;
        *var = jv_number(jv_number_value(*var) + 1);

        struct stack_pos spos = stack_get_pos(jq);
-        stack_push(jq, jv_copy(max));
+        stack_push(jq, max);
        stack_save(jq, pc - 3, spos);

        stack_push(jq, curr);
@@ -1010,6 +1014,9 @@ jq_state *jq_init(void) {
  jq->attrs = jv_object();
  jq->path = jv_null();
  jq->value_at_path = jv_null();
+
+  jq->nomem_handler = NULL;
+  jq->nomem_handler_data = NULL;
  return jq;
 }

--- a/src/jq_test.c
+++ b/src/jq_test.c
@@ -6,20 +6,32 @@
 #include "jq.h"

 static void jv_test();
-static void run_jq_tests(jv, int, FILE *);
+static void run_jq_tests(jv, int, FILE *, int, int);


 int jq_testsuite(jv libdirs, int verbose, int argc, char* argv[]) {
  FILE *testdata = stdin;
+  int skip = -1;
+  int take = -1;
  jv_test();
  if (argc > 0) {
-    testdata = fopen(argv[0], "r");
+    for(int i = 0; i < argc; i++) {
+      if (!strcmp(argv[i], "--skip")) {
+        skip = atoi(argv[i+1]);
+        i++;
+      } else if (!strcmp(argv[i], "--take")) {
+        take = atoi(argv[i+1]);
+        i++;
+      } else {
+        testdata = fopen(argv[i], "r");
        if (!testdata) {
          perror("fopen");
          exit(1);
        }
      }
-  run_jq_tests(libdirs, verbose, testdata);
+    }
+  }
+  run_jq_tests(libdirs, verbose, testdata, skip, take);
  return 0;
 }

@@ -53,7 +65,7 @@ static void test_err_cb(void *data, jv e) {
  jv_free(e);
 }

-static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
+static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata, int skip, int take) {
  char prog[4096];
  char buf[4096];
  struct err_data err_msg;
@@ -63,6 +75,9 @@ static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
  int check_msg = 0;
  jq_state *jq = NULL;

+  int tests_to_skip = skip;
+  int tests_to_take = take;
+
  jq = jq_init();
  assert(jq);
  if (jv_get_kind(lib_dirs) == JV_KIND_NULL)
@@ -80,6 +95,34 @@ static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
      continue;
    }
    if (prog[strlen(prog)-1] == '\n') prog[strlen(prog)-1] = 0;
+
+    if (skip > 0) {
+      skip--;
+
+      // skip past test data
+      while (fgets(buf, sizeof(buf), testdata)) {
+        lineno++;
+        if (buf[0] == '\n' || (buf[0] == '\r' && buf[1] == '\n'))
+          break;
+      }
+      
+      must_fail = 0;
+      check_msg = 0;
+
+      continue;
+    } else if (skip == 0) {
+      printf("Skipped %d tests\n", tests_to_skip);
+      skip = -1;
+    }
+
+    if (take > 0) {
+      take--;
+    } else if (take == 0) {
+      printf("Hit the number of tests limit (%d), breaking\n", tests_to_take);
+      take = -1;
+      break;
+    }
+
    printf("Testing '%s' at line number %u\n", prog, lineno);
    int pass = 1;
    tests++;
@@ -179,7 +222,21 @@ static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
    passed+=pass;
  }
  jq_teardown(&jq);
-  printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
+
+  int total_skipped = tests_to_skip > 0 ? tests_to_skip : 0;
+
+  if (skip > 0) {
+    total_skipped = tests_to_skip - skip;
+  }
+
+  printf("%d of %d tests passed (%d malformed, %d skipped)\n", 
+    passed, tests, invalid, total_skipped);
+
+  if (skip > 0) {
+    printf("WARN: skipped past the end of file, exiting with status 2\n");
+    exit(2);
+  }
+
  if (passed != tests) exit(1);
 }

--- a/src/jv.c
+++ b/src/jv.c
--- a/src/jv.h
+++ b/src/jv.h
@@ -54,16 +54,19 @@ jv jv_invalid_with_msg(jv);
 jv jv_invalid_get_msg(jv);
 int jv_invalid_has_msg(jv);

-
 jv jv_null(void);
 jv jv_true(void);
 jv jv_false(void);
 jv jv_bool(int);

 jv jv_number(double);
+jv jv_number_with_literal(const char*);
 double jv_number_value(jv);
 int jv_is_integer(jv);

+int jv_number_has_literal(jv n);
+const char* jv_number_get_literal(jv);
+
 jv jv_array(void);
 jv jv_array_sized(int);
 int jv_array_length(jv);
--- a/src/jv_aux.c
+++ b/src/jv_aux.c
@@ -2,6 +2,16 @@
 #include <stdlib.h>
 #include <assert.h>
 #include "jv_alloc.h"
+#include "jv_type_private.h"
+
+// making this static verbose function here
+// until we introduce a less confusing naming scheme
+// of jv_* API with regards to the memory management
+static double jv_number_get_value_and_consume(jv number) {
+  double value = jv_number_value(number);
+  jv_free(number);
+  return value;
+}

 static int parse_slice(jv j, jv slice, int* pstart, int* pend) {
  // Array slices
@@ -32,6 +42,8 @@ static int parse_slice(jv j, jv slice, int* pstart, int* pend) {
  } else {
    double dstart = jv_number_value(start_jv);
    double dend = jv_number_value(end_jv);
+    jv_free(start_jv);
+    jv_free(end_jv);
    if (dstart < 0) dstart += len;
    if (dend < 0) dend += len;
    if (dstart < 0) dstart = 0;
@@ -69,6 +81,7 @@ jv jv_get(jv t, jv k) {
        jv_free(v);
        v = jv_null();
      }
+      jv_free(k);
    } else {
      jv_free(t);
      jv_free(k);
@@ -135,6 +148,7 @@ jv jv_set(jv t, jv k, jv v) {
             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
    if (isnull) t = jv_array();
    t = jv_array_set(t, (int)jv_number_value(k), v);
+    jv_free(k);
  } else if (jv_get_kind(k) == JV_KIND_OBJECT &&
             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
    if (isnull) t = jv_array();
@@ -202,6 +216,7 @@ jv jv_has(jv t, jv k) {
             jv_get_kind(k) == JV_KIND_NUMBER) {
    jv elem = jv_array_get(t, (int)jv_number_value(k));
    ret = jv_bool(jv_is_valid(elem));
+    jv_free(k);
    jv_free(elem);
  } else {
    ret = jv_invalid_with_msg(jv_string_fmt("Cannot check whether %s has a %s key",
@@ -240,6 +255,7 @@ static jv jv_dels(jv t, jv keys) {
          ends = jv_array_append(ends, jv_number(end));
        } else {
          jv_free(new_array);
+          jv_free(key);
          new_array = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
          goto arr_out;
        }
@@ -258,7 +274,7 @@ static jv jv_dels(jv t, jv keys) {
    jv_array_foreach(t, i, elem) {
      int del = 0;
      while (neg_idx < jv_array_length(jv_copy(neg_keys))) {
-        int delidx = len + (int)jv_number_value(jv_array_get(jv_copy(neg_keys), neg_idx));
+        int delidx = len + (int)jv_number_get_value_and_consume(jv_array_get(jv_copy(neg_keys), neg_idx));
        if (i == delidx) {
          del = 1;
        }
@@ -268,7 +284,7 @@ static jv jv_dels(jv t, jv keys) {
        neg_idx++;
      }
      while (nonneg_idx < jv_array_length(jv_copy(nonneg_keys))) {
-        int delidx = (int)jv_number_value(jv_array_get(jv_copy(nonneg_keys), nonneg_idx));
+        int delidx = (int)jv_number_get_value_and_consume(jv_array_get(jv_copy(nonneg_keys), nonneg_idx));
        if (i == delidx) {
          del = 1;
        }
@@ -278,8 +294,8 @@ static jv jv_dels(jv t, jv keys) {
        nonneg_idx++;
      }
      for (int sidx=0; !del && sidx<jv_array_length(jv_copy(starts)); sidx++) {
-        if ((int)jv_number_value(jv_array_get(jv_copy(starts), sidx)) <= i &&
-            i < (int)jv_number_value(jv_array_get(jv_copy(ends), sidx))) {
+        if ((int)jv_number_get_value_and_consume(jv_array_get(jv_copy(starts), sidx)) <= i &&
+            i < (int)jv_number_get_value_and_consume(jv_array_get(jv_copy(ends), sidx))) {
          del = 1;
        }
      }
@@ -511,14 +527,13 @@ int jv_cmp(jv a, jv b) {
    break;

  case JV_KIND_NUMBER: {
-    double da = jv_number_value(a), db = jv_number_value(b);
-
-    // handle NaN as though it were null
-    if (da != da) r = jv_cmp(jv_null(), jv_number(db));
-    else if (db != db) r = jv_cmp(jv_number(da), jv_null());
-    else if (da < db) r = -1;
-    else if (da == db) r = 0;
-    else r = 1;
+    if (jvp_number_is_nan(a)) {
+      r = jv_cmp(jv_null(), jv_copy(b));
+    } else if (jvp_number_is_nan(b)) {
+      r = jv_cmp(jv_copy(a), jv_null());
+    } else {
+      r = jvp_number_cmp(a, b);
+    }
    break;
  }

--- a/src/jv_dtoa_tsd.c
+++ b/src/jv_dtoa_tsd.c
@@ -0,0 +1,46 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <pthread.h>
+
+#include "jv_dtoa_tsd.h"
+#include "jv_dtoa.h"
+#include "jv_alloc.h"
+
+
+static pthread_key_t dtoa_ctx_key;
+static pthread_once_t dtoa_ctx_once = PTHREAD_ONCE_INIT;
+
+static void tsd_dtoa_ctx_dtor(struct dtoa_context *ctx) {
+  if (ctx) {
+    jvp_dtoa_context_free(ctx);
+    jv_mem_free(ctx);
+  }
+}
+
+static void tsd_dtoa_ctx_fini() {
+  struct dtoa_context *ctx = pthread_getspecific(dtoa_ctx_key);
+  tsd_dtoa_ctx_dtor(ctx);
+  pthread_setspecific(dtoa_ctx_key, NULL);
+}
+
+static void tsd_dtoa_ctx_init() {
+  if (pthread_key_create(&dtoa_ctx_key, tsd_dtoa_ctx_dtor) != 0) {
+    fprintf(stderr, "error: cannot create thread specific key");
+    abort();
+  }
+  atexit(tsd_dtoa_ctx_fini);
+}
+
+inline struct dtoa_context *tsd_dtoa_context_get() {
+  pthread_once(&dtoa_ctx_once, tsd_dtoa_ctx_init); // cannot fail
+  struct dtoa_context *ctx = (struct dtoa_context*)pthread_getspecific(dtoa_ctx_key);
+  if (!ctx) {
+    ctx = malloc(sizeof(struct dtoa_context));
+    jvp_dtoa_context_init(ctx);
+    if (pthread_setspecific(dtoa_ctx_key, ctx) != 0) {
+      fprintf(stderr, "error: cannot set thread specific data");
+      abort();
+    }
+  }
+  return ctx;
+}
--- a/src/jv_dtoa_tsd.h
+++ b/src/jv_dtoa_tsd.h
@@ -0,0 +1,4 @@
+#ifndef JV_DTOA_TSD_H
+#define JV_DTOA_TSD_H
+struct dtoa_context *tsd_dtoa_context_get();
+#endif
--- a/src/jv_parse.c
+++ b/src/jv_parse.c
@@ -124,14 +124,19 @@ static void parser_free(struct jv_parser* p) {

 static pfunc value(struct jv_parser* p, jv val) {
  if ((p->flags & JV_PARSE_STREAMING)) {
-    if (jv_is_valid(p->next) || p->last_seen == JV_LAST_VALUE)
+    if (jv_is_valid(p->next) || p->last_seen == JV_LAST_VALUE) {
+      jv_free(val);
      return "Expected separator between values";
+    }
    if (p->stacklen > 0)
      p->last_seen = JV_LAST_VALUE;
    else
      p->last_seen = JV_LAST_NONE;
  } else {
-    if (jv_is_valid(p->next)) return "Expected separator between values";
+    if (jv_is_valid(p->next)) {
+      jv_free(val);
+      return "Expected separator between values";
+    }
  }
  jv_free(p->next);
  p->next = val;
@@ -256,8 +261,12 @@ static pfunc stream_token(struct jv_parser* p, char ch) {
    break;

  case ':':
-    if (p->stacklen == 0 || jv_get_kind(jv_array_get(jv_copy(p->path), p->stacklen - 1)) == JV_KIND_NUMBER)
+    last = jv_invalid();
+    if (p->stacklen == 0 || jv_get_kind(last = jv_array_get(jv_copy(p->path), p->stacklen - 1)) == JV_KIND_NUMBER) {
+      jv_free(last);
      return "':' not as part of an object";
+    }
+    jv_free(last);
    if (!jv_is_valid(p->next) || p->last_seen == JV_LAST_NONE)
      return "Expected string key before ':'";
    if (jv_get_kind(p->next) != JV_KIND_STRING)
@@ -492,11 +501,11 @@ static pfunc check_literal(struct jv_parser* p) {
  } else {
    // FIXME: better parser
    p->tokenbuf[p->tokenpos] = 0;
-    char* end = 0;
-    double d = jvp_strtod(&p->dtoa, p->tokenbuf, &end);
-    if (end == 0 || *end != 0)
+    jv number = jv_number_with_literal(p->tokenbuf);
+    if (jv_get_kind(number) == JV_KIND_INVALID) {
      return "Invalid numeric literal";
-    TRY(value(p, jv_number(d)));
+    }
+    TRY(value(p, number));
  }
  p->tokenpos = 0;
  return 0;
--- a/src/jv_print.c
+++ b/src/jv_print.c
@@ -11,8 +11,10 @@

 #include "jv.h"
 #include "jv_dtoa.h"
+#include "jv_dtoa_tsd.h"
 #include "jv_unicode.h"
 #include "jv_alloc.h"
+#include "jv_type_private.h"

 #ifndef MAX_PRINT_DEPTH
 #define MAX_PRINT_DEPTH (256)
@@ -229,6 +231,13 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
    put_str("true", F, S, flags & JV_PRINT_ISATTY);
    break;
  case JV_KIND_NUMBER: {
+    if (jvp_number_is_nan(x)) {
+      jv_dump_term(C, jv_null(), flags, indent, F, S);
+    } else {
+      const char * literal_data = jv_number_get_literal(x);
+      if (literal_data) {
+        put_str(literal_data, F, S, flags & JV_PRINT_ISATTY);
+      } else {
        double d = jv_number_value(x);
        if (d != d) {
          // JSON doesn't have NaN, so we'll render it as "null"
@@ -239,6 +248,8 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
          if (d < -DBL_MAX) d = -DBL_MAX;
          put_str(jvp_dtoa_fmt(C, buf, d), F, S, flags & JV_PRINT_ISATTY);
        }
+      }
+    }
    break;
  }
  case JV_KIND_STRING:
@@ -357,10 +368,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
 }

 void jv_dumpf(jv x, FILE *f, int flags) {
-  struct dtoa_context C;
-  jvp_dtoa_context_init(&C);
-  jv_dump_term(&C, x, flags, 0, f, 0);
-  jvp_dtoa_context_free(&C);
+  jv_dump_term(tsd_dtoa_context_get(), x, flags, 0, f, 0);
 }

 void jv_dump(jv x, int flags) {
@@ -376,11 +384,8 @@ void jv_show(jv x, int flags) {
 }

 jv jv_dump_string(jv x, int flags) {
-  struct dtoa_context C;
-  jvp_dtoa_context_init(&C);
  jv s = jv_string("");
-  jv_dump_term(&C, x, flags, 0, 0, &s);
-  jvp_dtoa_context_free(&C);
+  jv_dump_term(tsd_dtoa_context_get(), x, flags, 0, 0, &s);
  return s;
 }

--- a/src/jv_type_private.h
+++ b/src/jv_type_private.h
@@ -0,0 +1,7 @@
+#ifndef JV_TYPE_PRIVATE
+#define JV_TYPE_PRIVATE
+
+int jvp_number_cmp(jv, jv);
+int jvp_number_is_nan(jv);
+
+#endif //JV_TYPE_PRIVATE
--- a/src/parser.c
+++ b/src/parser.c
--- a/src/parser.h
+++ b/src/parser.h
@@ -1,8 +1,9 @@
-/* A Bison parser, made by GNU Bison 3.0.4.  */
+/* A Bison parser, made by GNU Bison 3.3.2.  */

 /* Bison interface for Yacc-like parsers in C

-   Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
+   Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2019 Free Software Foundation,
+   Inc.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -30,6 +31,9 @@
   This special exception was added by the Free Software Foundation in
   version 2.2 of Bison.  */

+/* Undocumented macros, especially those whose name start with YY_,
+   are private implementation details.  Do not rely on them.  */
+
 #ifndef YY_YY_SRC_PARSER_H_INCLUDED
 # define YY_YY_SRC_PARSER_H_INCLUDED
 /* Debug traces.  */
@@ -40,7 +44,7 @@
 extern int yydebug;
 #endif
 /* "%code requires" blocks.  */
-#line 11 "src/parser.y" /* yacc.c:1909  */
+#line 11 "src/parser.y" /* yacc.c:1927  */

 #include "locfile.h"
 struct lexer_param;
@@ -57,7 +61,7 @@ struct lexer_param;
    }                                           \
  } while (0)

-#line 61 "src/parser.h" /* yacc.c:1909  */
+#line 65 "src/parser.h" /* yacc.c:1927  */

 /* Token type.  */
 #ifndef YYTOKENTYPE
@@ -163,12 +167,12 @@ struct lexer_param;

 union YYSTYPE
 {
-#line 31 "src/parser.y" /* yacc.c:1909  */
+#line 31 "src/parser.y" /* yacc.c:1927  */

  jv literal;
  block blk;

-#line 172 "src/parser.h" /* yacc.c:1909  */
+#line 176 "src/parser.h" /* yacc.c:1927  */
 };

 typedef union YYSTYPE YYSTYPE;
--- a/src/parser.y
+++ b/src/parser.y
@@ -172,7 +172,7 @@ static jv check_object_key(block k) {
    char errbuf[15];
    return jv_string_fmt("Cannot use %s (%s) as object key",
        jv_kind_name(block_const_kind(k)),
-        jv_dump_string_trunc(jv_copy(block_const(k)), errbuf, sizeof(errbuf)));
+        jv_dump_string_trunc(block_const(k), errbuf, sizeof(errbuf)));
  }
  return jv_invalid();
 }
@@ -216,19 +216,25 @@ static block constant_fold(block a, block b, int op) {
  jv res = jv_invalid();

  if (block_const_kind(a) == JV_KIND_NUMBER) {
-    double na = jv_number_value(block_const(a));
-    double nb = jv_number_value(block_const(b));
+    jv jv_a = block_const(a);
+    jv jv_b = block_const(b);
+
+    double na = jv_number_value(jv_a);
+    double nb = jv_number_value(jv_b);
+
+    int cmp = jv_cmp(jv_a, jv_b);
+
    switch (op) {
    case '+': res = jv_number(na + nb); break;
    case '-': res = jv_number(na - nb); break;
    case '*': res = jv_number(na * nb); break;
    case '/': res = jv_number(na / nb); break;
-    case EQ:  res = (na == nb ? jv_true() : jv_false()); break;
-    case NEQ: res = (na != nb ? jv_true() : jv_false()); break;
-    case '<': res = (na < nb ? jv_true() : jv_false()); break;
-    case '>': res = (na > nb ? jv_true() : jv_false()); break;
-    case LESSEQ: res = (na <= nb ? jv_true() : jv_false()); break;
-    case GREATEREQ: res = (na >= nb ? jv_true() : jv_false()); break;
+    case EQ:  res = (cmp == 0 ? jv_true() : jv_false()); break;
+    case NEQ: res = (cmp != 0 ? jv_true() : jv_false()); break;
+    case '<': res = (cmp < 0 ? jv_true() : jv_false()); break;
+    case '>': res = (cmp > 0 ? jv_true() : jv_false()); break;
+    case LESSEQ: res = (cmp <= 0 ? jv_true() : jv_false()); break;
+    case GREATEREQ: res = (cmp >= 0 ? jv_true() : jv_false()); break;
    default: break;
    }
  } else if (op == '+' && block_const_kind(a) == JV_KIND_STRING) {
--- a/tests/jq.test
+++ b/tests/jq.test
@@ -1673,3 +1673,42 @@ false
 #null


+#
+# Tests to cover the new toliteral number functionality
+# For an example see #1652 and other linked issues
+#
+
+# We are backward and sanity compatible
+
+map(. == 1)
+[1, 1.0, 1.000, 100e-2, 1e+0, 0.0001e4]
+[true, true, true, true, true, true]
+
+# When no arithmetic is involved jq should preserve the literal value
+
+.[0] | tostring
+[13911860366432393]
+"13911860366432393"
+
+.x | tojson
+{"x":13911860366432393}
+"13911860366432393"
+
+13911860366432393 == 13911860366432392
+null
+false
+
+
+# Applying arithmetic to the value will truncate the result to double
+
+. - 10
+13911860366432393
+13911860366432382
+
+.[0] - 10
+[13911860366432393]
+13911860366432382
+
+.x - 10
+{"x":13911860366432393}
+13911860366432382
--- a/tests/local.supp
+++ b/tests/local.supp
@@ -0,0 +1,14 @@
+{
+   macos valgrind 1
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:calloc
+   fun:map_images_nolock
+   ...
+   fun:_dyld_objc_notify_register
+   fun:_objc_init
+   fun:_os_object_init
+   fun:libdispatch_init
+   fun:libSystem_initializer
+   ...
+}
--- a/tests/setup
+++ b/tests/setup
@@ -14,7 +14,8 @@ JQ=$JQBASEDIR/jq

 if [ -z "${NO_VALGRIND-}" ] && which valgrind > /dev/null; then
    VALGRIND="valgrind --error-exitcode=1 --leak-check=full \
-                       --suppressions=$JQTESTDIR/onig.supp"
+                       --suppressions=$JQTESTDIR/onig.supp \
+                       --suppressions=$JQTESTDIR/local.supp"
    VG_EXIT0=--error-exitcode=0
    Q=-q
 else