From bc12eca461a317df842ce2e616afa97670cd0ce3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Mar 2018 15:47:17 +1100 Subject: [PATCH] py/formatfloat: Fix rounding of %f format with edge-case FP values. Prior to this patch the %f formatting of some FP values could be off by up to 1, eg '%.0f' % 123 would return "122" (unix x64). Depending on the FP precision (single vs double) certain numbers would format correctly, but others wolud not. This patch should fix all cases of rounding for %f. --- py/formatfloat.c | 2 +- tests/float/float_format.py | 11 +++++++++++ tests/unix/extra_coverage.py.exp | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/float/float_format.py diff --git a/py/formatfloat.c b/py/formatfloat.c index 4228f99ff5..22dd8aaacc 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -341,7 +341,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch // Round // If we print non-exponential format (i.e. 'f'), but a digit we're going // to round by (e) is too far away, then there's nothing to round. - if ((org_fmt != 'f' || e <= 1) && f >= FPCONST(5.0)) { + if ((org_fmt != 'f' || e <= num_digits) && f >= FPCONST(5.0)) { char *rs = s; rs--; while (1) { diff --git a/tests/float/float_format.py b/tests/float/float_format.py new file mode 100644 index 0000000000..4d5ad1d693 --- /dev/null +++ b/tests/float/float_format.py @@ -0,0 +1,11 @@ +# test float formatting + +# general rounding +for val in (116, 1111, 1234, 5010, 11111): + print('%.0f' % val) + print('%.1f' % val) + print('%.3f' % val) + +# make sure rounding is done at the correct precision +for prec in range(8): + print(('%%.%df' % prec) % 6e-5) diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index a030155ba8..54e19d14ac 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -56,7 +56,7 @@ Warning: test +1e+00 +1e+00 # binary -122 +123 456 # VM 2 1