py/formatfloat: Fix rounding of %f format with edge-case FP values.

dpgeorge · dpgeorge · commit bc12eca461a3 · 2018-03-01T15:51:03.000+11:00
Prior to this patch the %f formatting of some FP values could be off by up
to 1, eg '%.0f' % 123 would return "122" (unix x64).  Depending on the FP
precision (single vs double) certain numbers would format correctly, but
others wolud not.  This patch should fix all cases of rounding for %f.
diff --git a/py/formatfloat.c b/py/formatfloat.c
@@ -341,7 +341,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch
     // Round
     // If we print non-exponential format (i.e. 'f'), but a digit we're going
     // to round by (e) is too far away, then there's nothing to round.
-    if ((org_fmt != 'f' || e <= 1) && f >= FPCONST(5.0)) {
+    if ((org_fmt != 'f' || e <= num_digits) && f >= FPCONST(5.0)) {
         char *rs = s;
         rs--;
         while (1) {
diff --git a/tests/float/float_format.py b/tests/float/float_format.py
@@ -0,0 +1,11 @@
+# test float formatting
+
+# general rounding
+for val in (116, 1111, 1234, 5010, 11111):
+    print('%.0f' % val)
+    print('%.1f' % val)
+    print('%.3f' % val)
+
+# make sure rounding is done at the correct precision
+for prec in range(8):
+    print(('%%.%df' % prec) % 6e-5)
diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp
@@ -56,7 +56,7 @@ Warning: test
 +1e+00
 +1e+00
 # binary
-122
+123
 456
 # VM
 2 1

Original file line number	Diff line number	Diff line change
`@@ -56,7 +56,7 @@ Warning: test`
`56`	`56`	`+1e+00`
`57`	`57`	`+1e+00`
`58`	`58`	`# binary`
`59`		`-122`
	`59`	`+123`
`60`	`60`	`456`
`61`	`61`	`# VM`
`62`	`62`	`2 1`