fix Decimal format parsing

python · mdickinson · Apr 11, 2022 · Dec 7, 2021 · Dec 11, 2021 · Dec 13, 2021
commit 0cbff6ac9853cbbe57b2dd7057dde30add4a1de4
diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
@@ -1118,6 +1118,7 @@ def test_formatting(self):
             ('z>6.1f', '-0.', 'zz-0.0'),
             ('z>z6.1f', '-0.', 'zzz0.0'),
             ('x>z6.1f', '-0.', 'xxx0.0'),
+            ('🖤>z6.1f', '-0.', '🖤🖤🖤0.0'),  # multi-byte fill char
 
             # issue 6850
             ('a=-7.0', '0.12345', 'aaaa0.1'),
@@ -1139,6 +1140,9 @@ def test_negative_zero_format_directed_rounding(self):
             self.assertEqual(format(self.decimal.Decimal('-0.001'), 'z.2f'),
                             '0.00')
 
+    def test_negative_zero_bad_format(self):
+        self.assertRaises(ValueError, format, self.decimal.Decimal('1.23'), 'fz')
+
     def test_n_format(self):
         Decimal = self.decimal.Decimal
 

diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py
@@ -601,11 +601,14 @@ def test_negative_zero(self):
         self.assertEqual(f"{-0.:z>6.1f}", "zz-0.0")  # test fill, esp. 'z' fill
         self.assertEqual(f"{-0.:z>z6.1f}", "zzz0.0")
         self.assertEqual(f"{-0.:x>z6.1f}", "xxx0.0")
+        self.assertEqual(f"{-0.:🖤>z6.1f}", "🖤🖤🖤0.0")  # multi-byte fill char
 
     def test_specifier_z_error(self):
         error_msg = re.compile("Invalid format specifier '.*z.*'")
         with self.assertRaisesRegex(ValueError, error_msg):
             f"{0:z+f}"  # wrong position
+        with self.assertRaisesRegex(ValueError, error_msg):
+            f"{0:fz}"  # wrong position
 
         error_msg = re.escape("Negative zero coercion (z) not allowed")
         with self.assertRaisesRegex(ValueError, error_msg):

diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c
@@ -3184,8 +3184,9 @@ dotsep_as_utf8(const char *s)
 }
 
 /* copy of libmpdec _mpd_round() */
-static void _mpd_round(mpd_t *result, const mpd_t *a, mpd_ssize_t prec,
-                       const mpd_context_t *ctx, uint32_t *status)
+static void
+_mpd_round(mpd_t *result, const mpd_t *a, mpd_ssize_t prec,
+           const mpd_context_t *ctx, uint32_t *status)
 {
     mpd_ssize_t exp = a->exp + a->digits - prec;
 
@@ -3204,6 +3205,34 @@ static void _mpd_round(mpd_t *result, const mpd_t *a, mpd_ssize_t prec,
     }
 }
 
+/* Locate negative zero "z" option within a UTF-8 format spec string.
+ * Returns pointer to "z", else NULL.
+ * The portion of the spec we're working with is [[fill]align][sign][z] */
+static const char *
+format_spec_z_search(char const *fmt, Py_ssize_t size) {
+    char const *pos = fmt;
+    char const *fmt_end = fmt + size;
+    /* skip over [[fill]align] (fill may be multi-byte character) */
+    pos += 1;
+    while (pos < fmt_end && *pos & 0x80) {
+        pos += 1;
+    }
+    if (pos < fmt_end && strchr("<>=^", *pos) != NULL) {
+        pos += 1;
+    } else {
+        /* fill not present-- skip over [align] */
+        pos = fmt;
+        if (pos < fmt_end && strchr("<>=^", *pos) != NULL) {
+            pos += 1;
+        }
+    }
+    /* skip over [sign] */
+    if (pos < fmt_end && strchr("+- ", *pos) != NULL) {
+        pos += 1;
+    }
+    return pos < fmt_end && *pos == 'z' ? pos : NULL;
+}
+
 static int
 dict_get_item_string(PyObject *dict, const char *key, PyObject **valueobj, const char **valuestr)
 {
@@ -3259,10 +3288,13 @@ dec_format(PyObject *dec, PyObject *args)
     }
 
     if (PyUnicode_Check(fmtarg)) {
-        fmt = (char *)PyUnicode_AsUTF8AndSize(fmtarg, &size);
+        fmt = PyUnicode_AsUTF8AndSize(fmtarg, &size);
         if (fmt == NULL) {
             return NULL;
         }
+        /* NOTE: If https://github.com/python/cpython/pull/29438 lands, the
+         *   format string manipulation below can be eliminated by enhancing
+         *   the forked mpd_parse_fmt_str(). */
         if (size > 0 && fmt[0] == '\0') {
             /* NUL fill character: must be replaced with a valid UTF-8 char
                before calling mpd_parse_fmt_str(). */
@@ -3274,14 +3306,11 @@ dec_format(PyObject *dec, PyObject *args)
             fmt_copy[0] = '_';
         }
         /* Strip 'z' option, which isn't understood by mpd_parse_fmt_str().
-         * First, skip [[fill]align], since 'fill' itself may be 'z'.
          * NOTE: fmt is always null terminated by PyUnicode_AsUTF8AndSize() */
-        char const *fmt_offset = size >= 2 && strchr("<>=^", fmt[1]) != NULL ?
-                                 fmt + 2 : fmt;
-        char *z_start = strchr(fmt_offset, 'z');
-        if (z_start != NULL) {
+        char const *z_position = format_spec_z_search(fmt, size);
+        if (z_position != NULL) {
             no_neg_0 = 1;
-            size_t z_index = z_start - fmt;
+            size_t z_index = z_position - fmt;
             if (fmt_copy == NULL) {
                 fmt = fmt_copy = dec_strdup(fmt, size);
                 if (fmt_copy == NULL) {