diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 9bf801ad608c773..f8ea83476abacc7 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -477,6 +477,30 @@ APIs: | ``-`` | The converted value is left adjusted (overrides the ``0`` | | | flag if both are given). | +-------+-------------------------------------------------------------+ + | ``#`` | Escape special and non-printable characters for the ``c``, | + | | ``s``, ``S``, ``U`` and ``V`` conversions. | + | | Use a colon as a separator for the ``T`` and ``N`` | + | | conversions. | + +-------+-------------------------------------------------------------+ + | ``+`` | Escape also all non-ASCII characters (only in combination | + | | with the ``#`` flag for the ``c``, ``s``, ``S``, ``U`` and | + | | ``V`` conversions). | + +-------+-------------------------------------------------------------+ + + With the ``#`` flag, the ``c``, ``s``, ``S``, ``U`` and ``V`` + conversions escape special and non-printable characters + like :func:`repr` does, + but without adding surrounding quotes: + backslash, tab, carriage return and line feed are escaped as + ``\\``, ``\t``, ``\r`` and ``\n`` respectively, + other non-printable characters are escaped as ``\xNN`` if they are + ASCII, and as ``\uNNNN`` or ``\UNNNNNNNN`` otherwise. + With the ``+`` flag, all non-ASCII characters are escaped as well. + For the ``s`` and ``V`` conversions, + bytes which cannot be decoded from UTF-8 are escaped as ``\xNN``, + so they can always be distinguished from valid non-ASCII characters. + The precision is applied to the string before escaping, + the width -- after escaping. The length modifiers for following integer conversions (``d``, ``i``, ``o``, ``u``, ``x``, or ``X``) specify the type of the argument @@ -630,6 +654,10 @@ APIs: .. versionchanged:: 3.13 Support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats added. + .. versionchanged:: next + Support for flags ``#`` and ``+`` for the ``c``, ``s``, ``S``, + ``U`` and ``V`` conversions added. + .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 06e831ea1e34631..b96cdddb14f9307 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -814,7 +814,12 @@ C API changes New features ------------ -* TODO +* :c:func:`PyUnicode_FromFormat` now supports the escaping mode for + the ``c``, ``s``, ``S``, ``U`` and ``V`` conversions: + with the ``#`` flag, special and non-printable characters in the output + are escaped, + and with the additional ``+`` flag -- also all non-ASCII characters. + (Contributed by Serhiy Storchaka in :gh:`154668`.) Porting to Python 3.16 ---------------------- diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 0dcd8a25ad0128d..af377e6c95d3540 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -535,6 +535,114 @@ def check_format(expected, format, *args): check_format("repr= 12", b'repr=%5.2V', None, b'123') + # test the escaping mode ("#" flag) with %c + check_format('A', + b'%#c', c_int(ord('A'))) + check_format('\\n', + b'%#c', c_int(ord('\n'))) + check_format('\\\\', + b'%#c', c_int(ord('\\'))) + check_format('\\x1b', + b'%#c', c_int(0x1b)) + check_format('\\x7f', + b'%#c', c_int(0x7f)) + check_format('\\u009f', + b'%#c', c_int(0x9f)) + check_format('\xe9', + b'%#c', c_int(0xe9)) + check_format('\\u2028', + b'%#c', c_int(0x2028)) + check_format('\U0001f600', + b'%#c', c_int(0x1f600)) + check_format('\\ud800', + b'%#c', c_int(0xd800)) + check_format('\\udc9f', + b'%#c', c_int(0xdc9f)) + check_format('\\u00e9', + b'%+#c', c_int(0xe9)) + check_format('\\u20ac', + b'%+#c', c_int(0x20ac)) + check_format('\\U0001f600', + b'%+#c', c_int(0x1f600)) + + # test the escaping mode with %s + check_format('a\\tb\\r\\nc\\\\d', + b'%#s', b'a\tb\r\nc\\d') + check_format('quote\'"', + b'%#s', b'quote\'"') + check_format('caf\xe9', + b'%#s', b'caf\xc3\xa9') + check_format('caf\\u00e9', + b'%+#s', b'caf\xc3\xa9') + # valid UTF-8 is escaped as a character, invalid -- as a raw byte + check_format('\\u009f', + b'%#s', b'\xc2\x9f') + check_format('\\x9f', + b'%#s', b'\x9f') + check_format('\\x9f', + b'%+#s', b'\x9f') + check_format('a\\xffb', + b'%#s', b'a\xffb') + check_format('abc\\xc3', + b'%#s', b'abc\xc3') + # the precision is applied before escaping, the width -- after + check_format('ab\\n', + b'%#.3s', b'ab\ncd') + check_format(' ab\\n', + b'%#6.3s', b'ab\ncd') + check_format('ab\\n ', + b'%-#6.3s', b'ab\ncd') + check_format('a\\xff', + b'%#.2s', b'a\xffbc') + check_format('abc', + b'%#.4s', b'abc\xc3\xa9') + + # test the escaping mode with %U, %V and %S + check_format('a\\tb', + b'%#U', 'a\tb') + check_format('caf\xe9', + b'%#U', 'caf\xe9') + check_format('caf\\u00e9', + b'%+#U', 'caf\xe9') + check_format('a\\tb', + b'%#S', 'a\tb') + check_format('caf\xe9', + b'%#S', 'caf\xe9') + check_format('caf\\u00e9', + b'%+#S', 'caf\xe9') + check_format(' ab', + b'%#5.2S', 'ab\ncd') + # a lone surrogate in a string argument stays a surrogate + check_format('\\udc9f', + b'%#U', '\udc9f') + check_format('\\udc9f', + b'%+#U', '\udc9f') + check_format(' ab', + b'%#5.2U', 'ab\ncd') + check_format('a\\tb', + b'%#V', 'a\tb', b'ignored') + check_format('\\u009f', + b'%#V', None, b'\xc2\x9f') + check_format('\\x9f', + b'%#V', None, b'\x9f') + + # test the escaping mode with %ls and %lV + check_format('a\\tb', + b'%#ls', c_wchar_p('a\tb')) + check_format('caf\xe9', + b'%#ls', c_wchar_p('caf\xe9')) + check_format('caf\\u00e9', + b'%+#ls', c_wchar_p('caf\xe9')) + check_format('a\\tb', + b'%#lV', None, c_wchar_p('a\tb')) + + # "+" requires "#" and is only supported for %c, %s, %S, %U and %V + for format in (b'%+c', b'%+s', b'%+S', b'%+U', b'%+d', b'%+#d', + b'%+#R', b'%+#A', b'%+#T', b'%+#p'): + with self.subTest(format=format): + with self.assertRaises(SystemError): + PyUnicode_FromFormat(format, py_object('abc')) + # test integer formats (%i, %d, %u, %o, %x, %X) check_format('010', b'%03i', c_int(10)) @@ -2003,6 +2111,11 @@ def test_format(self): writer.write_char(ord('.')) self.assertEqual(writer.finish(), 'abc 123.') + def test_format_escape(self): + writer = self.create_writer(0) + self.writer_format(writer, b'%#s %+#s', b'a\tb\xff', b'caf\xc3\xa9') + self.assertEqual(writer.finish(), 'a\\tb\\xff caf\\u00e9') + def test_recover_error(self): # test recovering from PyUnicodeWriter_Format() error writer = self.create_writer(0) diff --git a/Misc/NEWS.d/next/C_API/2026-07-25-10-40-00.gh-issue-154668.fEsc4p.rst b/Misc/NEWS.d/next/C_API/2026-07-25-10-40-00.gh-issue-154668.fEsc4p.rst new file mode 100644 index 000000000000000..4f9bfd38542691f --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-07-25-10-40-00.gh-issue-154668.fEsc4p.rst @@ -0,0 +1,5 @@ +Add the escaping mode to :c:func:`PyUnicode_FromFormat`: +with the ``#`` flag, +the ``c``, ``s``, ``S``, ``U`` and ``V`` conversions now escape special +and non-printable characters, +and with the additional ``+`` flag -- also all non-ASCII characters. diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c index c7a23d5d1cbd71a..f5f51fee5f07e8c 100644 --- a/Modules/_testlimitedcapi/unicode.c +++ b/Modules/_testlimitedcapi/unicode.c @@ -1824,6 +1824,29 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); + // Strings: escaping mode ('#' and '+' flags) + CHECK_FORMAT_1("%#c", "c", 'c'); + CHECK_FORMAT_1("%#c", "\\n", '\n'); + CHECK_FORMAT_1("%+#c", "\\n", '\n'); + CHECK_FORMAT_1("%#s", "a\\tb", "a\tb"); + CHECK_FORMAT_1("%+#s", "a\\tb", "a\tb"); + CHECK_FORMAT_1("%#ls", "a\\tb", L"a\tb"); + CHECK_FORMAT_1("%#S", "None", Py_None); + CHECK_FORMAT_1("%+#S", "None", Py_None); + CHECK_FORMAT_1("%#U", "None", unicode); + CHECK_FORMAT_2("%#V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%#V", "a\\tb", NULL, "a\tb"); + CHECK_FORMAT_2("%#lV", "a\\tb", NULL, L"a\tb"); + CHECK_FORMAT_1("%#6.3s", " ab\\n", "ab\ncd"); + + // The '+' flag requires the '#' flag and is only supported for + // the 'c', 's', 'S', 'U' and 'V' conversions + CHECK_FORMAT_1("%+c", NULL, 'c'); + CHECK_FORMAT_1("%+s", NULL, "abc"); + CHECK_FORMAT_1("%+d", NULL, 1); + CHECK_FORMAT_1("%+#d", NULL, 1); + CHECK_FORMAT_1("%+#R", NULL, Py_None); + Py_XDECREF(unicode); Py_RETURN_NONE; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a4550c0f5f3363a..fc75562965f6f22 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -2521,6 +2521,141 @@ PyUnicode_AsUCS4Copy(PyObject *string) plus 1 for the terminal NUL. */ #define MAX_INTMAX_CHARS (5 + (sizeof(intmax_t)*8-1) / 3) +/* Internal flag: escape the output of the 'c', 's', 'S', 'U' and 'V' + conversions with the '#' flag ('#' itself maps to F_ALT, which has a + different meaning for the 'T' and 'N' conversions). */ +#define F_ESCAPE (1<<6) +/* Internal flag for the escaping mode: surrogates in the range + U+DC80..U+DCFF come from decoding a byte string with the + "surrogateescape" error handler and represent raw bytes which could + not be decoded; escape them as "\xNN". */ +#define F_RAWBYTES (1<<7) + +/* Return the size of the escaped form of the character: + 1 if it is output as is, or the length of its escape sequence. */ +static inline Py_ssize_t +escaped_char_size(Py_UCS4 ch, int flags) +{ + switch (ch) { + case '\\': case '\t': case '\r': case '\n': + return 2; + default: + if (ch < ' ' || ch == 0x7f) + return 4; /* \xHH */ + if (ch < 0x7f || (!(flags & F_SIGN) && Py_UNICODE_ISPRINTABLE(ch))) + return 1; /* printable, keep as is */ + if ((flags & F_RAWBYTES) && 0xdc80 <= ch && ch <= 0xdcff) + return 4; /* \xHH -- a raw byte */ + if (ch < 0x10000) + return 6; /* \uHHHH */ + return 10; /* \UHHHHHHHH */ + } +} + +/* Write the escaped form of the character. + The writer buffer must have enough space reserved. */ +static void +unicode_write_escaped_char(_PyUnicodeWriter *writer, Py_UCS4 ch, int flags) +{ + int kind = writer->kind; + void *data = writer->data; + Py_ssize_t o = writer->pos; + switch (escaped_char_size(ch, flags)) { + case 1: + PyUnicode_WRITE(kind, data, o++, ch); + break; + case 2: + PyUnicode_WRITE(kind, data, o++, '\\'); + switch (ch) { + case '\t': ch = 't'; break; + case '\r': ch = 'r'; break; + case '\n': ch = 'n'; break; + } + PyUnicode_WRITE(kind, data, o++, ch); + break; + case 4: + PyUnicode_WRITE(kind, data, o++, '\\'); + PyUnicode_WRITE(kind, data, o++, 'x'); + PyUnicode_WRITE(kind, data, o++, Py_hexdigits[(ch >> 4) & 0xf]); + PyUnicode_WRITE(kind, data, o++, Py_hexdigits[ch & 0xf]); + break; + default: + { + int ndigits = (ch < 0x10000) ? 4 : 8; + PyUnicode_WRITE(kind, data, o++, '\\'); + PyUnicode_WRITE(kind, data, o++, (ch < 0x10000) ? 'u' : 'U'); + while (--ndigits >= 0) { + PyUnicode_WRITE(kind, data, o++, + Py_hexdigits[(ch >> (4*ndigits)) & 0xf]); + } + break; + } + } + writer->pos = o; +} + +/* Escape special and non-printable characters like repr() does, but + without surrounding quotes, and write the result to the writer. + If the F_SIGN flag is set, also escape all non-ASCII characters. + Characters are escaped with "\uHHHH" and "\UHHHHHHHH", raw bytes + (see F_RAWBYTES) -- with "\xHH". + The precision applies to the original string, the width -- to the + escaped one. */ +static int +unicode_fromformat_write_escaped(_PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t width, Py_ssize_t precision, + int flags) +{ + Py_ssize_t isize = PyUnicode_GET_LENGTH(str); + const void *idata = PyUnicode_DATA(str); + int ikind = PyUnicode_KIND(str); + if (precision >= 0 && precision < isize) { + isize = precision; + } + + /* Compute the length and the maximum character of the escaped string */ + Py_ssize_t osize = 0; + Py_UCS4 maxch = 127; + for (Py_ssize_t i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + Py_ssize_t incr = escaped_char_size(ch, flags); + if (incr == 1) { + maxch = Py_MAX(ch, maxch); + } + if (osize > PY_SSIZE_T_MAX - incr) { + PyErr_SetString(PyExc_OverflowError, "string is too long"); + return -1; + } + osize += incr; + } + + if (_PyUnicodeWriter_Prepare(writer, Py_MAX(osize, width), maxch) == -1) + return -1; + + Py_ssize_t fill = Py_MAX(width - osize, 0); + if (fill && !(flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1) + return -1; + writer->pos += fill; + } + + Py_ssize_t start = writer->pos; + for (Py_ssize_t i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + unicode_write_escaped_char(writer, ch, flags); + } + assert(writer->pos - start == osize); + (void)start; + + if (fill && (flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1) + return -1; + writer->pos += fill; + } + + return 0; +} + static int unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, Py_ssize_t width, Py_ssize_t precision, int flags) @@ -2528,6 +2663,11 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, Py_ssize_t length, fill, arglen; Py_UCS4 maxchar; + if (flags & F_ESCAPE) { + return unicode_fromformat_write_escaped(writer, str, width, precision, + flags); + } + length = PyUnicode_GET_LENGTH(str); if ((precision == -1 || precision >= length) && width <= length) @@ -2590,18 +2730,21 @@ unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str, } } - if (width < 0) { + if (width < 0 && !(flags & F_ESCAPE)) { return _PyUnicode_DecodeUTF8Writer(writer, str, length, _Py_ERROR_REPLACE, "replace", pconsumed); } + /* In the escaping mode, preserve undecodable bytes as surrogates + to escape them as "\xNN". */ + const char *errors = (flags & F_ESCAPE) ? "surrogateescape" : "replace"; PyObject *unicode = PyUnicode_DecodeUTF8Stateful(str, length, - "replace", pconsumed); + errors, pconsumed); if (unicode == NULL) return -1; int res = unicode_fromformat_write_str(writer, unicode, - width, -1, flags); + width, -1, flags | F_RAWBYTES); Py_DECREF(unicode); return res; } @@ -2621,7 +2764,7 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, } } - if (width < 0) { + if (width < 0 && !(flags & F_ESCAPE)) { return PyUnicodeWriter_WriteWideChar((PyUnicodeWriter*)writer, str, length); } @@ -2661,15 +2804,15 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } /* Parse flags. Example: "%-i" => flags=F_LJUST. */ - /* Flags '+', ' ' and '#' are not particularly useful. - * They are not worth the implementation and maintenance costs. - * In addition, '#' should add "0" for "o" conversions for compatibility - * with printf, but it would confuse Python users. */ + /* Flags '#' and '+' are only supported for some conversions. + * In particular, '#' should add "0" for "o" conversions for + * compatibility with printf, but it would confuse Python users. */ while (1) { switch (*f++) { case '-': flags |= F_LJUST; continue; case '0': flags |= F_ZERO; continue; case '#': flags |= F_ALT; continue; + case '+': flags |= F_SIGN; continue; } f--; break; @@ -2763,6 +2906,22 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, if (sizemod) goto invalid_format; break; } + if (flags & F_ALT) { + /* The '#' flag enables the escaping mode for the 'c', 's', 'S', + * 'U' and 'V' conversions. The '+' flag requires the '#' flag + * and extends the escaping to all non-ASCII characters. */ + switch (*f) { + case 'c': case 's': case 'S': case 'U': case 'V': + flags |= F_ESCAPE; + break; + default: + if (flags & F_SIGN) goto invalid_format; + break; + } + } + else if (flags & F_SIGN) { + goto invalid_format; + } switch (*f) { case 'c': @@ -2773,7 +2932,14 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, "character argument not in range(0x110000)"); return NULL; } - if (_PyUnicodeWriter_WriteCharInline(writer, ordinal) < 0) + if (flags & F_ESCAPE) { + Py_ssize_t len = escaped_char_size(ordinal, flags); + Py_UCS4 maxch = (len == 1) ? (Py_UCS4)ordinal : 127; + if (_PyUnicodeWriter_Prepare(writer, len, maxch) < 0) + return NULL; + unicode_write_escaped_char(writer, ordinal, flags); + } + else if (_PyUnicodeWriter_WriteCharInline(writer, ordinal) < 0) return NULL; break; }