diff --git a/pcapkit/corekit/fields/strings.py b/pcapkit/corekit/fields/strings.py index b1efac3fac..f118f6a5b2 100644 --- a/pcapkit/corekit/fields/strings.py +++ b/pcapkit/corekit/fields/strings.py @@ -110,8 +110,9 @@ class StringField(_TextField[str]): The default is ``'strict'``. Should decoding fail under it, the field retries with ``'replace'`` rather than raising, so an octet the charset cannot decode reads as U+FFFD -- the octets themselves are not lost, - see below. A codec that supports no error handler at all still raises: - ``idna`` reports ``Unsupported error handling: replace``, measured. + see below. A codec that raises under ``'replace'`` as well, such as + ``idna``, which supports ``'strict'`` alone, decodes as UTF-8 with + ``'replace'`` instead, and its octets are not lost either. Other possible values are ``'ignore'`` and ``'replace'`` as well as any other name registered with :func:`codecs.register_error` that can handle :exc:`UnicodeDecodeError`. @@ -206,7 +207,15 @@ def post_process(self, value: 'bytes', packet: 'dict[str, Any]') -> 'str': # py try: ret = value.decode(charset, self._errors) except UnicodeError: - ret = value.decode(charset, 'replace') + # NOTE: A codec may raise under ``'replace'`` too -- ``idna`` + # supports ``'strict'`` alone, and ``punycode`` before Python 3.13 + # decodes some non-ASCII octets strictly whatever the handler -- so + # the last resort is the fallback encoding, UTF-8, which cannot. + # The check below keeps the octets, as for any lossy decoding. + try: + ret = value.decode(charset, 'replace') + except UnicodeError: + ret = value.decode('utf-8', 'replace') # NOTE: Encoding may itself fail -- ``errors='strict'`` over text that # decoded under a lone-surrogate scheme, for one -- which is as much a diff --git a/tests/corekit/test_stringfield_lossless_unit.py b/tests/corekit/test_stringfield_lossless_unit.py index accf1b89cd..188ea64300 100644 --- a/tests/corekit/test_stringfield_lossless_unit.py +++ b/tests/corekit/test_stringfield_lossless_unit.py @@ -35,6 +35,18 @@ #: fallback through one would raise where ``'replace'`` does not. NO_SURROGATE_CHARSETS = ('utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-7') +#: ``(octets, field keyword arguments, decoded text, whether it is a DecodedString)`` +#: for ``idna``, which raises under ``'replace'`` as well, on every Python (#1706): +#: it supports ``'strict'`` alone, so another scheme, or octets it cannot decode, +#: reach the UTF-8 fallback. The last row is valid ``idna`` under ``'strict'``. +REPLACE_REJECTED = ( + (b'a', {'errors': 'ignore'}, 'a', True), + (b'a', {'errors': 'replace'}, 'a', True), + (b'xn--caf-dma', {'errors': 'ignore'}, 'xn--caf-dma', True), + (b'caf\xe9', {}, 'caf�', True), + (b'xn--caf-dma', {}, 'café', False), +) + def pcapng(comment: bytes) -> bytes: """A minimal pcapng: SHB, IDB and one EPB, the first two carrying ``comment``. @@ -111,6 +123,23 @@ def test_charsets_without_surrogateescape_do_not_raise(self) -> None: _, packed = self.roundtrip(b'\xff\xfe\x41', encoding=charset) self.assertEqual(packed, b'\xff\xfe\x41') + def test_a_codec_that_rejects_replace_does_not_raise(self) -> None: + from pcapkit.corekit.fields.strings import DecodedString + + for octets, kwargs, text, decoded in REPLACE_REJECTED: + with self.subTest(octets=octets, kwargs=kwargs): + value, packed = self.roundtrip(octets, encoding='idna', **kwargs) # type: ignore[arg-type] + self.assertEqual(packed, octets) + self.assertEqual(str(value), text) + self.assertIs(isinstance(value, DecodedString), decoded) + + # ``punycode`` raises under ``'replace'`` on non-ASCII octets only before + # Python 3.13, so its text differs by version; the octets do not. + for errors in ('strict', 'ignore', 'replace'): + with self.subTest(charset='punycode', errors=errors): + _, packed = self.roundtrip(b'caf\xe9', encoding='punycode', errors=errors) + self.assertEqual(packed, b'caf\xe9') + def test_valid_text_decodes_to_a_plain_str(self) -> None: value, packed = self.roundtrip('héllo'.encode()) self.assertIs(type(value), str)