fix(reassembly)!: move TCP's gap cap into config as PCAPKIT_TCP_GAP_LIMIT (#2546)

docs/source/contributing/conventions/reassembly-integration.rst

@@ -22,6 +22,12 @@ The Rule
    message, accept overlaps -- the engine does the same and adds no timeout, cap or
    ordering rule the RFC does not call for.
 
+**TCP's gap cap is a safety limit, not a reassembly rule.** No RFC calls for
+:data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`, the widest gap TCP reassembly
+zero-fills to merge a segment. It bounds the memory a capture can have zero-filled, as
+:data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH` does for a field, and ``-1``
+lifts it to give the uncapped reassembly point 4 asks for. Ruled on :issue:`2546`.
+
 Compliance, Measured
 ~~~~~~~~~~~~~~~~~~~~~
 

docs/source/pcapkit/foundation/reassembly/tcp.rst

@@ -21,7 +21,6 @@ which reconstructs fragmented TCP packets back to origin.
    .. autoattribute:: __callback_fn__
       :no-value:
    .. autoattribute:: __timeout__
-   .. autoattribute:: __window__
 
 Algorithm
 =========

docs/source/pcapkit/utilities/config.rst

@@ -61,6 +61,11 @@ Limits
      - ``32``
      - ``>= 0``, or ``-1``
      - each capture remembered pins one packet in memory
+   * - :envvar:`PCAPKIT_TCP_GAP_LIMIT`
+     - ``16777216``
+     - ``>= 0``, or ``-1``
+     - a TCP gap that wide is zero-filled in memory, up to 2 GiB at ``-1``
+       (:issue:`2546`)
    * - :envvar:`PCAPKIT_FALLBACK_LIMIT`
      - ``10000``
      - ``>= 0``, or ``-1``
@@ -99,6 +104,10 @@ Limits
 
    Sets :data:`~pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT`.
 
+.. envvar:: PCAPKIT_TCP_GAP_LIMIT
+
+   Sets :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`.
+
 .. envvar:: PCAPKIT_FALLBACK_LIMIT
 
    Sets :data:`~pcapkit.utilities.config.FALLBACK_LIMIT`.
@@ -127,6 +136,7 @@ Limits
 .. autodata:: pcapkit.utilities.config.FRAME_LAYER_LIMIT
 .. autodata:: pcapkit.utilities.config.EXTENSION_HEADER_LIMIT
 .. autodata:: pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT
+.. autodata:: pcapkit.utilities.config.TCP_GAP_LIMIT
 .. autodata:: pcapkit.utilities.config.FALLBACK_LIMIT
 .. autodata:: pcapkit.utilities.config.DETECT_CACHE_SIZE
 .. autodata:: pcapkit.utilities.config.READ_CHUNK

examples/generators/legacy.py

@@ -47,8 +47,9 @@
    each direction is a buffer of its own.
    :meth:`~pcapkit.foundation.reassembly.tcp.TCP.submit` runs on that
    direction's RST; once its FIN has been seen and every octet below the FIN
-   has arrived; when a SYN, or a segment farther than ``__window__`` from the
-   buffer, opens a new one on the same four-tuple; on a timeout, if one is
+   has arrived; when a SYN, or a segment farther than
+   :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT` from the buffer, opens a
+   new one on the same four-tuple; on a timeout, if one is
    set; and, for whatever is still buffered, when the datagrams are read --
    ``extraction.reassembly`` flushes every open buffer through
    :meth:`~pcapkit.foundation.reassembly.reassembly.Reassembly.fetch`. So a

pcapkit/foundation/reassembly/data/tcp.py

@@ -236,9 +236,8 @@ class Fragment(Info):
     #: merge only ever shrinks or removes one, filling it from the arriving
     #: segment. The one exception lies wholly *outside* :attr:`raw`: when a
     #: stream whose SYN was captured is split rather than padded across a gap
-    #: wider than :attr:`TCP.__window__
-    #: <pcapkit.foundation.reassembly.tcp.TCP.__window__>`, both halves record
-    #: the range left between them. Any entry at all makes the fragment
+    #: wider than :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`, both halves
+    #: record the range left between them. Any entry at all makes the fragment
     #: incomplete.
     #:
     #: This is deliberately **not** derived from

pcapkit/foundation/reassembly/tcp.py

@@ -18,6 +18,7 @@
                                                     DatagramID, Fragment, HoleDescriptor, Packet)
 from pcapkit.foundation.reassembly.reassembly import ReassemblyBase
 from pcapkit.protocols.transport.tcp import TCP as TCP_Protocol
+from pcapkit.utilities import config
 
 if TYPE_CHECKING:
     from typing import Type
@@ -99,6 +100,11 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]):
        <pcapkit.foundation.reassembly.data.tcp.Datagram.conflict>`; 1.4.x
        kept the later octets without a record (:issue:`443`).
 
+    .. versionremoved:: 1.6.0
+
+       ``__window__``, now :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT`
+       (:issue:`2546`).
+
     """
     if TYPE_CHECKING:
         protocol: 'Type[TCP_Protocol]'
@@ -129,41 +135,33 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]):
     #: ask for one, e.g. ``2 * 120`` for 2·MSL if that is the policy wanted.
     __timeout__ = math.inf
 
-    #: int: Widest gap, in octets, that merging a segment into the payload
-    #: buffer it is read against may zero-fill: 16 MiB. The gap is measured on
-    #: whichever side the segment lands, from the buffer's end to the
-    #: segment's start, or from the segment's end to the buffer's start.
-    #:
-    #: A segment can only legitimately open a gap as wide as the data in flight,
-    #: which the receive window bounds. :rfc:`7323` allows windows up to
-    #: ``2 ** 30``, but zero-filling a gap that wide costs 1 GiB, and real
-    #: windows are far smaller -- Linux's default ``net.ipv4.tcp_rmem`` ceiling
-    #: is 6 MiB -- so 16 MiB clears them with room to spare. A segment that
-    #: would open a wider gap is not padded to; the buffer held so far is
-    #: submitted and a fresh one opened for the segment instead. What that
-    #: means depends on whether the buffer's SYN was captured:
-    #:
-    #: * **No SYN** -- the segment is taken to start a new stream on a reused
-    #:   4-tuple, and the two buffers are reported independently.
-    #: * **SYN captured** -- the SYN proves both are the same stream with the
-    #:   range in between lost, so that range is recorded in the
-    #:   :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.gap` of the
-    #:   payload buffer the segment was read against and of the new one, and
-    #:   both are reported
-    #:   :attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL`
-    #:   (in strict mode, as a single run each). The new buffer keeps the
-    #:   SYN's header and counts as having seen it.
-    #:
-    #: Override in a subclass to change the bound.
-    __window__ = 1 << 24
-
     ##########################################################################
     # Methods.
     ##########################################################################
 
     def reassembly(self, info: 'Packet') -> 'None':
         """Reassembly procedure.
 
+        A segment is merged into the payload buffer it is read against by
+        zero-filling the gap between them, measured on whichever side the
+        segment lands: from the buffer's end to the segment's start, or from
+        the segment's end to the buffer's start. A gap wider than
+        :data:`~pcapkit.utilities.config.TCP_GAP_LIMIT` is not padded to; the
+        buffer held so far is submitted and a fresh one opened for the segment
+        instead. What that means depends on whether the buffer's SYN was
+        captured:
+
+        * **No SYN** -- the segment is taken to start a new stream on a reused
+          4-tuple, and the two buffers are reported independently.
+        * **SYN captured** -- the SYN proves both are the same stream with the
+          range in between lost, so that range is recorded in the
+          :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.gap` of the
+          payload buffer the segment was read against and of the new one, and
+          both are reported
+          :attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL`
+          (in strict mode, as a single run each). The new buffer keeps the
+          SYN's header and counts as having seen it.
+
         Arguments:
             info: :term:`info <reasm.tcp.packet>` dict of packets to be reassembled
 
@@ -203,8 +201,8 @@ def reassembly(self, info: 'Packet') -> 'None':
         # it most plausibly continues -- this ACK's own payload buffer, or else
         # the most recently opened one -- per :rfc:`9293#section-3.4`. Plain
         # integer comparison reads a stream crossing 2**32 as a gap of about
-        # 4 GiB. A segment landing farther than ``__window__`` outside that
-        # buffer starts a new stream instead -- see ``__window__``.
+        # 4 GiB. A segment landing farther than ``config.TCP_GAP_LIMIT``
+        # outside that buffer splits the stream instead -- see the docstring.
         SHIFT = 0
         HDR = info.header if SYN else b''  # header of the buffer opened below, if any
         ANCHORED = SYN                     # whether that buffer's stream start is known
@@ -217,7 +215,8 @@ def reassembly(self, info: 'Packet') -> 'None':
             END = fragment.isn + fragment.len
             UNWRAPPED = self._unwrap(PSN, END)
             # the zero-filled gap merging this segment would open, either side
-            if max(UNWRAPPED - END, fragment.isn - (UNWRAPPED + info.len)) > self.__window__:
+            limit = config.TCP_GAP_LIMIT
+            if limit != -1 and max(UNWRAPPED - END, fragment.isn - (UNWRAPPED + info.len)) > limit:
                 old = self._buffer.pop(BUFID)
                 if self._anchored(old):
                     # The SYN proves this is still the same stream, with the
@@ -552,7 +551,7 @@ def _trim_gaps(gap: 'list[tuple[int, int]]', first: 'int', last: 'int') -> 'None
         """Remove a received range from a fragment's gap list, in place.
 
         Only an entry lying outside :attr:`~pcapkit.foundation.reassembly.data.tcp.Fragment.raw`
-        -- the range a split stream left out, see :attr:`TCP.__window__` -- can
+        -- the range a split stream left out, see :meth:`TCP.reassembly` -- can
         still overlap a segment once it is merged; :meth:`_merge_overlap` has
         already closed every entry inside ``raw`` that the segment covers.
 
@@ -806,7 +805,7 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID',  # type: ignore[override]
             # How completely this buffer came out, and why it stopped. Derived
             # once per buffer, so the two branches cannot disagree about it.
             # Any gap entry counts, including one lying wholly outside ``raw``:
-            # that is the range a split stream left out (see ``__window__``).
+            # that is the range a split stream left out (see ``reassembly``).
             completion = Completion.COMPLETE if not buffer.gap else (
                 Completion.TIMEOUT if timeout else Completion.PARTIAL
             )

pcapkit/utilities/config.py

@@ -51,7 +51,7 @@
 __all__ = [
     # limits
     'FRAME_LAYER_LIMIT', 'EXTENSION_HEADER_LIMIT', 'IPV6_TCP_WARNED_LIMIT',
-    'FALLBACK_LIMIT', 'DETECT_CACHE_SIZE', 'READ_CHUNK',
+    'TCP_GAP_LIMIT', 'FALLBACK_LIMIT', 'DETECT_CACHE_SIZE', 'READ_CHUNK',
     'MAX_ZERO_PAD_LENGTH', 'MAX_ZERO_PAD_SHORTFALL', 'ZERO_PAD_BUDGET_RATIO',
     # readers
     'read_limit', 'read_flag',
@@ -215,6 +215,31 @@ def read_flag(name: 'str', default: 'bool' = False) -> 'bool':
 IPV6_TCP_WARNED_LIMIT = read_limit('IPV6_TCP_WARNED_LIMIT', 32, risk=(
     'each capture remembered pins one of its packets in memory'))
 
+#: int: Widest gap, in octets, that
+#: :meth:`TCP.reassembly <pcapkit.foundation.reassembly.tcp.TCP.reassembly>`
+#: zero-fills to merge a segment into the payload buffer it is read against,
+#: :envvar:`PCAPKIT_TCP_GAP_LIMIT`; ``-1`` always merges.
+#:
+#: This is a safety limit, not a reassembly rule: no RFC calls for one
+#: (:issue:`2546`). A segment can only legitimately open a gap as wide as the
+#: data in flight, which the receive window bounds. :rfc:`7323` allows windows
+#: up to ``2 ** 30``, but zero-filling a gap that wide costs 1 GiB, and real
+#: windows are far smaller -- Linux's default ``net.ipv4.tcp_rmem`` ceiling is
+#: 6 MiB -- so the default of 16 MiB clears them with room to spare. Past it,
+#: the stream is split rather than padded, as :meth:`TCP.reassembly
+#: <pcapkit.foundation.reassembly.tcp.TCP.reassembly>` describes.
+#:
+#: At ``-1`` only the sequence space bounds the gap: a sequence number is read
+#: within ``2 ** 31`` of the data buffered (:rfc:`9293#section-3.4`), so one
+#: segment can have up to 2 GiB zero-filled.
+#:
+#: .. versionadded:: 1.6.0
+#:
+#:    Moved from ``pcapkit.foundation.reassembly.tcp.TCP.__window__`` (:issue:`2546`).
+TCP_GAP_LIMIT = read_limit('TCP_GAP_LIMIT', 1 << 24, risk=(
+    'merging a segment that far from the data buffered zero-fills the gap in memory, '
+    'up to 2 GiB per segment at -1'))
+
 #: int: Most objects the ``_append_fallback`` method of a dumper
 #: :func:`~pcapkit.dumpkit.common.make_dumper` builds
 #: expands for one top-level object, counting an object once per time it is met,

tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py

@@ -13,6 +13,9 @@
 * #1356 -- a FIN that submits the buffer at once splits data reordered behind
   it off into a buffer of its own.
 
+The widest gap a segment is padded across is ``config.TCP_GAP_LIMIT``, a safety
+limit that ``-1`` lifts (#2546).
+
 Every case feeds hand-built segments and reads no capture. Classes are imported
 inside each test, after :func:`~tests._support.reimport_once_per_class`.
 
@@ -174,14 +177,62 @@ def test_a_segment_far_below_the_buffer_starts_a_new_stream(self) -> None:
         reasm = self._run(self._seg(0x30000000, b'A' * 5), self._seg(0x10000000, b'B' * 5))
         self.assertEqual(sorted(p for (_, _, _, p) in self._summary(reasm)), [b'A' * 5, b'B' * 5])
 
-    def test_a_gap_within_the_window_is_still_padded(self) -> None:
-        from pcapkit.foundation.reassembly.tcp import TCP
+    def test_a_gap_at_the_default_limit_is_still_padded(self) -> None:
+        from pcapkit.utilities import config
 
-        far = TCP.__window__
+        far = config.TCP_GAP_LIMIT
         reasm = self._run(self._seg(0, b'A' * 5), self._seg(5 + far, b'B' * 5), strict=False)
         (completed, _, index, payload), = self._summary(reasm)
         self.assertEqual((completed, index, len(payload)), ('PARTIAL', (1, 2), 10 + far))
 
+    # #2546 ------------------------------------------------------------------
+
+    def _gap_limit(self, limit):
+        """Patch ``config.TCP_GAP_LIMIT`` to ``limit`` for the rest of the test."""
+        from unittest import mock
+
+        from pcapkit.utilities import config
+
+        patcher = mock.patch.object(config, 'TCP_GAP_LIMIT', limit)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_a_gap_up_to_the_limit_is_padded(self) -> None:
+        self._gap_limit(100)
+        # the gap is measured on either side of the buffer
+        for side, segs, runs in (('after', (self._seg(0, b'A' * 5), self._seg(105, b'B' * 5)),
+                                  (b'A' * 5, b'B' * 5)),
+                                 ('before', (self._seg(200, b'A' * 5), self._seg(95, b'B' * 5)),
+                                  (b'B' * 5, b'A' * 5))):
+            with self.subTest(side=side):
+                self.assertEqual(self._summary(self._run(*segs)), [('PARTIAL', 500, (1, 2), runs)])
+
+    def test_a_gap_past_the_limit_splits_the_stream(self) -> None:
+        self._gap_limit(100)
+        for side, seq in (('after', 106), ('before', 94)):
+            with self.subTest(side=side, syn=False):
+                reasm = self._run(self._seg(200 if side == 'before' else 0, b'A' * 5),
+                                  self._seg(seq, b'B' * 5))
+                self.assertEqual(sorted((c, p) for (c, _, _, p) in self._summary(reasm)),
+                                 [('COMPLETE', b'A' * 5), ('COMPLETE', b'B' * 5)])
+        # a captured SYN keeps both halves one stream, each missing the gap
+        reasm = self._run(self._seg(0, syn=True), self._seg(1, b'A' * 5), self._seg(107, b'B' * 5))
+        self.assertEqual(self._summary(reasm), [('PARTIAL', 500, (3,), (b'B' * 5,)),
+                                                ('PARTIAL', 500, (1, 2), (b'A' * 5,))])
+        self.assertEqual(reasm._buffer[BUFID].ack[500].gap, [(6, 106)])
+
+    def test_minus_one_pads_a_gap_past_the_default(self) -> None:
+        self._gap_limit(-1)
+        far = (1 << 24) + 1
+        start = far + 10
+        for side, segs, runs in (('after', (self._seg(0, b'A' * 5), self._seg(5 + far, b'B' * 5)),
+                                  (b'A' * 5, b'B' * 5)),
+                                 ('before', (self._seg(start, b'A' * 5),
+                                             self._seg(start - far - 5, b'B' * 5)),
+                                  (b'B' * 5, b'A' * 5))):
+            with self.subTest(side=side):
+                self.assertEqual(self._summary(self._run(*segs)), [('PARTIAL', 500, (1, 2), runs)])
+
     def test_data_reordered_behind_the_fin_joins_the_stream(self) -> None:
         reasm = self._run(self._seg(100, syn=True, ack=0), self._seg(101, b'AAAAA'),
                           self._seg(111, fin=True), self._seg(106, b'BBBBB'))

tests/project/test_rfc_constants_one_home_1739_unit.py

@@ -350,8 +350,6 @@
         'an if_tsoffset option: 4 octets of header and 8 of value',
     ('pcapkit/foundation/engines/scapy.py', 'options[4:12]'):
         'the 8-octet value of an if_tsoffset option',
-    ('pcapkit/foundation/reassembly/tcp.py', '1 << 24'):
-        'a bit width, not a PCAP global header',
     ('pcapkit/toolkit/dpkt.py', '_fragment(6)'):
         'the IP version number',
     ('pcapkit/toolkit/dpkt.py', 'version == 6'):

tests/utilities/test_config_unit.py

@@ -72,6 +72,7 @@
     ('FRAME_LAYER_LIMIT', 64, 0, 'RecursionError (#1610)'),
     ('EXTENSION_HEADER_LIMIT', 32, 0, 'RecursionError (#1610)'),
     ('IPV6_TCP_WARNED_LIMIT', 32, 0, 'pins one of its packets in memory'),
+    ('TCP_GAP_LIMIT', 16_777_216, 0, 'zero-fills the gap in memory'),
     ('FALLBACK_LIMIT', 10_000, 0, 'a large or endless object graph'),
     ('DETECT_CACHE_SIZE', 1024, 0, 'never-repeating text'),
     ('READ_CHUNK', 1_048_576, 1, 'a bogus record length allocates'),
@@ -301,6 +302,12 @@ def test_a_moved_name_has_no_re_export(self) -> None:
                 with self.subTest(module=module, name=name):
                     self.assertFalse(hasattr(loaded, name))
 
+    def test_the_tcp_gap_limit_left_no_class_attribute(self) -> None:
+        # #2546: TCP.__window__ moved here as TCP_GAP_LIMIT.
+        from pcapkit.foundation.reassembly.tcp import TCP
+
+        self.assertFalse(hasattr(TCP, '__window__'))
+
     def test_every_variable_is_documented(self) -> None:
         from pcapkit.utilities import config