Fixes #6097

Author: stamparm

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.8.38"
+VERSION = "1.10.8.39"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/utils/keysetdump.py

@@ -149,7 +149,10 @@ def _lit(value):
     (e.g. 0x.. hex) form for string keys. Both forms are self-contained (no surrounding quotes).
     """
 
-    if value is not None and re.match(r"\A-?[0-9]+\Z", value):
+    if value is None:
+        return NULL                                 # unescaper.escape() passes None through, and a bare
+                                                    # None formatted into a predicate is not even SQL
+    if re.match(r"\A-?[0-9]+\Z", value):
         return value
     return unescaper.escape(value, False)
 
@@ -161,18 +164,24 @@ def _embed(template, value, *fixed):
     template = template.replace("'%s'", "%s")
     return template % (fixed + (_lit(value),))
 
-def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
-    blind = queries[Backend.getIdentifiedDbms()].dump_table.blind
-    field = agent.preprocessField(tbl, cursor)
+def _target(count):
+    """Rows the walk is expected to produce, honouring --start/--stop."""
 
     if conf.limitStart and conf.limitStop:
-        target = max(0, conf.limitStop - conf.limitStart + 1)
+        return max(0, conf.limitStop - conf.limitStart + 1)
     elif conf.limitStop:
-        target = conf.limitStop
+        return conf.limitStop
     elif conf.limitStart:
-        target = max(0, count - conf.limitStart + 1)
-    else:
-        target = count
+        return max(0, count - conf.limitStart + 1)
+
+    return count
+
+def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
+    """False when the walk gave up mid-table (the caller then discards the partial result)."""
+
+    blind = queries[Backend.getIdentifiedDbms()].dump_table.blind
+    field = agent.preprocessField(tbl, cursor)
+    target = _target(count)
 
     pivotValue = None
 
@@ -182,7 +191,7 @@ def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
         seed = unArrayizeValue(inject.getValue(query))
 
         if isNoneValue(seed) or seed == NULL:
-            return
+            return False                            # no seed, no walk - and an empty table is not that
 
         pivotValue = safechardecode(seed)
 
@@ -205,7 +214,7 @@ def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
         # safety latch against a non-advancing cursor (e.g. encoding edge cases)
         if value == pivotValue:
             singleTimeWarnMessage("keyset cursor stopped advancing prematurely")
-            break
+            return False
 
         pivotValue = value
 
@@ -223,20 +232,17 @@ def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
 
         produced += 1
 
+    return True
+
 def _dumpComposite(tbl, colList, count, cursorCols, tableRef, entries, lengths):
+    """False when the walk gave up mid-table (the caller then discards the partial result)."""
+
     blind = queries[Backend.getIdentifiedDbms()].dump_table.blind
     fields = [agent.preprocessField(tbl, _) for _ in cursorCols]
     orderExpr = ','.join(fields)
 
     startSkip = (conf.limitStart - 1) if conf.limitStart else 0
-    if conf.limitStart and conf.limitStop:
-        target = max(0, conf.limitStop - conf.limitStart + 1)
-    elif conf.limitStop:
-        target = conf.limitStop
-    elif conf.limitStart:
-        target = max(0, count - conf.limitStart + 1)
-    else:
-        target = count
+    target = _target(count)
 
     prev = None
     produced = 0
@@ -263,11 +269,18 @@ def _dumpComposite(tbl, colList, count, cursorCols, tableRef, entries, lengths):
             tup.append(None if isNoneValue(value) else safechardecode(value))
 
         if all(isNoneValue(_) for _ in tup):
-            break
+            break                                   # nothing past the cursor: the table is walked
+
+        # A key column that did not come back (an error-channel miss, a blocked payload) cannot be
+        # seeked on, and its equality would pin the rest of the row to a NULL - so the walk stops
+        # here rather than emitting a row of empty cells and carrying the hole into the next seek
+        if any(isNoneValue(_) for _ in tup):
+            singleTimeWarnMessage("keyset cursor could not be retrieved for one of the key column(s)")
+            return False
 
         if prev is not None and tup == prev:
             singleTimeWarnMessage("keyset cursor stopped advancing prematurely")
-            break
+            return False
 
         prev = tup
         seen += 1
@@ -290,6 +303,8 @@ def _dumpComposite(tbl, colList, count, cursorCols, tableRef, entries, lengths):
 
         produced += 1
 
+    return True
+
 def keysetDumpTable(tbl, colList, count, cursor):
     """
     Dumps a table one row at a time using keyset (seek) pagination on 'cursor' (a list of
@@ -298,6 +313,10 @@ def keysetDumpTable(tbl, colList, count, cursor):
     exact equality on the cursor (index point seek), so no row is skipped via OFFSET and no
     per-row ORDER BY filesort is needed. A deep --start uses a single OFFSET "seed" jump
     (single-column cursors), after which the walk is pure keyset.
+
+    Returns None when the walk gave up mid-table (a key value that did not come back, a cursor
+    that stopped advancing): a short table is worse than a slow one, so the caller redoes it
+    with the standard OFFSET dump instead of showing whatever was reached.
     """
 
     tableRef = _tableRef(tbl)
@@ -309,9 +328,16 @@ def keysetDumpTable(tbl, colList, count, cursor):
         entries[column] = BigArray()
 
     if len(cursor) == 1:
-        _dumpSingle(tbl, colList, count, cursor[0], tableRef, entries, lengths)
+        complete = _dumpSingle(tbl, colList, count, cursor[0], tableRef, entries, lengths)
     else:
-        _dumpComposite(tbl, colList, count, cursor, tableRef, entries, lengths)
+        complete = _dumpComposite(tbl, colList, count, cursor, tableRef, entries, lengths)
+
+    if not complete:
+        warnMsg = "keyset pagination did not complete for table '%s', " % unsafeSQLIdentificatorNaming(tbl)
+        warnMsg += "falling back to the standard dump"
+        logger.warning(warnMsg)
+
+        return None
 
     debugMsg = "keyset pagination retrieved %d row(s) for table '%s'" % (len(entries[colList[0]]) if colList and colList[0] in entries else 0, unsafeSQLIdentificatorNaming(tbl))
     logger.debug(debugMsg)

plugins/generic/entries.py

@@ -231,12 +231,17 @@ def _dumpCountQuery():
                             logger.info(infoMsg)
 
                             try:
-                                entries, lengths = keysetDumpTable(tbl, colList, int(count), keysetCursor)
-                                for column, columnEntries in entries.items():
-                                    length = max(lengths[column], getConsoleLength(column))
-                                    kb.data.dumpedTable[column] = {"length": length, "values": columnEntries}
-                                    entriesCount = len(columnEntries)
-                                keysetDone = bool(kb.data.dumpedTable)
+                                # None when the walk gave up mid-table: nothing is kept, so the
+                                # standard dump below redoes it rather than showing a short table
+                                result = keysetDumpTable(tbl, colList, int(count), keysetCursor)
+
+                                if result is not None:
+                                    entries, lengths = result
+                                    for column, columnEntries in entries.items():
+                                        length = max(lengths[column], getConsoleLength(column))
+                                        kb.data.dumpedTable[column] = {"length": length, "values": columnEntries}
+                                        entriesCount = len(columnEntries)
+                                    keysetDone = bool(kb.data.dumpedTable)
                             except KeyboardInterrupt:
                                 kb.dumpKeyboardInterrupt = True
                                 clearConsoleLine()
@@ -379,6 +384,26 @@ def _dumpCountQuery():
                     lengths = {}
                     entries = {}
 
+                    # Attempted before the chain below so that a walk which gave up mid-table (None) can
+                    # fall through to the standard OFFSET paths - a short table is worse than a slow one.
+                    # An interrupted walk keeps the chain out of a re-dump by claiming the branch itself.
+                    keysetResult = None
+
+                    if keysetCursor:
+                        infoMsg = "using keyset (seek) pagination on column(s) '%s' " % ', '.join(keysetCursor)
+                        infoMsg += "for table '%s'" % unsafeSQLIdentificatorNaming(tbl)
+                        logger.info(infoMsg)
+
+                        try:
+                            keysetResult = keysetDumpTable(tbl, colList, count, keysetCursor)
+                        except KeyboardInterrupt:
+                            kb.dumpKeyboardInterrupt = True
+                            clearConsoleLine()
+                            warnMsg = "Ctrl+C detected in dumping phase"
+                            logger.warning(warnMsg)
+
+                            keysetResult = (entries, lengths)
+
                     if count == 0:
                         warnMsg = "table '%s' " % unsafeSQLIdentificatorNaming(tbl)
                         warnMsg += "in database '%s' " % unsafeSQLIdentificatorNaming(conf.db)
@@ -399,18 +424,8 @@ def _dumpCountQuery():
 
                         continue
 
-                    elif keysetCursor:
-                        infoMsg = "using keyset (seek) pagination on column(s) '%s' " % ', '.join(keysetCursor)
-                        infoMsg += "for table '%s'" % unsafeSQLIdentificatorNaming(tbl)
-                        logger.info(infoMsg)
-
-                        try:
-                            entries, lengths = keysetDumpTable(tbl, colList, count, keysetCursor)
-                        except KeyboardInterrupt:
-                            kb.dumpKeyboardInterrupt = True
-                            clearConsoleLine()
-                            warnMsg = "Ctrl+C detected in dumping phase"
-                            logger.warning(warnMsg)
+                    elif keysetResult is not None:
+                        entries, lengths = keysetResult
 
                     elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.SYBASE, DBMS.MAXDB, DBMS.MSSQL, DBMS.INFORMIX, DBMS.MCKOI, DBMS.RAIMA):
                         if Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.RAIMA):

tests/test_entries.py

@@ -765,6 +765,46 @@ def gv(query, *a, **k):
         self.assertEqual(list(dumped["id"]["values"]), ["1", "2"])
         self.assertEqual(list(dumped["name"]["values"]), ["alice", "bob"])
 
+    def test_dump_table_inference_keyset_giving_up_falls_back(self):
+        # A keyset walk that gives up mid-table hands back None (issue #6097: a key value that did
+        # not come back). The table must then be dumped by the standard OFFSET path IN FULL - a
+        # short table would otherwise be presented as the whole thing.
+        set_dbms("MySQL")
+        e = self._entries(cols=("id", "name"))
+        conf.db = "testdb"
+        conf.tbl = "users"
+        conf.col = None
+        conf.noKeyset = False
+        conf.keyset = True                          # keyset regardless of the row-count threshold
+
+        savedResolve, savedDump = emod.resolveKeysetCursor, emod.keysetDumpTable
+        attempted = []
+        emod.resolveKeysetCursor = lambda tbl, colList: ["id"]
+        emod.keysetDumpTable = lambda *a, **k: attempted.append(a) or None   # gave up, nothing kept
+
+        data = {0: {"id": "1", "name": "alice"}, 1: {"id": "2", "name": "bob"}}
+
+        def gv(query, *a, **k):
+            if k.get("expected") == EXPECTED.INT:
+                return "2"
+            import re as _re
+            idx = int(_re.search(r"LIMIT\s+(\d+)\s*,\s*1", query).group(1))
+            proj = query.split(" FROM ", 1)[0]
+            return data[idx]["name" if "name" in proj else "id"]
+
+        emod.inject.getValue = gv
+
+        try:
+            e.dumpTable()
+        finally:
+            emod.resolveKeysetCursor, emod.keysetDumpTable = savedResolve, savedDump
+
+        self.assertEqual(len(attempted), 1)         # the walk really was tried, then discarded
+        dumped = conf.dumper.tableValues[-1]
+        self.assertEqual(dumped["__infos__"]["count"], 2)
+        self.assertEqual(list(dumped["id"]["values"]), ["1", "2"])
+        self.assertEqual(list(dumped["name"]["values"]), ["alice", "bob"])
+
     def test_dump_table_inference_empty_table(self):
         # A zero row count in the inference path yields empty per-column value
         # lists and no dbTableValues emission (dumpedTable stays effectively empty).

tests/test_keyset_engine.py

@@ -68,15 +68,19 @@ def tearDown(self):
         kb.data.cachedColumns = self._s["cachedColumns"]
         inject.getValue = self._s["gv"]
 
-    def _install_oracle(self, rowValueSupported):
+    def _install_oracle(self, rowValueSupported, dropped=()):
         def oracle(query=None, **kwargs):
             # a back-end without ANSI row-value support errors on (a,b)>(x,y) -> no result
             if re.search(r"\)\s*>\s*\(", query or "") and not rowValueSupported:
                 return None
             m = re.search(r"SELECT (\w+) FROM .+? WHERE (.+) ORDER BY .+ LIMIT 1", query or "")   # advance
             if m:
                 cand = sorted(r for r in _ROWS if _condTrue(m.group(2), r))
-                return None if not cand else str(cand[0][_COL_INDEX[m.group(1)]])
+                if not cand:
+                    return None
+                if (m.group(1), cand[0]) in dropped:   # a key cell the channel did not bring back
+                    return None
+                return str(cand[0][_COL_INDEX[m.group(1)]])
             m = re.search(r"SELECT MAX\((\w+)\) FROM .+? WHERE (.+)", query or "")                 # point fetch
             if m:
                 cand = [r for r in _ROWS if _condTrue(m.group(2), r)]
@@ -85,9 +89,16 @@ def oracle(query=None, **kwargs):
 
         inject.getValue = oracle
 
-    def _dump(self, rowValueSupported):
-        self._install_oracle(rowValueSupported)
-        entries, _ = ks.keysetDumpTable("users", ["a", "b", "d"], len(_ROWS), ["a", "b"])
+    def _walk(self, rowValueSupported, dropped=()):
+        """The raw result: (entries, lengths), or None when the walk gave up and the caller must fall back."""
+
+        self._install_oracle(rowValueSupported, dropped)
+
+        return ks.keysetDumpTable("users", ["a", "b", "d"], len(_ROWS), ["a", "b"])
+
+    def _dump(self, rowValueSupported, dropped=()):
+        entries, _ = self._walk(rowValueSupported, dropped)
+
         return list(zip(entries["a"], entries["b"], entries["d"]))
 
     def test_all_rows_when_row_value_supported(self):
@@ -100,6 +111,14 @@ def test_all_rows_when_row_value_rejected(self):
         self.assertEqual(len(rows), len(_ROWS))
         self.assertEqual([r[2] for r in rows], [r[2] for r in _ROWS])
 
+    def test_unretrieved_key_cell_is_handed_back_for_the_offset_fallback(self):
+        # one key column of a row that does not come back (an error-channel miss, a blocked payload)
+        # leaves a None in the cursor tuple. Seeking on it is impossible, so the walk must hand back
+        # NOTHING - it used to format that None into the next seek predicate (a TypeError, issue
+        # #6097) and, once the predicate became a plain '%s', to emit a row of empty cells and then
+        # silently truncate the table
+        self.assertIsNone(self._walk(rowValueSupported=True, dropped={("b", (2, 5, "gamma"))}))
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)