console-tui: the JSON reader decodes \uXXXX, so a name with an accent is not cafu00e9

os/v3/puppy/console-tui/README.md

@@ -134,6 +134,13 @@ visible repair: a cell is simply not the place to execute what a log file says.
 space and drops the other C0/DEL; everything else is untouched (`SoC 78°C`, `café`, `┌─┐`), and it is applied where
 external text enters, so no draw site can forget it (`a_log_line_or_a_model_name_cannot_carry_an_escape_into_the_screen`).
 
+**And the reader decodes a JSON escape before the sanitiser sees it.** Lemonade's catalogue arrives with `\uXXXX` in it
+whenever a serialiser escapes non-ASCII (Python's `json.dumps` does that by default), so `facts::json_unicode()` turns
+`caf\u00e9` into `café` - surrogate pairs into one character, a lone half into U+FFFD, a truncated escape into the bytes it
+had - and *then* `screen_text()` strips whatever control characters an escape decodes to (`\u001b[2J`). The order matters:
+before this the pane drew the escape's four characters (`cafu00e9`), and a `\u001b` body had nothing for the sanitiser to
+remove (`a_json_escape_is_decoded_before_it_becomes_a_cell`).
+
 **An interrupted paint repaints; a terminal that cannot be written still ends the console.** A paint that fails with
 `ErrorKind::Interrupted` drew nothing - a signal arrived before the write reached the terminal - so the next pass simply
 paints again instead of ending the screen. That is reachable in normal use: crossterm's event source installs a SIGWINCH
@@ -232,7 +239,7 @@ dynamic one (`readelf` PT_INTERP, `ldd`).
 ## Tests
 
 ```sh
-cargo test                                    # 58 tests: the probe parsers, the QR drawing (incl. a literal known answer), the links, the short-window degradation, the two renders, the engine row's model choice, the escape sanitiser, the interrupted paint, the rows that have to fit the pane
+cargo test                                    # the suite: the probe parsers (incl. the JSON escapes), the QR drawing (incl. a literal known answer), the links, the short-window degradation, the two renders, the engine row's model choice, the escape sanitiser, the interrupted paint
 cargo run -- --snapshot 240x67 --fixture      # the frame tests/snapshots/240x67.txt holds
 cargo run -- --snapshot 80x25 --fixture
 cargo run -- --status                         # the same facts as plain text (what a shell or a gate row can read)

os/v3/puppy/console-tui/src/facts.rs

@@ -762,26 +762,77 @@ pub fn json_raw(obj: &str, key: &str) -> Option<String> {
         return Some(rest[..end].trim().to_string());
     };
     let mut out = String::new();
-    let mut esc = false;
-    for c in s.chars() {
-        if esc {
-            out.push(match c {
-                'n' => '\n',
-                't' => '\t',
-                other => other,
-            });
-            esc = false;
-        } else if c == '\\' {
-            esc = true;
-        } else if c == '"' {
-            return Some(out);
-        } else {
-            out.push(c);
+    let mut it = s.chars();
+    while let Some(c) = it.next() {
+        match c {
+            '"' => return Some(out),
+            '\\' => match it.next() {
+                Some('n') => out.push('\n'),
+                Some('t') => out.push('\t'),
+                Some('r') => out.push('\r'),
+                Some('b') => out.push('\u{8}'),
+                Some('f') => out.push('\u{c}'),
+                Some('u') => out.push_str(&json_unicode(&mut it)),
+                Some(other) => out.push(other),
+                None => out.push('\\'),
+            },
+            other => out.push(other),
         }
     }
     None
 }
 
+/// The `\uXXXX` escape (and a surrogate pair, which is two of them). A serialiser that escapes non-ASCII delivers
+/// `caf\u00e9` - Python's `json.dumps` does that by default - and a model name or a recipe from the catalogue is DRAWN,
+/// so before this the screen said `cafu00e9`. A code point that is not a character (a lone surrogate) becomes U+FFFD, and
+/// a truncated escape keeps the bytes it actually had and consumes only the hex digits it took.
+fn json_unicode(it: &mut std::str::Chars) -> String {
+    let mut probe = it.clone();
+    let mut digits = String::new();
+    for _ in 0..4 {
+        match probe.next() {
+            Some(d) if d.is_ascii_hexdigit() => digits.push(d),
+            // not a hex digit: it is the next character of the string, not part of this escape
+            _ => break,
+        }
+    }
+    if digits.len() != 4 {
+        for _ in 0..digits.chars().count() {
+            it.next();
+        }
+        return format!("u{digits}");
+    }
+    *it = probe;
+    let Ok(v) = u32::from_str_radix(&digits, 16) else {
+        return format!("u{digits}");
+    };
+    let cp = if (0xD800..0xDC00).contains(&v) {
+        // a high surrogate is one half of a pair: take the low one with it, or say so
+        let mut peek = it.clone();
+        let mut lo = String::new();
+        if peek.next() == Some('\\') && peek.next() == Some('u') {
+            for _ in 0..4 {
+                match peek.next() {
+                    Some(d) if d.is_ascii_hexdigit() => lo.push(d),
+                    _ => break,
+                }
+            }
+        }
+        match u32::from_str_radix(&lo, 16) {
+            Ok(w) if lo.len() == 4 && (0xDC00..0xE000).contains(&w) => {
+                *it = peek;
+                0x10000 + ((v - 0xD800) << 10) + (w - 0xDC00)
+            }
+            _ => 0xFFFD,
+        }
+    } else if (0xDC00..0xE000).contains(&v) {
+        0xFFFD
+    } else {
+        v
+    };
+    char::from_u32(cp).unwrap_or('\u{FFFD}').to_string()
+}
+
 pub fn json_str(obj: &str, key: &str) -> Option<String> {
     json_raw(obj, key)
 }
@@ -1513,8 +1564,8 @@ mod tests {
         }
         assert_eq!(screen_text(""), "");
         // and the catalogue is sanitised where it enters, because a model name is drawn: whatever bytes the JSON hands
-        // over, what is DRAWN is cleaned. (A literal ESC is used rather
-        // than a `\u001b` escape because this crate's JSON reader does not implement `\uXXXX` - see the note in json_raw.)
+        // over, what is DRAWN is cleaned - first the escape is decoded (`\uXXXX`, see json_unicode), then the control
+        // bytes it decodes to are stripped, which is the order a real body arrives in.
         let esc = '\u{1b}';
         let json = format!(
             "{{\"data\":[{{\"id\":\"evil{esc}[2Jmodel\",\"created\":0,\"object\":\"model\",\"recipe\":\"llamacpp\",\"downloaded\":true,\"recipe_options\":{{\"onebit_backend\":\"gpu{esc}[31m\"}}}}]}}"
@@ -1525,6 +1576,67 @@ mod tests {
         assert_eq!(m.engines[0].backend.as_deref(), Some("gpu"));
     }
 
+    #[test]
+    fn a_json_escape_is_decoded_before_it_becomes_a_cell() {
+        // The shape a serialiser actually produces for a non-ASCII name: Python's `json.dumps` escapes by default, so
+        // Lemonade's catalogue hands over `\u00e9` rather than the UTF-8 byte. Before this the row said `cafu00e9`.
+        assert_eq!(
+            json_str(r#"{"id":"caf\u00e9","recipe":"x"}"#, "id").as_deref(),
+            Some("café")
+        );
+        assert_eq!(
+            json_str(r#"{"id":"x","recipe":"\u65e5\u672c\u8a9e"}"#, "recipe").as_deref(),
+            Some("日本語")
+        );
+        // a surrogate pair is ONE character; a lone half of one is not a character at all
+        assert_eq!(
+            json_str(r#"{"id":"\ud83d\ude00"}"#, "id").as_deref(),
+            Some("\u{1F600}")
+        );
+        assert_eq!(
+            json_str(r#"{"id":"a\ud83db"}"#, "id").as_deref(),
+            Some("a\u{FFFD}b")
+        );
+        // escapes that are characters in their own right, which used to come through as the letter
+        assert_eq!(
+            json_str(r#"{"id":"a\rb\bc\fd"}"#, "id").as_deref(),
+            Some("a\rb\u{8}c\u{c}d")
+        );
+        // an escape this reader does not know, and a truncated one, keep what the bytes said
+        assert_eq!(json_str(r#"{"id":"a\qb"}"#, "id").as_deref(), Some("aqb"));
+        assert_eq!(json_str(r#"{"id":"a\u12"}"#, "id").as_deref(), Some("au12"));
+        // and nothing else moved
+        assert_eq!(
+            json_str(r#"{"id":"plain-1bit","recipe":"llamacpp"}"#, "id").as_deref(),
+            Some("plain-1bit")
+        );
+    }
+
+    #[test]
+    fn an_escaped_catalogue_name_reaches_the_engine_row_decoded() {
+        // The unit test above is about the reader; this is about the row a person reads: the escape is decoded, the
+        // sanitiser then has real bytes to clean, and what is drawn is the name the catalogue meant.
+        let json = r#"{"data":[{"id":"qwen3-caf\u00e9","created":0,"object":"model","recipe":"llamacpp","downloaded":true,
+            "recipe_options":{"onebit_backend":"gpu"}}]}"#;
+        let m = models_from(json);
+        assert_eq!(m.downloaded, vec!["qwen3-café"]);
+        assert_eq!(m.engines[0].recipe, "llamacpp");
+        assert_eq!(m.engines[0].backend.as_deref(), Some("gpu"));
+        // (loaded: the catalogue's own names; serving: what /api/v1/health says answered)
+        let loaded = m.downloaded.clone();
+        assert_eq!(
+            engine_of(&loaded, Some("qwen3-café"), &m.engines, &[]),
+            Some("llamacpp (gpu)".into())
+        );
+        // and a body that escapes a CONTROL character is decoded first and then stripped, in that order: the cell never
+        // sees the ESC. (`\u001b` used to arrive here as the four characters `u001b`, so the sanitiser had nothing to do.)
+        let evil = r#"{"data":[{"id":"evil\u001b[2Jmodel","created":0,"object":"model","recipe":"llamacpp","downloaded":true,
+            "recipe_options":{"onebit_backend":"gpu\u001b[31m"}}]}"#;
+        let e = models_from(evil);
+        assert_eq!(e.downloaded, vec!["evilmodel"]);
+        assert_eq!(e.engines[0].backend.as_deref(), Some("gpu"));
+    }
+
     #[test]
     fn models_keep_only_downloaded() {
         // the shape /api/v1/models documents, with the catalogue's undownloaded entry alongside