fix: keep 8-bit color codes, strip them, and drop other 8-bit sequences whole

src/utils/__tests__/ansi.test.ts

@@ -9,6 +9,7 @@ import {
     getVisibleWidth,
     restoreBackgroundAfterResets,
     stripOscCodes,
+    stripSgrCodes,
     truncateStyledText
 } from '../ansi';
 
@@ -112,6 +113,19 @@ describe('truncateStyledText', () => {
     });
 });
 
+describe('stripSgrCodes', () => {
+    it('removes 7-bit and 8-bit color codes', () => {
+        expect(stripSgrCodes('\x9b31mred\x9b39m')).toBe('red');
+        expect(stripSgrCodes('\x1b[1m\x9b38;5;208mbold\x9b0m\x1b[22m')).toBe('bold');
+    });
+
+    it('leaves other CSI sequences and hyperlinks alone', () => {
+        const text = `\x1b[2K\x9b2J${OSC8_OPEN}link${OSC8_CLOSE}${C1_OSC8_OPEN}c1${C1_OSC8_CLOSE}`;
+
+        expect(stripSgrCodes(text)).toBe(text);
+    });
+});
+
 describe('restoreBackgroundAfterResets', () => {
     const BG = '\x1b[44m';
 
@@ -120,6 +134,11 @@ describe('restoreBackgroundAfterResets', () => {
             .toBe(`\x1b[31ma\x1b[0m${BG}b\x1b[m${BG}c\x1b[00m${BG}d\x1b[;1m${BG}e\x1b[39;49m${BG}f`);
     });
 
+    it('re-applies the background after an 8-bit reset', () => {
+        expect(restoreBackgroundAfterResets('\x9b31ma\x9b0mb\x9b49mc', BG))
+            .toBe(`\x9b31ma\x9b0m${BG}b\x9b49m${BG}c`);
+    });
+
     it('leaves other styling, and the arguments of extended colors, alone', () => {
         const text = '\x1b[1;31ma\x1b[38;5;0mb\x1b[48;2;0;49;0mc\x1b[58;5;49md\x1b[39me';
 

src/utils/__tests__/renderer-sanitize.test.ts

@@ -20,4 +20,13 @@ describe('preRenderAllWidgets', () => {
         expect(line[0]?.content).toBe('safe text');
         expect(line[0]?.plainLength).toBe('safe text'.length);
     });
+
+    it('keeps 8-bit colors in widget output, in their 7-bit form', () => {
+        const widget: WidgetItem = { id: 'text', type: 'custom-text', customText: '\x9b31mred\x9b39m text' };
+
+        const [line = []] = preRenderAllWidgets([[widget]], DEFAULT_SETTINGS, context);
+
+        expect(line[0]?.content).toBe('\x1b[31mred\x1b[39m text');
+        expect(line[0]?.plainLength).toBe('red text'.length);
+    });
 });

src/utils/__tests__/terminal-sanitize.test.ts

@@ -84,17 +84,37 @@ describe('sanitizeTerminalText', () => {
         ['shift out', '\x0e'],
         ['form feed', '\f'],
         ['DEL', '\x7f'],
-        ['8-bit CSI', '\x9b'],
-        ['8-bit OSC', '\x9d'],
         ['8-bit ST', '\x9c'],
-        ['8-bit DCS', '\x90'],
         ['NEL', '\x85']
     ])('drops the control character %s', (_label, control) => {
         expect(sanitizeTerminalText(`a${control}b`)).toBe('ab');
     });
 
-    it('drops an 8-bit CSI sequence\'s introducer, leaving its harmless parameters as text', () => {
-        expect(sanitizeTerminalText('a\x9b2Jb')).toBe('a2Jb');
+    // 8-bit C1 introducers stand for ESC and a character: CSI (0x9b) is ESC [,
+    // OSC (0x9d) is ESC ], and ST (0x9c) is ESC \
+    it('keeps 8-bit color codes, written in their 7-bit form', () => {
+        expect(sanitizeTerminalText('\x9b38;5;208mred\x9b39m')).toBe(`${ESC}[38;5;208mred${ESC}[39m`);
+    });
+
+    it('keeps an 8-bit hyperlink with a safe URL, written in its 7-bit form', () => {
+        const link = `\x9d8;;https://github.com/o/r\x9ctext\x9d8;;${BEL}`;
+
+        expect(sanitizeTerminalText(link)).toBe(`${ESC}]8;;https://github.com/o/r${ST}text${ESC}]8;;${BEL}`);
+    });
+
+    it.each([
+        ['clearing the screen', '\x9b2J'],
+        ['moving the cursor', '\x9b10;20H'],
+        ['repeating a character', '\x9bb'],
+        ['an OSC 52 clipboard write', `\x9d52;c;ZWNobyBwd25lZA==${BEL}`],
+        ['an OSC 0 title change', '\x9d0;hacked\x9c'],
+        ['a hyperlink with an unsafe URL', '\x9d8;;https://x/a b\x9c'],
+        ['a DCS string', '\x90q#0;2;0;0;0\x9c'],
+        ['an APC string', `\x9fpayload${ST}`],
+        ['a PM string', '\x9epayload\x9c'],
+        ['a SOS string', '\x98payload\x9c']
+    ])('drops an 8-bit sequence entirely: %s', (_label, sequence) => {
+        expect(sanitizeTerminalText(`a${sequence}b`)).toBe('ab');
     });
 });
 

src/utils/ansi.ts

@@ -19,7 +19,8 @@ const VARIATION_SELECTOR_SUPPLEMENT_END = 0xe01ef;
 const REGIONAL_INDICATOR_START = 0x1f1e6;
 const REGIONAL_INDICATOR_END = 0x1f1ff;
 
-const SGR_REGEX = /\x1b\[[0-9;]*m/g;
+// Color codes, written with ESC [ or the 8-bit CSI
+const SGR_REGEX = /(?:\x1b\[|\x9b)[0-9;]*m/g;
 // Lowest code point that can extend a display cluster (U+0300 is the first
 // \p{Mark}; ZWJ, variation selectors, the keycap and emoji modifiers are all
 // higher), so a printable ASCII character followed by anything below it is a
@@ -399,7 +400,8 @@ export function stripSgrCodes(text: string): string {
 // reset (0, an empty parameter, or 49). Skip extended color arguments so their
 // values aren't mistaken for resets or background operations.
 function sgrClearsBackground(sequence: string): boolean {
-    const params = sequence.slice(2, -1).split(';').map(param => Number.parseInt(param || '0', 10));
+    const introducerLength = sequence.startsWith(C1_CSI) ? 1 : 2;
+    const params = sequence.slice(introducerLength, -1).split(';').map(param => Number.parseInt(param || '0', 10));
     let clearsBackground = false;
     for (let i = 0; i < params.length; i++) {
         const param = params[i];

src/utils/terminal-sanitize.ts

@@ -9,19 +9,29 @@ const ESC = '\x1b';
 const BEL = '\x07';
 const C1_ST = '\x9c';
 
-// Colors and text attributes: ESC [ <digits, ; and :> m
-const SGR = /\x1b\[[0-9;:]*m/y;
-// A hyperlink: ESC ] 8 ; <parameters> ; <URI> then ESC \ or BEL. The URI may not
-// hold control characters or spaces, so it can't end the sequence early or start
-// another.
-const OSC8 = /\x1b\]8;[A-Za-z0-9=:_.-]*;[^\x00-\x20\x7f-\x9f]*(?:\x1b\\|\x07)/y;
+// Colors and text attributes: ESC [ (or 8-bit CSI) <digits, ; and :> m
+const SGR = /(?:\x1b\[|\x9b)[0-9;:]*m/y;
+// A hyperlink: ESC ] (or 8-bit OSC) 8 ; <parameters> ; <URI> then ESC \, ST or
+// BEL. The URI may not hold control characters or spaces, so it can't end the
+// sequence early or start another.
+const OSC8 = /(?:\x1b\]|\x9d)8;[A-Za-z0-9=:_.-]*;[^\x00-\x20\x7f-\x9f]*(?:\x1b\\|\x9c|\x07)/y;
 const SAFE_URL = /^[^\x00-\x20\x7f-\x9f]*$/;
 // C0 controls other than tab and newline, DEL, and the 8-bit C1 controls, which
 // some terminals read like ESC sequences
 const CONTROL = /[\x00-\x08\x0b-\x1f\x7f-\x9f]/;
 const CONTROL_GLOBAL = /[\x00-\x08\x0b-\x1f\x7f-\x9f]/g;
 // Strings the terminal swallows up to ST: OSC, DCS, SOS, PM and APC
 const STRING_INTRODUCERS = new Set([']', 'P', 'X', '^', '_']);
+// The 8-bit C1 controls that start a sequence, and the character that follows
+// ESC in their 7-bit form: CSI is ESC [, OSC is ESC ], and so on
+const C1_INTRODUCERS = new Map([
+    ['\x90', 'P'],
+    ['\x98', 'X'],
+    ['\x9b', '['],
+    ['\x9d', ']'],
+    ['\x9e', '^'],
+    ['\x9f', '_']
+]);
 
 function matchAt(pattern: RegExp, text: string, index: number): string | null {
     pattern.lastIndex = index;
@@ -36,15 +46,31 @@ function isIntermediate(code: number): boolean {
     return code >= 0x20 && code <= 0x2f;
 }
 
-// Where the escape sequence starting at `index` ends
+function startsSequence(char: string | undefined): boolean {
+    return char === ESC || (char !== undefined && C1_INTRODUCERS.has(char));
+}
+
+// A kept sequence in its 7-bit form, which every terminal reads and the rest of
+// the renderer parses
+function toSevenBit(sequence: string): string {
+    const introducer = C1_INTRODUCERS.get(sequence.charAt(0));
+    const sevenBit = introducer ? ESC + introducer + sequence.slice(1) : sequence;
+    return sevenBit.endsWith(C1_ST) ? `${sevenBit.slice(0, -1)}${ESC}\\` : sevenBit;
+}
+
+// Where the escape sequence starting at `index`, with ESC or an 8-bit
+// introducer, ends
 function skipEscapeSequence(text: string, index: number): number {
-    const introducer = text[index + 1];
+    const c1Introducer = C1_INTRODUCERS.get(text.charAt(index));
+    const introducer = c1Introducer ?? text[index + 1];
     if (introducer === undefined) {
         return index + 1;
     }
+    // Where the sequence's parameters or string start
+    const bodyStart = c1Introducer ? index + 1 : index + 2;
 
     if (introducer === '[') {
-        let end = index + 2;
+        let end = bodyStart;
         while (end < text.length && isCsiParameterOrIntermediate(text.charCodeAt(end))) {
             end++;
         }
@@ -53,7 +79,7 @@ function skipEscapeSequence(text: string, index: number): number {
     }
 
     if (STRING_INTRODUCERS.has(introducer)) {
-        for (let end = index + 2; end < text.length; end++) {
+        for (let end = bodyStart; end < text.length; end++) {
             const char = text[end];
             if (char === BEL || char === C1_ST) {
                 return end + 1;
@@ -83,6 +109,7 @@ function skipEscapeSequence(text: string, index: number): number {
 /**
  * The text with every control character and escape sequence removed, except
  * color codes (SGR) and hyperlinks (OSC 8) whose URL holds no control characters or spaces.
+ * Those are kept in their 7-bit form, even when written with 8-bit C1 controls.
  */
 export function sanitizeTerminalText(text: string): string {
     if (!CONTROL.test(text)) {
@@ -92,11 +119,10 @@ export function sanitizeTerminalText(text: string): string {
     let result = '';
     let index = 0;
     while (index < text.length) {
-        const char = text[index];
-        if (char === ESC) {
+        if (startsSequence(text[index])) {
             const kept = matchAt(SGR, text, index) ?? matchAt(OSC8, text, index);
             if (kept) {
-                result += kept;
+                result += toSevenBit(kept);
                 index += kept.length;
             } else {
                 index = skipEscapeSequence(text, index);
@@ -105,7 +131,7 @@ export function sanitizeTerminalText(text: string): string {
         }
 
         let end = index;
-        while (end < text.length && text[end] !== ESC) {
+        while (end < text.length && !startsSequence(text[end])) {
             end++;
         }
         result += text.slice(index, end).replace(CONTROL_GLOBAL, '');

src/widgets/__tests__/CustomCommand.test.ts

@@ -216,6 +216,13 @@ describe('CustomCommandWidget', () => {
             expect(output.endsWith(SGR_RESET)).toBe(true);
         });
 
+        it('closes an 8-bit colour the truncation cut off', () => {
+            const output = renderWith('\x9b38;5;208mfeature/login-refactor\x9b0m', { preserveColors: true, maxWidth: 20 }) ?? '';
+
+            expect(getVisibleText(output)).toBe('feature/login-ref...');
+            expect(output.endsWith(SGR_RESET)).toBe(true);
+        });
+
         it('never cuts preserveColors output in the middle of an escape sequence', () => {
             const output = renderWith(`${SGR_ORANGE}feature/login-refactor${SGR_RESET}`, { preserveColors: true, maxWidth: 10 }) ?? '';