feat: interactively review failed snapshots (#11325)

Co-authored-by: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com>

Author: macarie

.gitattributes

@@ -1,3 +1,4 @@
 * text=auto eol=lf
 
 test/e2e/fixtures/reporters/indicator-position.test.js eol=crlf
+test/e2e/test/watch/raw/snapshot-review.diff -text

packages/browser/src/client/tester/runner.ts

@@ -189,8 +189,10 @@ function createBrowserRunner(
         task.type === 'test' &&
         !(
           lastErrorContext &&
-          Reflect.get(lastErrorContext, 'assertionName') === 'toMatchScreenshot' &&
-          Reflect.get(lastErrorContext, 'meta')?.outcome !== 'unstable-screenshot'
+          lastErrorContext.assertionName === 'toMatchScreenshot' &&
+          lastErrorContext.meta &&
+          'outcome' in lastErrorContext.meta &&
+          lastErrorContext.meta.outcome !== 'unstable-screenshot'
         )
       ) {
         const { retry, repeats } = this.testAttemptIndexMap.get(task.id) ?? { retry: 0, repeats: 0 }

packages/snapshot/src/client.ts

@@ -43,7 +43,7 @@ interface AssertOptions {
   error?: Error
   errorMessage?: string
   rawSnapshot?: RawSnapshotInfo
-  assertionName?: string
+  assertionName: string
 }
 
 interface AssertDomainOptions extends Omit<AssertOptions, 'received'> {
@@ -63,8 +63,19 @@ export interface MatchResult {
   message: () => string
   actual?: unknown
   expected?: unknown
+  /** @internal */
+  meta?: MatchMeta
 }
 
+/**
+ * Information about the kind of processed snapshot and data useful for constructing an update
+ */
+export type MatchMeta = { type: 'snapshot'; assertionName: string; key: string } & (
+  | { kind: 'file'; file: string; renderedContent?: string }
+  | { kind: 'inline'; renderedContent?: string }
+  | { kind: 'raw'; file: string }
+)
+
 interface SnapshotClientOptions {
   isEqual?: (received: unknown, expected: unknown) => boolean
 }
@@ -167,6 +178,12 @@ export class SnapshotClient {
           message: () => errorMessage || 'Snapshot properties mismatched',
           actual: received,
           expected: properties,
+          meta: createMetaObject({
+            assertionName,
+            isInline,
+            rawSnapshot,
+            snapshotState,
+          }),
         }
       }
       received = deepMergeSnapshot(received, properties)
@@ -188,6 +205,13 @@ export class SnapshotClient {
       message: () => `Snapshot \`${key || 'unknown'}\` mismatched`,
       actual: rawSnapshot ? actual : actual?.trim(),
       expected: rawSnapshot ? expected : expected?.trim(),
+      meta: createMetaObject({
+        assertionName,
+        isInline,
+        key,
+        rawSnapshot,
+        snapshotState,
+      }),
     }
   }
 
@@ -215,6 +239,7 @@ export class SnapshotClient {
       isInline = false,
       inlineSnapshot,
       error,
+      assertionName,
     } = options
 
     if (!filepath) {
@@ -245,14 +270,21 @@ export class SnapshotClient {
       matchResult,
       isInline,
       error,
-      assertionName: options.assertionName,
+      assertionName,
     })
 
     return {
       pass,
       message: () => `Snapshot \`${key}\` mismatched`,
       actual: actual?.trim(),
       expected: expected?.trim(),
+      meta: createMetaObject({
+        rendered,
+        assertionName,
+        isInline,
+        key,
+        snapshotState,
+      }),
     }
   }
 
@@ -269,6 +301,7 @@ export class SnapshotClient {
       error,
       timeout = 1000,
       interval = 50,
+      assertionName,
     } = options
 
     if (!filepath) {
@@ -335,14 +368,21 @@ export class SnapshotClient {
       matchResult,
       isInline,
       error,
-      assertionName: options.assertionName,
+      assertionName,
     })
 
     return {
       pass,
       message: () => `Snapshot \`${key}\` mismatched`,
       actual: actual?.trim(),
       expected: expected?.trim(),
+      meta: createMetaObject({
+        rendered: stableResult.rendered,
+        assertionName,
+        isInline,
+        key,
+        snapshotState,
+      }),
     }
   }
 
@@ -446,3 +486,52 @@ function raceWith<A, B>(
   }
   return Promise.race([left, other.then((value) => ({ ok: false as const, value }))])
 }
+
+interface CreateMetaOptions extends Pick<
+  AssertOptions,
+  'assertionName' | 'isInline' | 'rawSnapshot'
+> {
+  rendered?: string
+  snapshotState: SnapshotState
+  key?: string
+}
+
+function createMetaObject({
+  rendered,
+  assertionName,
+  isInline,
+  key,
+  rawSnapshot,
+  snapshotState,
+}: CreateMetaOptions): MatchMeta {
+  const k = key ?? 'unknown'
+
+  if (isInline) {
+    return {
+      type: 'snapshot',
+      kind: 'inline',
+      assertionName,
+      key: k,
+      renderedContent: rendered,
+    }
+  }
+
+  if (rawSnapshot) {
+    return {
+      type: 'snapshot',
+      kind: 'raw',
+      assertionName,
+      file: rawSnapshot.file,
+      key: k,
+    }
+  }
+
+  return {
+    type: 'snapshot',
+    kind: 'file',
+    assertionName,
+    file: snapshotState.snapshotPath,
+    key: k,
+    renderedContent: rendered,
+  }
+}

packages/snapshot/src/index.ts

@@ -1,5 +1,5 @@
 export { SnapshotClient } from './client'
-export type { MatchResult } from './client'
+export type { MatchResult, MatchMeta } from './client'
 
 export type { DomainMatchResult, DomainSnapshotAdapter } from './domain'
 
@@ -18,3 +18,6 @@ export type {
   SnapshotUpdateState,
   UncheckedSnapshot,
 } from './types'
+
+export { saveSnapshotUpdates } from './update'
+export type { SnapshotUpdate } from './update'

packages/snapshot/src/port/rawSnapshot.ts

@@ -8,7 +8,6 @@ export interface RawSnapshotInfo {
 
 export interface RawSnapshot extends RawSnapshotInfo {
   snapshot: string
-  file: string
 }
 
 export async function saveRawSnapshots(

packages/snapshot/src/update.ts

@@ -0,0 +1,79 @@
+import type { InlineSnapshot } from './port/inlineSnapshot'
+import type { RawSnapshot } from './port/rawSnapshot'
+import type { SnapshotData, SnapshotEnvironment } from './types'
+import { saveInlineSnapshots } from './port/inlineSnapshot'
+import { saveRawSnapshots } from './port/rawSnapshot'
+import { evaluateSnapshotFile, saveSnapshotFile } from './port/utils'
+
+interface InlineSnapshotUpdate extends InlineSnapshot {
+  type: 'inline'
+}
+
+interface FileSnapshotUpdate {
+  type: 'file'
+  file: string
+  key: string
+  snapshot: string
+}
+
+interface RawSnapshotUpdate extends RawSnapshot {
+  type: 'raw'
+}
+
+export type SnapshotUpdate = InlineSnapshotUpdate | FileSnapshotUpdate | RawSnapshotUpdate
+
+export async function saveSnapshotUpdates(
+  snapshots: Map<SnapshotEnvironment, readonly SnapshotUpdate[]>,
+): Promise<void> {
+  const updatesByFile = new Map<string, SnapshotUpdate[]>()
+  const saves: Promise<void>[] = []
+
+  for (const [environment, updates] of snapshots) {
+    for (const update of updates) {
+      const fileUpdates = updatesByFile.get(update.file) ?? []
+
+      if (fileUpdates.length === 0) {
+        updatesByFile.set(update.file, fileUpdates)
+      }
+
+      fileUpdates.push(update)
+    }
+
+    for (const [file, fileUpdates] of updatesByFile) {
+      const updates = Object.groupBy(fileUpdates.values(), (snapshot) => snapshot.type) as {
+        inline?: InlineSnapshotUpdate[]
+        file?: FileSnapshotUpdate[]
+        raw?: RawSnapshotUpdate[]
+      }
+
+      if (updates.inline) {
+        saves.push(saveInlineSnapshots(environment, updates.inline))
+      }
+
+      if (updates.file) {
+        let data: SnapshotData | null
+
+        if (environment.readSnapshotFileData) {
+          data = await environment.readSnapshotFileData(file)
+        } else {
+          const content = await environment.readSnapshotFile(file)
+          data = content == null ? null : evaluateSnapshotFile(file, content)
+        }
+
+        const snapshotData: SnapshotData = Object.assign(Object.create(null), data)
+
+        for (const snapshot of updates.file) {
+          snapshotData[snapshot.key] = snapshot.snapshot
+        }
+
+        saves.push(saveSnapshotFile(environment, snapshotData, file))
+      }
+
+      if (updates.raw) {
+        saves.push(saveRawSnapshots(environment, updates.raw))
+      }
+    }
+  }
+
+  await Promise.all(saves)
+}

packages/utils/src/types.ts

@@ -40,4 +40,9 @@ export interface TestError extends SerializedError {
   diff?: string
   actual?: string
   expected?: string
+  /** @internal */
+  __vitest_error_context__?: {
+    assertionName?: string
+    meta?: object
+  }
 }

packages/vitest/src/integrations/chai/poll.ts

@@ -3,6 +3,7 @@ import type { Test } from '../../runtime/runner/types'
 import { chai } from '@vitest/expect'
 import { delay, getSafeTimers } from '@vitest/utils/timers'
 import { getWorkerState } from '../../runtime/utils'
+import { copyStackTrace } from '../../utils/errors'
 import { vi } from '../vi'
 
 // these matchers are not supported because they don't make sense with poll
@@ -244,13 +245,6 @@ export function createExpectPoll(expect: ExpectStatic): ExpectStatic['poll'] {
   }
 }
 
-function copyStackTrace(target: Error, source: Error) {
-  if (source.stack !== undefined) {
-    target.stack = source.stack.replace(source.message, target.message)
-  }
-  return target
-}
-
 function raceWith<A, B>(
   promise: Promise<A>,
   other?: Promise<B>,