diff --git a/Analyzer/AnalyzerTool.cs b/Analyzer/AnalyzerTool.cs index c2204b7..9e48ebd 100644 --- a/Analyzer/AnalyzerTool.cs +++ b/Analyzer/AnalyzerTool.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using Newtonsoft.Json; using UnityDataTools.Analyzer.SQLite.Handlers; using UnityDataTools.Analyzer.SQLite.Parsers; using UnityDataTools.Analyzer.SQLite.Writers; @@ -46,6 +45,9 @@ public class AnalyzeOptions public bool SkipCrc { get; init; } public bool Verbose { get; init; } public bool NoRecursion { get; init; } + // Build history folder (e.g. Library/BuildHistory); when set, the folder of the analyzed + // build is located in it and its ContentLayout.json and build report join the input. + public string BuildHistoryPath { get; init; } } public int Analyze(AnalyzeOptions options) @@ -183,44 +185,21 @@ public int Analyze(AnalyzeOptions options) } // Validates the ContentDirectory-related inputs and prepares the file list (issue #99): - // enforces that a single build is analyzed, selects the ContentLayout.json whose + // enforces that a single build is analyzed, adds the files of the matching build history + // folder when --build-history is used, selects the ContentLayout.json whose // BuildManifestHash matches that build (dropping any others), and moves it to the front of // the list so it is imported before the content files whose references it resolves. Returns // false, after printing an error, when the input combination is invalid. bool PrepareContentDirectoryInputs(List<(string FullPath, string DisplayRoot)> files) { - const string hashFileName = "BuildManifestHash.txt"; - - var layoutCandidates = files.Where(f => ContentLayoutParser.IsContentLayoutFile(f.FullPath)).ToList(); - var hashFiles = files - .Where(f => string.Equals(Path.GetFileName(f.FullPath), hashFileName, StringComparison.OrdinalIgnoreCase)) - .Select(f => f.FullPath) - .ToList(); - - // ContentDirectory output is recognized by its .cf content files, or by its archive when - // built compressed. The BuildManifestHash.txt identifying the build sits next to them; it - // is not always on the input (e.g. when specific files are passed), so pick it up from - // the directories containing the content. - var contentDirectories = files - .Where(f => HasExtension(f.FullPath, ".cf") || HasExtension(f.FullPath, ".archive")) - .Select(f => Path.GetDirectoryName(Path.GetFullPath(f.FullPath))) - .Distinct(StringComparer.OrdinalIgnoreCase); - - if (hashFiles.Count == 0) - { - hashFiles.AddRange(contentDirectories - .Select(dir => Path.Combine(dir, hashFileName)) - .Where(File.Exists)); - } - List buildHashes; try { - buildHashes = hashFiles.Select(f => File.ReadAllText(f).Trim()).Distinct().ToList(); + buildHashes = BuildHistoryHelper.FindBuildHashes(files.Select(f => f.FullPath)); } catch (Exception e) { - Console.Error.WriteLine($"Error reading {hashFileName}: {e.Message}"); + Console.Error.WriteLine($"Error reading {BuildHistoryHelper.HashFileName}: {e.Message}"); return false; } @@ -231,7 +210,14 @@ bool PrepareContentDirectoryInputs(List<(string FullPath, string DisplayRoot)> f } var buildHash = buildHashes.Count == 1 ? buildHashes[0] : null; - var hasContentDirectory = buildHash != null || files.Any(f => HasExtension(f.FullPath, ".cf")); + + if (m_Options.BuildHistoryPath != null && !AddBuildHistoryFiles(buildHash, files)) + { + return false; + } + + var layoutCandidates = files.Where(f => ContentLayoutParser.IsContentLayoutFile(f.FullPath)).ToList(); + var hasContentDirectory = buildHash != null || BuildHistoryHelper.HasContentFiles(files.Select(f => f.FullPath)); if (layoutCandidates.Count == 0) { @@ -259,7 +245,7 @@ bool PrepareContentDirectoryInputs(List<(string FullPath, string DisplayRoot)> f // The hash match guarantees the layout describes exactly this build; a stale or // unrelated layout would silently produce misleading results. - selected = layoutCandidates.FirstOrDefault(c => TryReadBuildManifestHash(c.FullPath) == buildHash); + selected = layoutCandidates.FirstOrDefault(c => BuildHistoryHelper.TryReadBuildManifestHash(c.FullPath) == buildHash); if (selected.FullPath == null) { @@ -282,7 +268,10 @@ bool PrepareContentDirectoryInputs(List<(string FullPath, string DisplayRoot)> f { if (candidate != selected) { - Console.Error.WriteLine($"Ignoring \"{candidate.FullPath}\": its BuildManifestHash does not match the analyzed build."); + var reason = BuildHistoryHelper.TryReadBuildManifestHash(candidate.FullPath) == buildHash + ? "it duplicates the selected layout" + : "its BuildManifestHash does not match the analyzed build"; + Console.Error.WriteLine($"Ignoring \"{candidate.FullPath}\": {reason}."); files.Remove(candidate); } } @@ -295,34 +284,48 @@ bool PrepareContentDirectoryInputs(List<(string FullPath, string DisplayRoot)> f return true; } - static bool HasExtension(string path, string extension) + // --build-history: locate the analyzed build's folder inside the build history and add its + // ContentLayout.json and build report to the input. Purely additive — the added layout goes + // through the same candidate validation as a manually passed one. + bool AddBuildHistoryFiles(string buildHash, List<(string FullPath, string DisplayRoot)> files) { - return string.Equals(Path.GetExtension(path), extension, StringComparison.OrdinalIgnoreCase); - } + var historyPath = m_Options.BuildHistoryPath; - // Reads the top-level BuildManifestHash of a ContentLayout.json without parsing the whole - // file (layouts of large builds are big; the hash is one of the first properties). Returns - // null when the value cannot be found or the file is not valid json. - static string TryReadBuildManifestHash(string path) - { - try + if (!Directory.Exists(historyPath)) { - using var reader = new JsonTextReader(File.OpenText(path)); + Console.Error.WriteLine($"--build-history path not found: {historyPath}"); + return false; + } - for (int i = 0; i < 64 && reader.Read(); ++i) - { - if (reader.TokenType == JsonToken.PropertyName && reader.Depth == 1 && - "BuildManifestHash".Equals(reader.Value)) - { - return reader.ReadAsString(); - } - } + if (buildHash == null) + { + Console.Error.WriteLine("--build-history requires a ContentDirectory build in the input: no BuildManifestHash.txt was found to identify the build to match."); + return false; } - catch (Exception) + + var buildFolder = BuildHistoryHelper.LocateBuildFolder(historyPath, buildHash); + if (buildFolder == null) { + Console.Error.WriteLine($"No build in \"{historyPath}\" matches the analyzed build (BuildManifestHash {buildHash})."); + return false; } - return null; + Console.WriteLine($"Using build history folder \"{buildFolder}\"."); + + // Skip files that are already on the input (e.g. the layout was also passed explicitly). + var newEntries = new List<(string FullPath, string DisplayRoot)>(); + foreach (var file in BuildHistoryHelper.CollectBuildFiles(buildFolder)) + { + var alreadyIncluded = files.Any(f => string.Equals( + Path.GetFullPath(f.FullPath), Path.GetFullPath(file), StringComparison.OrdinalIgnoreCase)); + + if (!alreadyIncluded) + newEntries.Add((file, buildFolder)); + } + + files.InsertRange(0, newEntries); + + return true; } // Expands the input paths into the concrete files to analyze. Each result pairs the file with the diff --git a/Analyzer/Util/BuildHistoryHelper.cs b/Analyzer/Util/BuildHistoryHelper.cs new file mode 100644 index 0000000..11f57d8 --- /dev/null +++ b/Analyzer/Util/BuildHistoryHelper.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using UnityDataTools.Models; + +namespace UnityDataTools.Analyzer.Util; + +// Identifies the ContentDirectory build present in the analyze input and locates its folder +// inside a build history (Library/BuildHistory, one folder per build). The folder of a build is +// found by matching the BuildManifestHash of its ContentLayout.json against the build's +// BuildManifestHash.txt. Pure file lookup; no knowledge of the parsers or the database. +public static class BuildHistoryHelper +{ + public const string HashFileName = "BuildManifestHash.txt"; + public const string LayoutFileName = "ContentLayout.json"; + public const string SummaryFileName = "BuildReportSummary.json"; + + // Reads the BuildManifestHash.txt values identifying the ContentDirectory build(s) in the + // input files: taken from the input itself when present, otherwise picked up next to the + // .cf/.archive content files (the hash file is not always on the input, e.g. when specific + // files are passed). More than one distinct hash means more than one build. + public static List FindBuildHashes(IEnumerable files) + { + var fileList = files.ToList(); + + var hashFiles = fileList + .Where(f => string.Equals(Path.GetFileName(f), HashFileName, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (hashFiles.Count == 0) + { + var contentDirectories = fileList + .Where(f => HasExtension(f, ".cf") || HasExtension(f, ".archive")) + .Select(f => Path.GetDirectoryName(Path.GetFullPath(f))) + .Distinct(StringComparer.OrdinalIgnoreCase); + + hashFiles.AddRange(contentDirectories + .Select(dir => Path.Combine(dir, HashFileName)) + .Where(File.Exists)); + } + + // Hashes are lowercase hex; normalizing here (and in TryReadBuildManifestHash) keeps all + // hash comparisons simple, exact matches. + return hashFiles.Select(f => File.ReadAllText(f).Trim().ToLowerInvariant()).Distinct().ToList(); + } + + // Fallback signal used when no BuildManifestHash.txt is available: a .cf file marks + // ContentDirectory output. This is only a rough sanity check — the reverse does not hold, as + // content files can be built without the extension or sit inside archives. + public static bool HasContentFiles(IEnumerable files) + { + return files.Any(f => HasExtension(f, ".cf")); + } + + // Searches the history root and its direct child folders for the ContentLayout.json whose + // BuildManifestHash matches the analyzed build. A build history is flat, so there is no + // recursion — which also keeps the scan bounded if a large unrelated folder is passed. When + // several folders match (rebuilds of identical content), the most recent build wins. + // Returns the containing folder, or null when nothing matches. + public static string LocateBuildFolder(string historyRoot, string buildHash) + { + var matches = Directory.EnumerateDirectories(historyRoot) + .Prepend(historyRoot) + .Where(dir => File.Exists(Path.Combine(dir, LayoutFileName))) + .Where(dir => TryReadBuildManifestHash(Path.Combine(dir, LayoutFileName)) == buildHash); + + string selected = null; + var selectedTime = default(DateTime); + foreach (var folder in matches) + { + var startTime = TryReadBuildStartTime(folder); + if (selected == null || startTime > selectedTime) + { + selected = folder; + selectedTime = startTime; + } + } + + return selected; + } + + // Reads the start time of the build from the BuildReportSummary.json of a build history + // folder. The file is expected to always be present; returns a zero timestamp when it is + // missing or unreadable. + public static DateTime TryReadBuildStartTime(string buildFolder) + { + try + { + var json = File.ReadAllText(Path.Combine(buildFolder, SummaryFileName)); + var summary = JsonConvert.DeserializeObject(json); + return summary?.BuildStartedAt ?? default; + } + catch (Exception) + { + return default; + } + } + + // The files of a build history folder that analyze imports: the layout and the build report. + public static List CollectBuildFiles(string buildFolder) + { + var files = new List { Path.Combine(buildFolder, LayoutFileName) }; + files.AddRange(Directory.EnumerateFiles(buildFolder, "*.buildreport")); + return files; + } + + // Reads the top-level BuildManifestHash of a ContentLayout.json without parsing the whole + // file (layouts of large builds are big; the hash is one of the first properties). Returns + // null when the value cannot be found or the file is not valid json. + public static string TryReadBuildManifestHash(string contentLayoutPath) + { + try + { + using var reader = new JsonTextReader(File.OpenText(contentLayoutPath)); + + for (int i = 0; i < 64 && reader.Read(); ++i) + { + if (reader.TokenType == JsonToken.PropertyName && reader.Depth == 1 && + "BuildManifestHash".Equals(reader.Value)) + { + return reader.ReadAsString()?.ToLowerInvariant(); + } + } + } + catch (Exception) + { + } + + return null; + } + + static bool HasExtension(string path, string extension) + { + return string.Equals(Path.GetExtension(path), extension, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/Documentation/command-analyze.md b/Documentation/command-analyze.md index c15a8ee..3aaf3fe 100644 --- a/Documentation/command-analyze.md +++ b/Documentation/command-analyze.md @@ -18,6 +18,7 @@ UnityDataTool analyze ... [options] | `-v, --verbose` | Show more information during analysis | `false` | | `--no-recurse` | Do not recurse into sub-directories when scanning directories | `false` | | `-d, --typetree-data ` | Load an external TypeTree data file before processing (Unity 6.5+) | — | +| `--build-history ` | Build history folder of the project (e.g. `Library/BuildHistory`). The build folder matching the analyzed build is located automatically and its `ContentLayout.json` and build report are included in the analysis. See [ContentDirectory builds](#contentdirectory-builds) | — | There is no way to append to an existing database, so every file you want in the results must be included in a single `analyze` invocation. Pass multiple paths to combine files from more than one @@ -40,6 +41,12 @@ Combine a build output directory with a build report file kept in a separate loc UnityDataTool analyze /path/to/build/output /path/to/Library/LastBuild.buildreport ``` +Analyze a ContentDirectory build together with its build history (the matching layout and build +report are located automatically): +```bash +UnityDataTool analyze /path/to/ContentDirectory --build-history /path/to/Library/BuildHistory +``` + Analyze only `.bundle` files and specify a custom database name: ```bash UnityDataTool analyze /path/to/asset/bundles -o my_database.db -p "*.bundle" @@ -79,7 +86,16 @@ command works with the following types of input: ## ContentDirectory builds Analyze a ContentDirectory build together with its `ContentLayout.json` by passing the build output -folder and the build report folder (or the layout file itself) in one call: +folder and the project's build history with `--build-history`: + +```bash +UnityDataTool analyze /path/to/ContentDirectory --build-history /path/to/Library/BuildHistory -o database.db +``` + +Analyze locates the build's own folder inside the build history and includes its +`ContentLayout.json` and `.buildreport` in the analysis, so there is no need to identify the +correct build folder by hand. Alternatively, pass the build folder (or the layout file itself) as a +regular input: ```bash UnityDataTool analyze /path/to/ContentDirectory /path/to/Library/BuildHistory/ -o database.db @@ -94,8 +110,10 @@ resolved and are recorded in `dangling_refs`. A `ContentLayout.json` is only used when its `BuildManifestHash` matches the analyzed build's `BuildManifestHash.txt` (a file guaranteed to exist in the build output folder, which analyze picks up automatically). This exact-match rule prevents a stale or unrelated layout from silently -producing misleading results, and it means you can point analyze at a folder containing several -layouts (such as `Library/BuildHistory`) and the matching one is selected. +producing misleading results. It is also how `--build-history` finds the build's folder: the folder +whose `ContentLayout.json` matches the hash is used (the most recent one, if the same content was +built several times). `--build-history` does not search recursively — the build folders must be +direct children of the given path (or the path may be a build folder itself). How the input combinations behave: @@ -104,6 +122,8 @@ How the input combinations behave: | ContentDirectory only | Analyzed, with a warning that the analysis is incomplete (broken references, no source mapping) | | Subset of files from a ContentDirectory | Same as above; recognized by the `.cf` extension | | ContentDirectory + matching ContentLayout.json | Analyzed with resolved references and source mapping | +| ContentDirectory + `--build-history` | The build's folder is located in the history; its layout and build report are analyzed too. Error when no folder matches | +| `--build-history` without a ContentDirectory build in the input | Error — only ContentDirectory builds can be matched against a build history | | Subset of ContentDirectory files + ContentLayout.json | Works when a `BuildManifestHash.txt` can be found for the hash match; error when the layout cannot be validated | | ContentDirectory + multiple ContentLayout.json | The one matching the build is used, the rest are ignored; error when none match | | ContentLayout.json only | Imported on its own (layout tables only) | diff --git a/Documentation/contentdirectory-format.md b/Documentation/contentdirectory-format.md index 6a393f9..147c1fa 100644 --- a/Documentation/contentdirectory-format.md +++ b/Documentation/contentdirectory-format.md @@ -204,10 +204,13 @@ is not distributed with your application. For the full picture, see the Manual t [Analyze builds](https://docs.unity3d.com/6000.6/Documentation/Manual/build-analyze-builds.html). Most of the files in a build history directory (the Trace Event Profile, `BuildLog.jsonl`, -`ScriptsOnlyCache.yaml`, `ContentSizeSummary.txt`, `BuildReportSummary.json`, and others) are outside +`ScriptsOnlyCache.yaml`, `ContentSizeSummary.txt`, and others) are outside the scope of UnityDataTool and are documented in the Manual's [Build history file reference](https://docs.unity3d.com/6000.6/Documentation/Manual/build-history-file-reference.html). -Two files in the build history are directly relevant here: +`BuildReportSummary.json` is not imported either, but UnityDataTool reads its build start time to +pick the most recent build when several build history folders match (and +[`UnityDataModels`](../UnityDataModels/BuildReportSummary.cs) provides a convenience type for +reading it in your own code). Two files in the build history are directly relevant here: * **[`ContentLayout.json`](contentlayout.md)** — maps the built content back to the source assets in the project and describes the dependencies between the produced files. It is the key file for @@ -399,31 +402,34 @@ The `dump` and `serialized-file` commands can be used to inspect serialized file If the content is distributed inside archive files (e.g. `content0.archive`) then the `archive` command can be used to view or extract the content. -When you run [`analyze`](command-analyze.md) on a content directory build, analyze the **build output -folder and its matching build history folder together**, in a single `analyze` call, by passing both -paths: +When you run [`analyze`](command-analyze.md) on a content directory build, analyze the **build +output together with its build history** by passing the build history folder with `--build-history`: ```bash -UnityDataTool analyze /path/to/ContentDirectory /path/to/Library/BuildHistory/ +UnityDataTool analyze /path/to/ContentDirectory --build-history /path/to/Library/BuildHistory ``` +Analyze locates the analyzed build's own directory inside the build history — by matching the +`BuildManifestHash` of each `ContentLayout.json` against the build's `BuildManifestHash.txt` — and +includes that directory's `ContentLayout.json` and `.buildreport` in the analysis. A stale layout +from a different build can never be picked up, and there is no need to identify the correct +build directory by hand. + Analyzing the build output alone records the objects, but not where they came from — and because the references between Content Files live in the manifest rather than in the files themselves (see [References between Content Files](#references-between-content-files) above), those references end up -in the `dangling_refs` table and analyze prints a warning. Including the build history folder fixes -both: the `ContentLayout.json` it contains provides the source-asset mapping (imported as the +in the `dangling_refs` table and analyze prints a warning. Including the build history fixes +both: the `ContentLayout.json` provides the source-asset mapping (imported as the `content_layout` tables, see [ContentLayout in the Analyze Database](contentlayout-database.md)) and the dependency information that analyze uses to resolve the references between Content Files. -Analyze verifies that the `ContentLayout.json` matches the build through the build's -`BuildManifestHash.txt`, so a stale layout from a different build is rejected rather than producing -misleading results. See the [`analyze` command](command-analyze.md#contentdirectory-builds) page for -the exact input combinations. - -Note: Passing in the entire build report folder in the build history means that the .buildreport file -will also be analyzed. That brings in statistics about the build (how long it took etc). For -content directory builds there is no PackedAssets data, but other build_report_* tables will be populated. -Call `analyze` with the path of the correct ContentLayout.json file, instead of the entire build report folder, if you do not need this extra data. +The `.buildreport` brings in statistics about the build (how long it took etc). For content +directory builds there is no PackedAssets data, but other build_report_* tables will be populated. +If you do not need this extra data, pass the path of the correct `ContentLayout.json` file as a +regular input instead of using `--build-history`; the same `BuildManifestHash.txt` verification +applies, so a layout that does not match the build is rejected rather than producing misleading +results. See the [`analyze` command](command-analyze.md#contentdirectory-builds) page for the exact +input combinations. ## Related documentation diff --git a/UnityDataModels/BuildReportSummary.cs b/UnityDataModels/BuildReportSummary.cs new file mode 100644 index 0000000..a21de28 --- /dev/null +++ b/UnityDataModels/BuildReportSummary.cs @@ -0,0 +1,38 @@ +// This file defines the structure of the BuildReportSummary.json file written into each build +// history folder (default location Library/BuildHistory) by Player and content directory builds +// (available starting in Unity 6.6). +// +// It is a convenience type for reading that data outside of the Unity Editor. The fields mirror +// the Editor's BuildReportSummary; see +// https://docs.unity3d.com/6000.6/Documentation/ScriptReference/Build.BuildReportSummary.html +// for their documentation. + +using System; + +namespace UnityDataTools.Models +{ + public class BuildReportSummary + { + public int Version; + public string[] BuildContentOptions; + public string BuildManifestHash; + public string BuildName; + public string[] BuildOptions; + public string BuildProfilePath; + public int BuildResult; + public string BuildResultName; + public string BuildSessionGUID; + public DateTime BuildStartedAt; + public int BuildType; + public string BuildTypeName; + public string OutputPath; + public int Platform; + public string PlatformName; + public int Subtarget; + public string SubtargetName; + public int TotalErrors; + public long TotalSizeBytes; + public long TotalTimeMs; + public int TotalWarnings; + } +} diff --git a/UnityDataTool.Tests/AnalyzeBuildHistoryTests.cs b/UnityDataTool.Tests/AnalyzeBuildHistoryTests.cs new file mode 100644 index 0000000..9ba0137 --- /dev/null +++ b/UnityDataTool.Tests/AnalyzeBuildHistoryTests.cs @@ -0,0 +1,271 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using NUnit.Framework; + +namespace UnityDataTools.UnityDataTool.Tests; + +#pragma warning disable NUnit2005, NUnit2006 + +// Tests the --build-history option of analyze (issue #99): the folder of the analyzed build is +// located inside a build history (Library/BuildHistory) by matching the BuildManifestHash of its +// ContentLayout.json, and that folder's layout and build report join the analysis. Build history +// folders are assembled per test from the BuildReport-ContentDirectory fixture of the LeadingEdge +// reference build. +public class AnalyzeBuildHistoryTests +{ + private const string BuildHash = "baff06b928d147276f2245dd3b19216a"; + + private string m_TestOutputFolder; + private string m_ContentDirectory; + private string m_FixtureReportFolder; + + [OneTimeSetUp] + public void OneTimeSetup() + { + m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "build_history_test_folder"); + m_ContentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + m_FixtureReportFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "BuildReport-ContentDirectory"); + Directory.CreateDirectory(m_TestOutputFolder); + Directory.SetCurrentDirectory(m_TestOutputFolder); + } + + [TearDown] + public void Teardown() + { + SqliteConnection.ClearAllPools(); + var outputFolder = new DirectoryInfo(m_TestOutputFolder); + outputFolder.EnumerateFiles().ToList().ForEach(f => f.Delete()); + outputFolder.EnumerateDirectories().ToList().ForEach(d => d.Delete(true)); + } + + // Creates a build folder holding the reference build's ContentLayout.json, .buildreport and + // BuildReportSummary.json. + private string CreateMatchingBuildFolder(string historyRoot, string name) + { + var folder = Path.Combine(historyRoot, name); + Directory.CreateDirectory(folder); + File.Copy(Path.Combine(m_FixtureReportFolder, "ContentLayout.json"), + Path.Combine(folder, "ContentLayout.json")); + File.Copy(Path.Combine(m_FixtureReportFolder, "BuildReportSummary.json"), + Path.Combine(folder, "BuildReportSummary.json")); + foreach (var report in Directory.EnumerateFiles(m_FixtureReportFolder, "*.buildreport")) + File.Copy(report, Path.Combine(folder, Path.GetFileName(report))); + return folder; + } + + // Creates a build folder whose layout belongs to some other build. + private string CreateStaleBuildFolder(string historyRoot, string name) + { + var folder = Path.Combine(historyRoot, name); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "ContentLayout.json"), + "{\"Version\":2,\"BuildManifestHash\":\"deadbeefdeadbeefdeadbeefdeadbeef\"}"); + return folder; + } + + private static async Task<(int exitCode, string stdErr)> RunCapturingStdErr(params string[] args) + { + using var sw = new StringWriter(); + var currentError = Console.Error; + int exitCode; + try + { + Console.SetError(sw); + exitCode = await Program.Main(args); + } + finally + { + Console.SetError(currentError); + } + + return (exitCode, sw.ToString()); + } + + private static async Task<(int exitCode, string stdOut)> RunCapturingStdOut(params string[] args) + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + int exitCode; + try + { + Console.SetOut(sw); + exitCode = await Program.Main(args); + } + finally + { + Console.SetOut(currentOut); + } + + return (exitCode, sw.ToString()); + } + + [Test] + public async Task Analyze_WithBuildHistory_ImportsLayoutAndBuildReport() + { + var history = Path.Combine(m_TestOutputFolder, "history"); + CreateStaleBuildFolder(history, "Build-Stale"); + CreateMatchingBuildFolder(history, "Build-Match"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] + { "analyze", m_ContentDirectory, "--build-history", history, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryString(db, "SELECT build_manifest_hash FROM content_layout", + BuildHash, "the matching layout should be imported"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM build_reports", 1, + "the build report of the matched folder should be imported"); + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + INNER JOIN serialized_files sf ON sf.id = d.serialized_file + WHERE sf.name LIKE '%.cfid'", 0, + "the layout should resolve the content file references"); + } + + [Test] + public async Task Analyze_BuildHistoryPointedAtBuildFolder_Works() + { + // Pointing --build-history directly at the per-build folder (instead of its parent) is + // accepted: the folder itself is checked as well as its children. + var history = Path.Combine(m_TestOutputFolder, "history"); + var buildFolder = CreateMatchingBuildFolder(history, "Build-Match"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] + { "analyze", m_ContentDirectory, "--build-history", buildFolder, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryString(db, "SELECT build_manifest_hash FROM content_layout", + BuildHash, "the layout of the folder itself should be found"); + } + + [Test] + public async Task Analyze_BuildHistoryWithoutMatch_Fails() + { + var history = Path.Combine(m_TestOutputFolder, "history"); + CreateStaleBuildFolder(history, "Build-Stale"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", m_ContentDirectory, "--build-history", history, "-o", databasePath); + + Assert.AreEqual(1, exitCode, "a build history without the analyzed build must fail"); + Assert.That(stdErr, Does.Contain("matches the analyzed build")); + Assert.IsFalse(File.Exists(databasePath), "no partial database should be left behind"); + } + + [Test] + public async Task Analyze_BuildHistoryWithoutContentDirectoryInput_Fails() + { + // Only ContentDirectory builds can be matched against a build history (via their + // BuildManifestHash.txt); for other content the option is an error. + var history = Path.Combine(m_TestOutputFolder, "history"); + CreateMatchingBuildFolder(history, "Build-Match"); + var bundlePath = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "AssetBundles", "assetbundleroot"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", bundlePath, "--build-history", history, "-o", databasePath); + + Assert.AreEqual(1, exitCode); + Assert.That(stdErr, Does.Contain("requires a ContentDirectory build")); + Assert.IsFalse(File.Exists(databasePath), "no partial database should be left behind"); + } + + [Test] + public async Task Analyze_NonexistentBuildHistoryPath_Fails() + { + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var exitCode = await Program.Main(new string[] + { + "analyze", m_ContentDirectory, + "--build-history", Path.Combine(m_TestOutputFolder, "does_not_exist"), + "-o", databasePath + }); + + Assert.AreNotEqual(0, exitCode, "a nonexistent build history path must fail"); + Assert.IsFalse(File.Exists(databasePath), "no partial database should be left behind"); + } + + // Overwrites the BuildReportSummary.json of a build folder with the given build start time. + private static void SetBuildStartTime(string buildFolder, string buildStartedAt) + { + File.WriteAllText(Path.Combine(buildFolder, "BuildReportSummary.json"), + $"{{\"Version\":2,\"BuildStartedAt\":\"{buildStartedAt}\"}}"); + } + + [Test] + public async Task Analyze_MultipleMatchingBuilds_UsesMostRecent() + { + // Rebuilding identical content produces several history folders with the same + // BuildManifestHash; the one with the latest BuildStartedAt (from + // BuildReportSummary.json) wins. The older build sorts first alphabetically, so a + // "first found wins" regression would pick it instead. + // A folder without a BuildReportSummary.json gets a zero timestamp and loses. + var history = Path.Combine(m_TestOutputFolder, "history"); + var older = CreateMatchingBuildFolder(history, "Build-A-Older"); + var newer = CreateMatchingBuildFolder(history, "Build-B-Newer"); + var noSummary = CreateMatchingBuildFolder(history, "Build-C-NoSummary"); + SetBuildStartTime(older, "2026-07-14T18:37:14.2095142Z"); + SetBuildStartTime(newer, "2026-07-22T09:00:00.0000000Z"); + File.Delete(Path.Combine(noSummary, "BuildReportSummary.json")); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdOut) = await RunCapturingStdOut( + "analyze", m_ContentDirectory, "--build-history", history, "-o", databasePath); + + Assert.AreEqual(0, exitCode); + Assert.That(stdOut, Does.Contain("Build-B-Newer"), "the selected folder should be reported"); + + using var db = SQLTestHelper.OpenDatabase(databasePath); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout", 1, + "a single layout should be imported"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM build_reports", 1, + "only the selected folder's build report should be imported"); + } + + [Test] + public async Task Analyze_BuildHistoryAndSameLayoutPassedPositionally_UsesOneLayout() + { + var history = Path.Combine(m_TestOutputFolder, "history"); + CreateMatchingBuildFolder(history, "Build-Match"); + var positionalLayout = Path.Combine(m_FixtureReportFolder, "ContentLayout.json"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", m_ContentDirectory, positionalLayout, "--build-history", history, "-o", databasePath); + + Assert.AreEqual(0, exitCode); + Assert.That(stdErr, Does.Contain("duplicates the selected layout"), + "the redundant copy should be reported as a duplicate, not as a mismatch"); + + using var db = SQLTestHelper.OpenDatabase(databasePath); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout", 1, + "the layout should be imported once"); + } + + [Test] + public async Task Analyze_ArchiveBuildWithBuildHistory_MatchesThroughArchiveHashFile() + { + // The compressed reference build is a different build, so the LeadingEdge history cannot + // match — proving the hash discovery next to .archive files feeds the history lookup. + var history = Path.Combine(m_TestOutputFolder, "history"); + CreateMatchingBuildFolder(history, "Build-Match"); + var archiveBuild = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "contentdirectory-zstd"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", archiveBuild, "--build-history", history, "-o", databasePath); + + Assert.AreEqual(1, exitCode); + Assert.That(stdErr, Does.Contain("matches the analyzed build")); + } +} diff --git a/UnityDataTool/Program.cs b/UnityDataTool/Program.cs index 00e199e..2fdffbf 100644 --- a/UnityDataTool/Program.cs +++ b/UnityDataTool/Program.cs @@ -76,6 +76,10 @@ static Command BuildAnalyzeCommand() var vOpt = new Option(aliases: new[] { "--verbose", "-v" }, description: "Verbose output"); var recurseOpt = new Option(aliases: new[] { "--no-recurse" }, description: "Do not analyze contents of subdirectories inside scanned directories"); var dOpt = new Option(aliases: new[] { "--typetree-data", "-d" }, description: TypeTreeDataDescription); + var bhOpt = new Option(aliases: new[] { "--build-history" }, + description: "Build history folder of the project (e.g. Library/BuildHistory). The build folder matching the " + + "analyzed build is located automatically and its ContentLayout.json and build report are included in the analysis.") + .ExistingOnly(); var analyzeCommand = new Command("analyze", "Analyze AssetBundles, SerializedFiles and build reports into a database.") { @@ -87,7 +91,8 @@ static Command BuildAnalyzeCommand() pOpt, vOpt, recurseOpt, - dOpt + dOpt, + bhOpt }; analyzeCommand.AddAlias("analyse"); @@ -111,7 +116,8 @@ static Command BuildAnalyzeCommand() context.ParseResult.GetValueForOption(rOpt), context.ParseResult.GetValueForOption(pOpt), context.ParseResult.GetValueForOption(vOpt), - context.ParseResult.GetValueForOption(recurseOpt)); + context.ParseResult.GetValueForOption(recurseOpt), + context.ParseResult.GetValueForOption(bhOpt)); }); return analyzeCommand; @@ -354,7 +360,8 @@ static int HandleAnalyze( bool extractReferences, string searchPattern, bool verbose, - bool noRecurse) + bool noRecurse, + DirectoryInfo buildHistory) { var analyzer = new AnalyzerTool(); @@ -372,6 +379,7 @@ static int HandleAnalyze( SkipCrc = skipCrc, Verbose = verbose, NoRecursion = noRecurse, + BuildHistoryPath = buildHistory?.FullName, }); }