Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ $(CommonProductDescription)]]></PackageDescription>
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportJournalConfiguration.cs" Link="Helpers\ReportJournalConfiguration.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportJournalRecord.cs" Link="Helpers\ReportJournalRecord.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportProcessLifetimeHandler.cs" Link="Helpers\ReportProcessLifetimeHandler.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\RetryArtifactManifest.cs" Link="Helpers\RetryArtifactManifest.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportProcessLifetimeHandler.Journal.cs" Link="Helpers\ReportProcessLifetimeHandler.Journal.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportGeneratorCommandLineBase.cs" Link="Helpers\ReportGeneratorCommandLineBase.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportFileNameValidator.cs" Link="Helpers\ReportFileNameValidator.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ $(CommonProductDescription)]]></PackageDescription>
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportJournalConfiguration.cs" Link="Helpers\ReportJournalConfiguration.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportJournalRecord.cs" Link="Helpers\ReportJournalRecord.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportProcessLifetimeHandler.cs" Link="Helpers\ReportProcessLifetimeHandler.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\RetryArtifactManifest.cs" Link="Helpers\RetryArtifactManifest.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportProcessLifetimeHandler.Journal.cs" Link="Helpers\ReportProcessLifetimeHandler.Journal.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportGeneratorCommandLineBase.cs" Link="Helpers\ReportGeneratorCommandLineBase.cs" />
</ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ $(CommonProductDescription)]]></PackageDescription>
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportJournalConfiguration.cs" Link="Helpers\ReportJournalConfiguration.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportJournalRecord.cs" Link="Helpers\ReportJournalRecord.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportProcessLifetimeHandler.cs" Link="Helpers\ReportProcessLifetimeHandler.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\RetryArtifactManifest.cs" Link="Helpers\RetryArtifactManifest.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportProcessLifetimeHandler.Journal.cs" Link="Helpers\ReportProcessLifetimeHandler.Journal.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\ReportGeneratorCommandLineBase.cs" Link="Helpers\ReportGeneratorCommandLineBase.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\TestResultCaptureHelper.cs" Link="Helpers\TestResultCaptureHelper.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,6 @@ private static void RecoverScratchArtifacts(string? scratchDirectory, string? re

private void RecoverRetryArtifactManifest()
{
const long MaxManifestBytes = 16L * 1024 * 1024;
const int MaxManifestLineChars = 64 * 1024;
const int MaxManifestRecords = 10_000;

if (_retryArtifactManifestPath is null
|| _retryArtifactManifestDestinationPath is null
|| _scratchDirectory is null)
Expand All @@ -146,27 +142,25 @@ private void RecoverRetryArtifactManifest()
try
{
if (!File.Exists(_retryArtifactManifestPath)
|| new FileInfo(_retryArtifactManifestPath).Length > MaxManifestBytes)
|| new FileInfo(_retryArtifactManifestPath).Length > RetryArtifactManifest.MaxBytes)
{
return;
}

var recoveredLines = new List<string>();
foreach (string line in File.ReadLines(_retryArtifactManifestPath).Take(MaxManifestRecords))
foreach (string line in File.ReadLines(_retryArtifactManifestPath).Take(RetryArtifactManifest.MaxRecords))
{
if (line.Length > MaxManifestLineChars)
if (line.Length > RetryArtifactManifest.MaxLineLength)
{
continue;
}

int separatorIndex = line.IndexOf('\t');
if (separatorIndex <= 0)
if (!RetryArtifactManifest.TrySplitEntry(line, out string? encodedPath, out string? encodedKindOrNullSentinel))
{
continue;
}

string sourcePath = Path.GetFullPath(
Encoding.UTF8.GetString(Convert.FromBase64String(line.Substring(0, separatorIndex))));
string sourcePath = Path.GetFullPath(RetryArtifactManifest.DecodePath(encodedPath));
string? recoveredPath = TryGetRecoveredArtifactPath(
sourcePath,
_resultsScratchDirectory,
Expand All @@ -180,8 +174,9 @@ private void RecoverRetryArtifactManifest()
continue;
}

recoveredLines.Add(
$"{Convert.ToBase64String(Encoding.UTF8.GetBytes(Path.GetFullPath(recoveredPath)))}{line[separatorIndex..]}");
recoveredLines.Add(RetryArtifactManifest.WriteEntryWithEncodedKind(
Path.GetFullPath(recoveredPath),
encodedKindOrNullSentinel));
}

if (recoveredLines.Count == 0)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ $(CommonProductDescription)]]></PackageDescription>
<!-- Reuse the platform's Windows command-line quoting so the activation-argument round-trip cannot
diverge from the rest of the product (same helper the Retry extension source-links). -->
<Compile Include="$(RepoRoot)src\Platform\Microsoft.Testing.Platform\Helpers\PasteArguments.cs" Link="Helpers\PasteArguments.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\RetryArtifactManifest.cs" Link="Helpers\RetryArtifactManifest.cs" />
</ItemGroup>

<!-- NuGet package layout -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
<Compile Include="$(RepoRoot)src\Platform\Microsoft.Testing.Platform\Helpers\PasteArguments.cs" Link="Helpers\PasteArguments.cs" />
<Compile Include="$(RepoRoot)src\Platform\Microsoft.Testing.Platform\Helpers\TimeoutHelper.cs" Link="Helpers\TimeoutHelper.cs" />
<Compile Include="$(RepoRoot)src\Platform\Microsoft.Testing.Platform\Helpers\TimeSpanParser.cs" Link="Helpers\TimeSpanParser.cs" />
<Compile Include="$(RepoRoot)src\Platform\SharedExtensionHelpers\RetryArtifactManifest.cs" Link="Helpers\RetryArtifactManifest.cs" />
</ItemGroup>

<!-- NuGet properties -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ internal sealed partial class RetryOrchestrator
private const string AppContainerArtifactRootsConfiguredEnvironmentVariable = "TESTINGPLATFORM_PACKAGEDAPP_APPCONTAINER_ARTIFACT_ROOTS_CONFIGURED";
private const string ArtifactPathDestinationRootEnvironmentVariable = "TESTINGPLATFORM_ARTIFACT_PATH_DESTINATION_ROOT";
private const string DiagnosticArtifactPathDestinationRootEnvironmentVariable = "TESTINGPLATFORM_DIAGNOSTIC_ARTIFACT_PATH_DESTINATION_ROOT";
private const long MaxRecoveredArtifactManifestBytes = 16L * 1024 * 1024;
private const int MaxRecoveredArtifactManifestLineBytes = 64 * 1024;
private const int MaxRecoveredArtifactManifestRecords = 10_000;
private const int MaxRecoveredArtifactPathChars = 32 * 1024;
private const int MaxRecoveredArtifactKindChars = 1024;

Expand All @@ -35,7 +32,7 @@ private static void CollectRecoveredArtifacts(
using IFileStream stream = fileSystem.NewFileStream(manifestPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var reader = new BoundedManifestLineReader(stream.Stream);
int recordCount = 0;
while (recordCount++ < MaxRecoveredArtifactManifestRecords)
while (recordCount++ < RetryArtifactManifest.MaxRecords)
{
BoundedManifestLineReadResult readResult = reader.ReadLine(out string line);
if (readResult == BoundedManifestLineReadResult.End)
Expand All @@ -49,20 +46,16 @@ private static void CollectRecoveredArtifacts(
return;
}

int separatorIndex = line.IndexOf('\t');
if (separatorIndex <= 0)
if (!RetryArtifactManifest.TrySplitEntry(line, out string? encodedPath, out string? encodedKindOrNullSentinel))
{
logger.LogWarning($"Ignoring malformed recovered retry artifact manifest entry in '{manifestPath}'.");
continue;
}

try
{
string path = Encoding.UTF8.GetString(Convert.FromBase64String(line.Substring(0, separatorIndex)));
string encodedKind = line.Substring(separatorIndex + 1);
string? kind = encodedKind == "-"
? null
: Encoding.UTF8.GetString(Convert.FromBase64String(encodedKind));
string path = RetryArtifactManifest.DecodePath(encodedPath);
string? kind = RetryArtifactManifest.DecodeKind(encodedKindOrNullSentinel);
if (path.Length > MaxRecoveredArtifactPathChars
|| kind?.Length > MaxRecoveredArtifactKindChars)
{
Expand Down Expand Up @@ -101,7 +94,7 @@ private static void CollectRecoveredArtifacts(
}
}

logger.LogWarning($"Stopped reading recovered retry artifact manifest '{manifestPath}' after the maximum of {MaxRecoveredArtifactManifestRecords} records.");
logger.LogWarning($"Stopped reading recovered retry artifact manifest '{manifestPath}' after the maximum of {RetryArtifactManifest.MaxRecords} records.");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Expand Down Expand Up @@ -195,7 +188,7 @@ private sealed class BoundedManifestLineReader(Stream stream)
private const int BufferSize = 8192;

private readonly byte[] _readBuffer = new byte[BufferSize];
private readonly byte[] _lineBuffer = new byte[MaxRecoveredArtifactManifestLineBytes];
private readonly byte[] _lineBuffer = new byte[RetryArtifactManifest.MaxLineLength];
private int _readOffset;
private int _readCount;
private long _bytesRead;
Expand All @@ -205,7 +198,7 @@ public BoundedManifestLineReadResult ReadLine(out string line)
int lineLength = 0;
while (TryReadByte(out byte value))
{
if (++_bytesRead > MaxRecoveredArtifactManifestBytes)
if (++_bytesRead > RetryArtifactManifest.MaxBytes)
{
line = string.Empty;
return BoundedManifestLineReadResult.LimitExceeded;
Expand All @@ -216,7 +209,7 @@ public BoundedManifestLineReadResult ReadLine(out string line)
return DecodeLine(lineLength, out line);
}

if (lineLength >= MaxRecoveredArtifactManifestLineBytes)
if (lineLength >= RetryArtifactManifest.MaxLineLength)
{
line = string.Empty;
return BoundedManifestLineReadResult.LimitExceeded;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -183,21 +183,18 @@ private async Task WriteRetryArtifactManifestAsync(
_fileSystem.CreateDirectory(directory);
}

string encodedPath = Convert.ToBase64String(Encoding.UTF8.GetBytes(artifactPath));
string encodedKind = artifactKind is null
? "-"
: Convert.ToBase64String(Encoding.UTF8.GetBytes(artifactKind));
string entry = RetryArtifactManifest.WriteEntry(artifactPath, artifactKind);
using IFileStream stream = _fileSystem.NewFileStream(
manifestPath,
FileMode.Append,
FileAccess.Write,
FileShare.Read);
using var writer = new StreamWriter(stream.Stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
#if NETCOREAPP
await writer.WriteLineAsync($"{encodedPath}\t{encodedKind}".AsMemory(), cancellationToken).ConfigureAwait(false);
await writer.WriteLineAsync(entry.AsMemory(), cancellationToken).ConfigureAwait(false);
#else
cancellationToken.ThrowIfCancellationRequested();
await writer.WriteLineAsync($"{encodedPath}\t{encodedKind}").ConfigureAwait(false);
await writer.WriteLineAsync(entry).ConfigureAwait(false);
#endif
}
catch (Exception ex) when (ex is not OperationCanceledException)
Expand Down
49 changes: 49 additions & 0 deletions src/Platform/SharedExtensionHelpers/RetryArtifactManifest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.Testing.Extensions;

#pragma warning disable RS0051 // Recovery infrastructure is shared-source implementation detail, not package API.

internal static class RetryArtifactManifest
{
public const long MaxBytes = 16L * 1024 * 1024;
public const int MaxLineLength = 64 * 1024;
public const int MaxRecords = 10_000;

public static bool TrySplitEntry(
string line,
[NotNullWhen(true)] out string? encodedPath,
[NotNullWhen(true)] out string? encodedKindOrNullSentinel)
{
int separatorIndex = line.IndexOf('\t');
if (separatorIndex <= 0)
{
encodedPath = null;
encodedKindOrNullSentinel = null;
return false;
}

encodedPath = line.Substring(0, separatorIndex);
encodedKindOrNullSentinel = line.Substring(separatorIndex + 1);
return true;
}

public static string DecodePath(string encodedPath)
=> Encoding.UTF8.GetString(Convert.FromBase64String(encodedPath));

public static string? DecodeKind(string encodedKindOrNullSentinel)
=> encodedKindOrNullSentinel == "-"
? null
: Encoding.UTF8.GetString(Convert.FromBase64String(encodedKindOrNullSentinel));

public static string WriteEntryWithEncodedKind(string path, string encodedKindOrNullSentinel)
=> $"{Convert.ToBase64String(Encoding.UTF8.GetBytes(path))}\t{encodedKindOrNullSentinel}";
Comment thread
Evangelink marked this conversation as resolved.
Resolved

public static string WriteEntry(string path, string? kind)
=> WriteEntryWithEncodedKind(
Comment thread
Evangelink marked this conversation as resolved.
Resolved
path,
kind is null ? "-" : Convert.ToBase64String(Encoding.UTF8.GetBytes(kind)));
}

#pragma warning restore RS0051
Loading
Loading