From 8508fe9c6687c12f06b15b01a8beb13daf8bd532 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 4 Feb 2025 23:16:02 -0500 Subject: [PATCH 1/5] Add LINQ Shuffle --- .../ref/System.Linq.AsyncEnumerable.cs | 1 + .../src/System.Linq.AsyncEnumerable.csproj | 1 + .../src/System/Linq/Shuffle.cs | 59 ++++++ .../tests/ShuffleTests.cs | 69 ++++++ .../System.Linq.AsyncEnumerable.Tests.csproj | 1 + .../ref/System.Linq.Queryable.cs | 1 + .../src/System/Linq/Queryable.cs | 12 ++ .../tests/ShuffleTests.cs | 37 ++++ .../tests/System.Linq.Queryable.Tests.csproj | 4 +- src/libraries/System.Linq/ref/System.Linq.cs | 3 + .../System.Linq/src/System.Linq.csproj | 2 + .../src/System/Linq/Shuffle.SpeedOpt.cs | 73 +++++++ .../System.Linq/src/System/Linq/Shuffle.cs | 91 ++++++++ .../System.Linq/tests/ShuffleTests.cs | 198 ++++++++++++++++++ src/libraries/System.Linq/tests/Shuffler.cs | 2 +- .../tests/System.Linq.Tests.csproj | 1 + .../UnitTests/Headers/HttpHeadersTest.cs | 2 +- .../tests/axes/AxisOrderValidation.cs | 3 +- 18 files changed, 554 insertions(+), 6 deletions(-) create mode 100644 src/libraries/System.Linq.AsyncEnumerable/src/System/Linq/Shuffle.cs create mode 100644 src/libraries/System.Linq.AsyncEnumerable/tests/ShuffleTests.cs create mode 100644 src/libraries/System.Linq.Queryable/tests/ShuffleTests.cs create mode 100644 src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs create mode 100644 src/libraries/System.Linq/src/System/Linq/Shuffle.cs create mode 100644 src/libraries/System.Linq/tests/ShuffleTests.cs diff --git a/src/libraries/System.Linq.AsyncEnumerable/ref/System.Linq.AsyncEnumerable.cs b/src/libraries/System.Linq.AsyncEnumerable/ref/System.Linq.AsyncEnumerable.cs index 6153d0bd56a07c..881b679a19f3ae 100644 --- a/src/libraries/System.Linq.AsyncEnumerable/ref/System.Linq.AsyncEnumerable.cs +++ b/src/libraries/System.Linq.AsyncEnumerable/ref/System.Linq.AsyncEnumerable.cs @@ -132,6 +132,7 @@ public static partial class AsyncEnumerable public static System.Collections.Generic.IAsyncEnumerable Select(this System.Collections.Generic.IAsyncEnumerable source, System.Func> selector) { throw null; } public static System.Collections.Generic.IAsyncEnumerable Select(this System.Collections.Generic.IAsyncEnumerable source, System.Func selector) { throw null; } public static System.Threading.Tasks.ValueTask SequenceEqualAsync(this System.Collections.Generic.IAsyncEnumerable first, System.Collections.Generic.IAsyncEnumerable second, System.Collections.Generic.IEqualityComparer? comparer = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Collections.Generic.IAsyncEnumerable Shuffle(this System.Collections.Generic.IAsyncEnumerable source) { throw null; } public static System.Threading.Tasks.ValueTask SingleAsync(this System.Collections.Generic.IAsyncEnumerable source, System.Func predicate, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.ValueTask SingleAsync(this System.Collections.Generic.IAsyncEnumerable source, System.Func> predicate, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.ValueTask SingleAsync(this System.Collections.Generic.IAsyncEnumerable source, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } diff --git a/src/libraries/System.Linq.AsyncEnumerable/src/System.Linq.AsyncEnumerable.csproj b/src/libraries/System.Linq.AsyncEnumerable/src/System.Linq.AsyncEnumerable.csproj index 65d7a694bb6a54..3c5a3aa8f2257d 100644 --- a/src/libraries/System.Linq.AsyncEnumerable/src/System.Linq.AsyncEnumerable.csproj +++ b/src/libraries/System.Linq.AsyncEnumerable/src/System.Linq.AsyncEnumerable.csproj @@ -12,6 +12,7 @@ + diff --git a/src/libraries/System.Linq.AsyncEnumerable/src/System/Linq/Shuffle.cs b/src/libraries/System.Linq.AsyncEnumerable/src/System/Linq/Shuffle.cs new file mode 100644 index 00000000000000..a798976a5aef84 --- /dev/null +++ b/src/libraries/System.Linq.AsyncEnumerable/src/System/Linq/Shuffle.cs @@ -0,0 +1,59 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace System.Linq +{ + public static partial class AsyncEnumerable + { +#if !NET + [ThreadStatic] + private static Random? t_random; +#endif + + /// Shuffles the order of the elements of a sequence. + /// The type of the elements of . + /// A sequence of values to shuffle. + /// A sequence whose elements correspond to those of the input sequence in randomized order. + /// Randomization is performed using a non-cryptographically-secure random number generator. + public static IAsyncEnumerable Shuffle( + this IAsyncEnumerable source) + { + ThrowHelper.ThrowIfNull(source); + + return Impl(source, default); + + static async IAsyncEnumerable Impl( + IAsyncEnumerable source, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + TSource[] array = await source.ToArrayAsync(cancellationToken).ConfigureAwait(false); + +#if NET + Random.Shared.Shuffle(array); +#else + Random random = t_random ??= new Random(Environment.TickCount ^ Environment.CurrentManagedThreadId); + int n = array.Length; + for (int i = 0; i < n - 1; i++) + { + int j = random.Next(i, n); + if (j != i) + { + TSource temp = array[i]; + array[i] = array[j]; + array[j] = temp; + } + } +#endif + + for (int i = 0; i < array.Length; i++) + { + yield return array[i]; + } + } + } + } +} diff --git a/src/libraries/System.Linq.AsyncEnumerable/tests/ShuffleTests.cs b/src/libraries/System.Linq.AsyncEnumerable/tests/ShuffleTests.cs new file mode 100644 index 00000000000000..39130f280099df --- /dev/null +++ b/src/libraries/System.Linq.AsyncEnumerable/tests/ShuffleTests.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.Linq.Tests +{ + public class ShuffleTests : AsyncEnumerableTests + { + [Fact] + public void InvalidInputs_Throws() + { + AssertExtensions.Throws("source", () => AsyncEnumerable.Shuffle(null)); + } + + [Theory] + [InlineData(new int[0])] + [InlineData(new int[] { 1 })] + [InlineData(new int[] { 2, 4, 8 })] + [InlineData(new int[] { -1, 2, 5, 6, 7, 8 })] + public async Task VariousValues_ContainsAllInputValues(int[] values) + { + foreach (IAsyncEnumerable source in CreateSources(values)) + { + int[] shuffled = await source.Shuffle().ToArrayAsync(); + Array.Sort(shuffled); + Assert.Equal(values, shuffled); + } + } + + [Fact] + public async Task ToArrayAsync_ElementsAreRandomized() + { + // The chance that shuffling a thousand elements produces the same order twice is infinitesimal. + int length = 1000; + foreach (IAsyncEnumerable source in CreateSources(Enumerable.Range(0, length).ToArray())) + { + int[] first = await source.Shuffle().ToArrayAsync(); + int[] second = await source.Shuffle().ToArrayAsync(); + Assert.Equal(length, first.Length); + Assert.Equal(length, second.Length); + Assert.NotEqual(first, second); + } + } + + [Fact] + public async Task Cancellation_Cancels() + { + IAsyncEnumerable source = CreateSource(2, 4, 8, 16); + await Assert.ThrowsAsync(async () => + { + await ConsumeAsync(source.Shuffle().WithCancellation(new CancellationToken(true))); + }); + } + + [Fact] + public async Task InterfaceCalls_ExpectedCounts() + { + TrackingAsyncEnumerable source = CreateSource(2, 4, 8, 16).Track(); + await ConsumeAsync(source.Shuffle()); + Assert.Equal(5, source.MoveNextAsyncCount); + Assert.Equal(4, source.CurrentCount); + Assert.Equal(1, source.DisposeAsyncCount); + } + } +} diff --git a/src/libraries/System.Linq.AsyncEnumerable/tests/System.Linq.AsyncEnumerable.Tests.csproj b/src/libraries/System.Linq.AsyncEnumerable/tests/System.Linq.AsyncEnumerable.Tests.csproj index 1a819d87b173b9..164e0e77aa690c 100644 --- a/src/libraries/System.Linq.AsyncEnumerable/tests/System.Linq.AsyncEnumerable.Tests.csproj +++ b/src/libraries/System.Linq.AsyncEnumerable/tests/System.Linq.AsyncEnumerable.Tests.csproj @@ -10,6 +10,7 @@ + diff --git a/src/libraries/System.Linq.Queryable/ref/System.Linq.Queryable.cs b/src/libraries/System.Linq.Queryable/ref/System.Linq.Queryable.cs index 2db91584c129d9..38bff9dcfbf048 100644 --- a/src/libraries/System.Linq.Queryable/ref/System.Linq.Queryable.cs +++ b/src/libraries/System.Linq.Queryable/ref/System.Linq.Queryable.cs @@ -158,6 +158,7 @@ public static partial class Queryable public static System.Linq.IQueryable Select(this System.Linq.IQueryable source, System.Linq.Expressions.Expression> selector) { throw null; } public static bool SequenceEqual(this System.Linq.IQueryable source1, System.Collections.Generic.IEnumerable source2) { throw null; } public static bool SequenceEqual(this System.Linq.IQueryable source1, System.Collections.Generic.IEnumerable source2, System.Collections.Generic.IEqualityComparer? comparer) { throw null; } + public static System.Linq.IQueryable Shuffle(this System.Linq.IQueryable source) { throw null; } public static TSource? SingleOrDefault(this System.Linq.IQueryable source) { throw null; } public static TSource? SingleOrDefault(this System.Linq.IQueryable source, System.Linq.Expressions.Expression> predicate) { throw null; } public static TSource SingleOrDefault(this System.Linq.IQueryable source, System.Linq.Expressions.Expression> predicate, TSource defaultValue) { throw null; } diff --git a/src/libraries/System.Linq.Queryable/src/System/Linq/Queryable.cs b/src/libraries/System.Linq.Queryable/src/System/Linq/Queryable.cs index 0bfe55333b9f1d..802ef47e0ee711 100644 --- a/src/libraries/System.Linq.Queryable/src/System/Linq/Queryable.cs +++ b/src/libraries/System.Linq.Queryable/src/System/Linq/Queryable.cs @@ -1587,6 +1587,18 @@ public static bool SequenceEqual(this IQueryable source1, IEnu Expression.Constant(comparer, typeof(IEqualityComparer)))); } + [DynamicDependency("Shuffle`1", typeof(Enumerable))] + public static IQueryable Shuffle(this IQueryable source) + { + ArgumentNullException.ThrowIfNull(source); + + return source.Provider.CreateQuery( + Expression.Call( + null, + new Func, IQueryable>(Shuffle).Method, + source.Expression)); + } + [DynamicDependency("Any`1", typeof(Enumerable))] public static bool Any(this IQueryable source) { diff --git a/src/libraries/System.Linq.Queryable/tests/ShuffleTests.cs b/src/libraries/System.Linq.Queryable/tests/ShuffleTests.cs new file mode 100644 index 00000000000000..6d9656a80c46e5 --- /dev/null +++ b/src/libraries/System.Linq.Queryable/tests/ShuffleTests.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace System.Linq.Tests +{ + public class ShuffleTests : EnumerableBasedTests + { + [Fact] + public void InvalidArguments() + { + AssertExtensions.Throws("source", () => ((IQueryable)null).Shuffle()); + } + + [Fact] + public void ProducesAllElements() + { + int[] shuffled = Enumerable.Range(0, 1000).AsQueryable().Shuffle().ToArray(); + Array.Sort(shuffled); + Assert.Equal(Enumerable.Range(0, shuffled.Length), shuffled); + } + + [Fact] + public void ElementsAreRandomized() + { + // The chance that shuffling a thousand elements produces the same order twice is infinitesimal. + const int Length = 1000; + IQueryable source = Enumerable.Range(0, Length).AsQueryable().Shuffle(); + int[] first = source.ToArray(); + int[] second = source.ToArray(); + Assert.Equal(Length, first.Length); + Assert.Equal(Length, second.Length); + Assert.NotEqual(first, second); + } + } +} diff --git a/src/libraries/System.Linq.Queryable/tests/System.Linq.Queryable.Tests.csproj b/src/libraries/System.Linq.Queryable/tests/System.Linq.Queryable.Tests.csproj index 5d35929945edfd..9fcf767d3e81b0 100644 --- a/src/libraries/System.Linq.Queryable/tests/System.Linq.Queryable.Tests.csproj +++ b/src/libraries/System.Linq.Queryable/tests/System.Linq.Queryable.Tests.csproj @@ -31,6 +31,7 @@ + @@ -62,7 +63,6 @@ - + diff --git a/src/libraries/System.Linq/ref/System.Linq.cs b/src/libraries/System.Linq/ref/System.Linq.cs index 20682db0deb76f..50657cf628e7d9 100644 --- a/src/libraries/System.Linq/ref/System.Linq.cs +++ b/src/libraries/System.Linq/ref/System.Linq.cs @@ -4,6 +4,8 @@ // Changes to this file must follow the https://aka.ms/api-review process. // ------------------------------------------------------------------------------ +using System.Collections.Generic; + namespace System.Linq { public static partial class Enumerable @@ -172,6 +174,7 @@ public static System.Collections.Generic.IEnumerable< public static System.Collections.Generic.IEnumerable Select(this System.Collections.Generic.IEnumerable source, System.Func selector) { throw null; } public static bool SequenceEqual(this System.Collections.Generic.IEnumerable first, System.Collections.Generic.IEnumerable second) { throw null; } public static bool SequenceEqual(this System.Collections.Generic.IEnumerable first, System.Collections.Generic.IEnumerable second, System.Collections.Generic.IEqualityComparer? comparer) { throw null; } + public static System.Collections.Generic.IEnumerable Shuffle(this System.Collections.Generic.IEnumerable source) { throw null; } public static TSource? SingleOrDefault(this System.Collections.Generic.IEnumerable source) { throw null; } public static TSource SingleOrDefault(this System.Collections.Generic.IEnumerable source, TSource defaultValue) { throw null; } public static TSource? SingleOrDefault(this System.Collections.Generic.IEnumerable source, System.Func predicate) { throw null; } diff --git a/src/libraries/System.Linq/src/System.Linq.csproj b/src/libraries/System.Linq/src/System.Linq.csproj index cbbec2028cf8d7..7d5854ad4dd983 100644 --- a/src/libraries/System.Linq/src/System.Linq.csproj +++ b/src/libraries/System.Linq/src/System.Linq.csproj @@ -54,6 +54,8 @@ + + diff --git a/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs new file mode 100644 index 00000000000000..2f91430e9114aa --- /dev/null +++ b/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace System.Linq +{ + public static partial class Enumerable + { + private sealed partial class ShuffleIterator + { + public override TSource[] ToArray() + { + TSource[] array = _source.ToArray(); + Random.Shared.Shuffle(array); + return array; + } + + public override List ToList() + { + List list = _source.ToList(); + Random.Shared.Shuffle(CollectionsMarshal.AsSpan(list)); + return list; + } + + public override int GetCount(bool onlyIfCheap) => + !onlyIfCheap ? _source.Count() : + TryGetNonEnumeratedCount(_source, out int count) ? count : + -1; + + public override TSource? TryGetFirst(out bool found) => + TryGetElementAt(0, out found); + + public override TSource? TryGetLast(out bool found) => + TryGetElementAt(0, out found); + + public override TSource? TryGetElementAt(int index, out bool found) + { + if (_source is Iterator iterator && + iterator.GetCount(onlyIfCheap: true) is int iteratorCount && + iteratorCount >= 0) + { + if ((uint)index < (uint)iteratorCount) + { + return iterator.TryGetElementAt(Random.Shared.Next(0, iteratorCount), out found); + } + } + else if (_source is IList list) + { + int listCount = list.Count; + if ((uint)index < (uint)listCount) + { + found = true; + return list[Random.Shared.Next(0, listCount)]; + } + } + else if (index >= 0) + { + TSource[] array = _source.ToArray(); + if (index < array.Length) + { + found = true; + return array[Random.Shared.Next(0, array.Length)]; + } + } + + found = false; + return default; + } + } + } +} diff --git a/src/libraries/System.Linq/src/System/Linq/Shuffle.cs b/src/libraries/System.Linq/src/System/Linq/Shuffle.cs new file mode 100644 index 00000000000000..a6340163cee367 --- /dev/null +++ b/src/libraries/System.Linq/src/System/Linq/Shuffle.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; + +namespace System.Linq +{ + public static partial class Enumerable + { + /// Shuffles the order of the elements of a sequence. + /// The type of the elements of . + /// A sequence of values to shuffle. + /// A sequence whose elements correspond to those of the input sequence in randomized order. + /// Randomization is performed using a non-cryptographically-secure random number generator. + public static IEnumerable Shuffle(this IEnumerable source) + { + if (source is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); + } + + if (IsEmptyArray(source)) + { + return []; + } + + if (source is ShuffleIterator shuffleIterator) + { + // Shuffling again won't impact anything. + return shuffleIterator; + } + + return new ShuffleIterator(source); + } + + private sealed partial class ShuffleIterator : Iterator + { + private readonly IEnumerable _source; + private TSource[]? _buffer; + + public ShuffleIterator(IEnumerable source) + { + Debug.Assert(source is not null); + _source = source; + } + + private protected override Iterator Clone() => new ShuffleIterator(_source); + + public override bool MoveNext() + { + int state = _state; + + Initialized: + if (state > 1) + { + TSource[]? buffer = _buffer; + Debug.Assert(buffer is not null); + + int i = state - 2; + if ((uint)i < (uint)buffer.Length) + { + _current = buffer[i]; + _state++; + return true; + } + } + else if (state == 1) + { + TSource[] buffer = _source.ToArray(); + if (buffer.Length != 0) + { + Random.Shared.Shuffle(buffer); + _buffer = buffer; + _state = state = 2; + goto Initialized; + } + } + + Dispose(); + return false; + } + + public override void Dispose() + { + _buffer = null; + base.Dispose(); + } + } + } +} diff --git a/src/libraries/System.Linq/tests/ShuffleTests.cs b/src/libraries/System.Linq/tests/ShuffleTests.cs new file mode 100644 index 00000000000000..71be4327db5934 --- /dev/null +++ b/src/libraries/System.Linq/tests/ShuffleTests.cs @@ -0,0 +1,198 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Xunit; + +namespace System.Linq.Tests +{ + public class ShuffleTests : EnumerableTests + { + [Fact] + public void InvalidArguments() + { + AssertExtensions.Throws("source", () => Enumerable.Shuffle(null)); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(100)] + public void Count_ExpectedCountReturned(int length) + { + Assert.All(CreateSources(Enumerable.Range(0, length)), source => + { + Assert.Equal(length, source.Shuffle().Count()); + + IEnumerable shuffled = source.Shuffle(); + Assert.Equal(length, shuffled.Count()); + Assert.Equal(length, shuffled.Count()); + }); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(100)] + public void Enumeration_AllElementsReturned(int length) + { + Assert.All(CreateSources(Enumerable.Range(0, length)), source => + { + List shuffled = []; + foreach (int i in source.Shuffle()) + { + shuffled.Add(i); + } + + Assert.Equal(length, shuffled.Count); + shuffled.Sort(); + Assert.Equal(Enumerable.Range(0, length), shuffled); + }); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(100)] + public void ToArray_AllElementsReturned(int length) + { + Assert.All(CreateSources(Enumerable.Range(0, length)), source => + { + int[] shuffled = source.Shuffle().ToArray(); + Assert.Equal(length, shuffled.Length); + Array.Sort(shuffled); + Assert.Equal(Enumerable.Range(0, length), shuffled); + }); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(100)] + public void ToList_AllElementsReturned(int length) + { + Assert.All(CreateSources(Enumerable.Range(0, length)), source => + { + List shuffled = source.Shuffle().ToList(); + Assert.Equal(length, shuffled.Count); + shuffled.Sort(); + Assert.Equal(Enumerable.Range(0, length), shuffled); + }); + } + + [Fact] + public void Enumeration_ElementsAreRandomized() + { + // The chance that shuffling a thousand elements produces the same order twice is infinitesimal. + int length = 1000; + Assert.All(CreateSources(Enumerable.Range(0, length)), source => + { + List first = [], second = []; + foreach (int i in source.Shuffle()) + { + first.Add(i); + } + foreach (int i in source.Shuffle()) + { + second.Add(i); + } + + Assert.Equal(length, first.Count); + Assert.Equal(length, second.Count); + Assert.NotEqual(first, second); + }); + } + + [Fact] + public void ToArray_ElementsAreRandomized() + { + // The chance that shuffling a thousand elements produces the same order twice is infinitesimal. + int length = 1000; + Assert.All(CreateSources(Enumerable.Range(0, length)), source => + { + int[] first = source.Shuffle().ToArray(), second = source.Shuffle().ToArray(); + Assert.Equal(length, first.Length); + Assert.Equal(length, second.Length); + Assert.NotEqual(first, second); + }); + } + + [Fact] + public void ToList_ElementsAreRandomized() + { + // The chance that shuffling a thousand elements produces the same order twice is infinitesimal. + int length = 1000; + Assert.All(CreateSources(Enumerable.Range(0, length)), source => + { + List first = source.Shuffle().ToList(), second = source.Shuffle().ToList(); + Assert.Equal(length, first.Count); + Assert.Equal(length, second.Count); + Assert.NotEqual(first, second); + }); + } + + [Fact] + public void ForcedToEnumeratorDoesntEnumerate() + { + var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Reverse(); + // Don't insist on this behaviour, but check it's correct if it happens + var en = iterator as IEnumerator; + Assert.False(en is not null && en.MoveNext()); + } + + [Fact] + public void First_Last_GetElement_Invalid_ExpectedExceptions() + { + Assert.All(CreateSources(Enumerable.Empty()), source => + { + Assert.Throws(() => source.Shuffle().First()); + Assert.Throws(() => source.Shuffle().Last()); + + Assert.Null(source.Shuffle().ElementAtOrDefault(1)); + Assert.Null(source.Shuffle().ElementAtOrDefault(-1)); + Assert.Null(source.Shuffle().FirstOrDefault()); + Assert.Null(source.Shuffle().LastOrDefault()); + }); + } + + [Fact] + public void First_Last_GetElement_Valid_ExpectedElements() + { + Assert.All(CreateSources(Enumerable.Range(0, 10)), source => + { + Assert.InRange(source.Shuffle().First(), 0, 9); + Assert.InRange(source.Shuffle().Last(), 0, 9); + Assert.InRange(source.Shuffle().ElementAt(5), 0, 9); + }); + } + + [Fact] + public void First_Last_GetElement_ProduceRandomElements() + { + Assert.All(CreateSources(Enumerable.Range(0, 100)), source => + { + AssertRetry(() => source.Shuffle().First() != source.Shuffle().First()); + AssertRetry(() => source.Shuffle().Last() != source.Shuffle().Last()); + AssertRetry(() => source.Shuffle().ElementAt(5) != source.Shuffle().ElementAt(5)); + }); + + static void AssertRetry(Func predicate) + { + for (int i = 0; i < 10; i++) + { + if (predicate()) return; + } + + Assert.Fail("Predicate was true for 10 iterations"); + } + } + } +} diff --git a/src/libraries/System.Linq/tests/Shuffler.cs b/src/libraries/System.Linq/tests/Shuffler.cs index 6339fe8037a3e8..4a96202245c88c 100644 --- a/src/libraries/System.Linq/tests/Shuffler.cs +++ b/src/libraries/System.Linq/tests/Shuffler.cs @@ -14,6 +14,6 @@ public static T[] Shuffle(T[] array) return array; } - public static IEnumerable Shuffle(this IEnumerable source) => Shuffle(source.ToArray()); + public static IEnumerable Shuffle(IEnumerable source) => Shuffle(source.ToArray()); } } diff --git a/src/libraries/System.Linq/tests/System.Linq.Tests.csproj b/src/libraries/System.Linq/tests/System.Linq.Tests.csproj index 2e1ad5368b276b..3b0b903654773e 100644 --- a/src/libraries/System.Linq/tests/System.Linq.Tests.csproj +++ b/src/libraries/System.Linq/tests/System.Linq.Tests.csproj @@ -59,6 +59,7 @@ + diff --git a/src/libraries/System.Net.Http/tests/UnitTests/Headers/HttpHeadersTest.cs b/src/libraries/System.Net.Http/tests/UnitTests/Headers/HttpHeadersTest.cs index ce306a2e4cd48e..411a6d9f70b3b1 100644 --- a/src/libraries/System.Net.Http/tests/UnitTests/Headers/HttpHeadersTest.cs +++ b/src/libraries/System.Net.Http/tests/UnitTests/Headers/HttpHeadersTest.cs @@ -2494,7 +2494,7 @@ static HttpRequestHeaders CreateHeaders() static headers => headers.TryAddWithoutValidation("h", new[] { "i", "j" }), }; - foreach (Action action in actions.OrderBy(_ => Random.Shared.Next())) + foreach (Action action in actions.Shuffle()) { action(headers); } diff --git a/src/libraries/System.Private.Xml.Linq/tests/axes/AxisOrderValidation.cs b/src/libraries/System.Private.Xml.Linq/tests/axes/AxisOrderValidation.cs index a3a68755381f89..deb1bbdd5a4e16 100644 --- a/src/libraries/System.Private.Xml.Linq/tests/axes/AxisOrderValidation.cs +++ b/src/libraries/System.Private.Xml.Linq/tests/axes/AxisOrderValidation.cs @@ -336,8 +336,7 @@ public static void IEnumerableInDocumentOrderVariationTwo() public static void ReorderToDocumentOrder() { XDocument xDoc = TestData.GetDocumentWithContacts(); - Random rnd = new Random(); - var randomOrderedElements = xDoc.Root.DescendantNodesAndSelf().OrderBy(n => rnd.Next(0, int.MaxValue)); + var randomOrderedElements = xDoc.Root.DescendantNodesAndSelf().Shuffle(); using (var en = randomOrderedElements.InDocumentOrder().GetEnumerator()) { en.MoveNext(); From a2843325445a6d8cc2225abd217ce778d1c22c08 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Wed, 5 Feb 2025 08:29:51 -0500 Subject: [PATCH 2/5] Remove bespoke Shuffler --- .../System.Linq/tests/AverageTests.cs | 15 ++--- src/libraries/System.Linq/tests/MaxTests.cs | 63 ++++++------------- src/libraries/System.Linq/tests/MinTests.cs | 50 ++++----------- src/libraries/System.Linq/tests/Shuffler.cs | 19 ------ .../tests/System.Linq.Tests.csproj | 9 +-- 5 files changed, 39 insertions(+), 117 deletions(-) delete mode 100644 src/libraries/System.Linq/tests/Shuffler.cs diff --git a/src/libraries/System.Linq/tests/AverageTests.cs b/src/libraries/System.Linq/tests/AverageTests.cs index 888d6e8043c1c4..6c3920d65c70eb 100644 --- a/src/libraries/System.Linq/tests/AverageTests.cs +++ b/src/libraries/System.Linq/tests/AverageTests.cs @@ -124,8 +124,7 @@ public static IEnumerable Int_TestData() for (int c = 1; c <= i; c++) sum += c; double expected = (double)sum / i; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i)), expected]; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).ToArray()), expected]; + yield return [Enumerable.Range(1, i).Shuffle(), expected]; } } @@ -255,8 +254,7 @@ public static IEnumerable Long_TestData() for (int c = 1; c <= i; c++) sum += c; double expected = (double)sum / i; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (long)i)), expected]; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (long)i).ToArray()), expected]; + yield return [Enumerable.Range(1, i).Select(i => (long)i).Shuffle(), expected]; } } @@ -385,8 +383,7 @@ public static IEnumerable Double_TestData() for (int c = 1; c <= i; c++) sum += c; double expected = (double)sum / i; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (double)i)), expected]; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (double)i).ToArray()), expected]; + yield return [Enumerable.Range(1, i).Select(i => (double)i).Shuffle(), expected]; } } @@ -507,8 +504,7 @@ public static IEnumerable Decimal_TestData() for (int c = 1; c <= i; c++) sum += c; decimal expected = (decimal)sum / i; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (decimal)i)), expected]; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (decimal)i).ToArray()), expected]; + yield return [Enumerable.Range(1, i).Select(i => (decimal)i).Shuffle(), expected]; } } @@ -636,8 +632,7 @@ public static IEnumerable Float_TestData() for (int c = 1; c <= i; c++) sum += c; float expected = (float)sum / i; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (float)i)), expected]; - yield return [Shuffler.Shuffle(Enumerable.Range(1, i).Select(i => (float)i).ToArray()), expected]; + yield return [Enumerable.Range(1, i).Select(i => (float)i).Shuffle(), expected]; } } diff --git a/src/libraries/System.Linq/tests/MaxTests.cs b/src/libraries/System.Linq/tests/MaxTests.cs index ad9ff571c8c5d4..94baed9a7ef520 100644 --- a/src/libraries/System.Linq/tests/MaxTests.cs +++ b/src/libraries/System.Linq/tests/MaxTests.cs @@ -13,56 +13,29 @@ public static IEnumerable Max_AllTypes_TestData() { for (int length = 2; length < 65; length++) { - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (byte)i)), (byte)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (byte)i).ToArray()), (byte)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (byte)i).Shuffle(), (byte)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (byte)i).ToArray().Shuffle(), (byte)(length + length - 1)]; // Unit Tests does +T.One so we should generate data up to one value below sbyte.MaxValue if ((length + length) < sbyte.MaxValue) { - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (sbyte)i)), (sbyte)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (sbyte)i).ToArray()), (sbyte)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (sbyte)i).Shuffle(), (sbyte)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (sbyte)i).ToArray().Shuffle(), (sbyte)(length + length - 1)]; } - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ushort)i)), (ushort)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ushort)i).ToArray()), (ushort)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (short)i)), (short)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (short)i).ToArray()), (short)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (char)i)), (char)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (char)i).ToArray()), (char)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (uint)i)), (uint)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (uint)i).ToArray()), (uint)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (int)i)), (int)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (int)i).ToArray()), (int)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ulong)i)), (ulong)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ulong)i).ToArray()), (ulong)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (long)i)), (long)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (long)i).ToArray()), (long)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (float)i)), (float)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (float)i).ToArray()), (float)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (double)i)), (double)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (double)i).ToArray()), (double)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (decimal)i)), (decimal)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (decimal)i).ToArray()), (decimal)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nuint)i)), (nuint)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nuint)i).ToArray()), (nuint)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nint)i)), (nint)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nint)i).ToArray()), (nint)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (Int128)i)), (Int128)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (Int128)i).ToArray()), (Int128)(length + length - 1)]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (UInt128)i)), (UInt128)(length + length - 1)]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (UInt128)i).ToArray()), (UInt128)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (ushort)i).Shuffle(), (ushort)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (short)i).Shuffle(), (short)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (char)i).Shuffle(), (char)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (uint)i).Shuffle(), (uint)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (int)i).Shuffle(), (int)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (ulong)i).Shuffle(), (ulong)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (long)i).Shuffle(), (long)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (float)i).Shuffle(), (float)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (double)i).Shuffle(), (double)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (decimal)i).Shuffle(), (decimal)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (nuint)i).Shuffle(), (nuint)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (nint)i).Shuffle(), (nint)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (Int128)i).Shuffle(), (Int128)(length + length - 1)]; + yield return [Enumerable.Range(length, length).Select(i => (UInt128)i).Shuffle(), (UInt128)(length + length - 1)]; } } diff --git a/src/libraries/System.Linq/tests/MinTests.cs b/src/libraries/System.Linq/tests/MinTests.cs index 1f4c6ed1097619..8774777bfa744e 100644 --- a/src/libraries/System.Linq/tests/MinTests.cs +++ b/src/libraries/System.Linq/tests/MinTests.cs @@ -13,48 +13,24 @@ public static IEnumerable Min_AllTypes_TestData() { for (int length = 2; length < 65; length++) { - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (byte)i)), (byte)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (byte)i).ToArray()), (byte)length - ]; + yield return [Enumerable.Range(length, length).Select(i => (byte)i).Shuffle(), (byte)length]; // Unit Tests does +T.One so we should generate data up to one value below sbyte.MaxValue, otherwise the type overflows if ((length + length) < sbyte.MaxValue) { - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (sbyte)i)), (sbyte)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (sbyte)i).ToArray()), (sbyte)length]; + yield return [Enumerable.Range(length, length).Select(i => (sbyte)i).Shuffle(), (sbyte)length]; } - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ushort)i)), (ushort)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ushort)i).ToArray()), (ushort)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (short)i)), (short)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (short)i).ToArray()), (short)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (uint)i)), (uint)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (uint)i).ToArray()), (uint)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (int)i)), (int)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (int)i).ToArray()), (int)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ulong)i)), (ulong)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (ulong)i).ToArray()), (ulong)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (long)i)), (long)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (long)i).ToArray()), (long)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (float)i)), (float)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (float)i).ToArray()), (float)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (double)i)), (double)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (double)i).ToArray()), (double)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (decimal)i)), (decimal)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (decimal)i).ToArray()), (decimal)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nuint)i)), (nuint)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nuint)i).ToArray()), (nuint)length]; - - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nint)i)), (nint)length]; - yield return [Shuffler.Shuffle(Enumerable.Range(length, length).Select(i => (nint)i).ToArray()), (nint)length]; + yield return [Enumerable.Range(length, length).Select(i => (ushort)i).Shuffle(), (ushort)length]; + yield return [Enumerable.Range(length, length).Select(i => (short)i).Shuffle(), (short)length]; + yield return [Enumerable.Range(length, length).Select(i => (uint)i).Shuffle(), (uint)length]; + yield return [Enumerable.Range(length, length).Select(i => (int)i).Shuffle(), (int)length]; + yield return [Enumerable.Range(length, length).Select(i => (ulong)i).Shuffle(), (ulong)length]; + yield return [Enumerable.Range(length, length).Select(i => (long)i).Shuffle(), (long)length]; + yield return [Enumerable.Range(length, length).Select(i => (float)i).Shuffle(), (float)length]; + yield return [Enumerable.Range(length, length).Select(i => (double)i).Shuffle(), (double)length]; + yield return [Enumerable.Range(length, length).Select(i => (decimal)i).Shuffle(), (decimal)length]; + yield return [Enumerable.Range(length, length).Select(i => (nuint)i).Shuffle(), (nuint)length]; + yield return [Enumerable.Range(length, length).Select(i => (nint)i).Shuffle(), (nint)length]; } } diff --git a/src/libraries/System.Linq/tests/Shuffler.cs b/src/libraries/System.Linq/tests/Shuffler.cs deleted file mode 100644 index 4a96202245c88c..00000000000000 --- a/src/libraries/System.Linq/tests/Shuffler.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; - -namespace System.Linq.Tests -{ - public static class Shuffler - { - public static T[] Shuffle(T[] array) - { - var r = new Random(42); - r.Shuffle(array); - return array; - } - - public static IEnumerable Shuffle(IEnumerable source) => Shuffle(source.ToArray()); - } -} diff --git a/src/libraries/System.Linq/tests/System.Linq.Tests.csproj b/src/libraries/System.Linq/tests/System.Linq.Tests.csproj index 3b0b903654773e..10dc8ed05e76c8 100644 --- a/src/libraries/System.Linq/tests/System.Linq.Tests.csproj +++ b/src/libraries/System.Linq/tests/System.Linq.Tests.csproj @@ -1,4 +1,4 @@ - + $(NetCoreAppCurrent) true @@ -58,7 +58,6 @@ - @@ -80,10 +79,8 @@ - - + + From 8d8c34a0894c91034858b1d7ba020d7adc8d1d4f Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Wed, 5 Feb 2025 08:38:31 -0500 Subject: [PATCH 3/5] Remove check for ShuffleIterator --- src/libraries/System.Linq/src/System/Linq/Shuffle.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Shuffle.cs b/src/libraries/System.Linq/src/System/Linq/Shuffle.cs index a6340163cee367..43d115d480dd36 100644 --- a/src/libraries/System.Linq/src/System/Linq/Shuffle.cs +++ b/src/libraries/System.Linq/src/System/Linq/Shuffle.cs @@ -25,12 +25,6 @@ public static IEnumerable Shuffle(this IEnumerable so return []; } - if (source is ShuffleIterator shuffleIterator) - { - // Shuffling again won't impact anything. - return shuffleIterator; - } - return new ShuffleIterator(source); } From 5bec834f34940c286cb55e5313bbdf364b516fc2 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Wed, 5 Feb 2025 12:19:43 -0500 Subject: [PATCH 4/5] Optimize Shuffle().Take() with reservoir sampling --- .../src/System/Linq/Shuffle.SpeedOpt.cs | 199 ++++++++++++++++++ .../System.Linq/tests/ShuffleTests.cs | 162 ++++++++++++-- 2 files changed, 339 insertions(+), 22 deletions(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs index 2f91430e9114aa..daad563f2ed78b 100644 --- a/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; namespace System.Linq @@ -68,6 +69,204 @@ public override int GetCount(bool onlyIfCheap) => found = false; return default; } + + public override Iterator? Take(int count) + { + // If the source is known to have fewer elements than count, we're best off just using the default implementation. + if (_source.TryGetNonEnumeratedCount(out int sourceCount) && sourceCount <= count) + { + return base.Take(count); + } + + // Otherwise, we either don't know how many elements are in the source, or we know it's more than count. + // Try to optimize by using reservoir sampling to get a random sample of count elements. + return new TakeShuffleIterator(_source, count); + } + } + + private sealed partial class TakeShuffleIterator : Iterator + { + private readonly IEnumerable _source; + private readonly int _takeCount; + private List? _buffer; + + public TakeShuffleIterator(IEnumerable source, int takeCount) + { + Debug.Assert(source is not null); + Debug.Assert(takeCount > 0); + + _source = source; + _takeCount = takeCount; + } + + private protected override Iterator Clone() => new TakeShuffleIterator(_source, _takeCount); + + public override bool MoveNext() + { + int state = _state; + + Initialized: + if (state > 1) + { + List? buffer = _buffer; + Debug.Assert(buffer is not null); + + int i = state - 2; + if (i < buffer.Count) + { + _current = buffer[i]; + _state++; + return true; + } + } + else if (state == 1) + { + List? buffer = SampleToList(_source, _takeCount); + if (buffer is not null) + { + _buffer = buffer; + _state = state = 2; + goto Initialized; + } + } + + Dispose(); + return false; + } + + public override void Dispose() + { + _buffer = null; + base.Dispose(); + } + + /// Uses reservoir sampling to randomly select elements from . + private static List? SampleToList(IEnumerable source, int takeCount) + { + List? reservoir = null; + + if (source is IList list) + { + int listCount = list.Count; + Debug.Assert(listCount > takeCount, "Known listCount <= takeCount should have been handled by Iterator.Take override"); + + reservoir = new(takeCount); + for (int i = 0; i < listCount; i++) + { + if (i < takeCount) + { + // Fill the reservoir with the first takeCount elements from the source. + reservoir.Add(list[i]); + } + else + { + // For each subsequent element in the source, randomly replace an element in the + // reservoir with a decreasing probability. + int r = Random.Shared.Next(i + 1); + if (r < takeCount) + { + reservoir[r] = list[i]; + } + } + } + } + else + { + using IEnumerator e = source.GetEnumerator(); + if (e.MoveNext()) + { + reservoir = [e.Current]; + + // Fill the reservoir with the first takeCount elements from the source. + // If we can't fill it, just return what we get. + while (reservoir.Count < takeCount) + { + if (!e.MoveNext()) + { + goto ReturnReservoir; + } + + reservoir.Add(e.Current); + } + + // For each subsequent element in the source, randomly replace an element in the + // reservoir with a decreasing probability. + long totalElementsSeen = reservoir.Count; + while (e.MoveNext()) + { + long r = Random.Shared.NextInt64(totalElementsSeen); + if (r < reservoir.Count) + { + reservoir[(int)r] = e.Current; + } + } + } + } + + ReturnReservoir: + if (reservoir is not null) + { + // Ensure that elements in the reservoir are in random order. The sampling helped + // to ensure we got a uniform distribution from the source into the reservoir, but + // it didn't randomize the order of the reservoir itself; this is especially relevant + // to the elements initially added into the reservoir. + Random.Shared.Shuffle(CollectionsMarshal.AsSpan(reservoir)); + } + + return reservoir; + } + + public override TSource[] ToArray() => SampleToList(_source, _takeCount)?.ToArray() ?? []; + + public override List ToList() => SampleToList(_source, _takeCount) ?? []; + + public override int GetCount(bool onlyIfCheap) => + !onlyIfCheap ? Math.Min(_takeCount, _source.Count()) : + TryGetNonEnumeratedCount(_source, out int count) ? Math.Min(_takeCount, count) : + -1; + + public override TSource? TryGetFirst(out bool found) => + TryGetElementAt(0, out found); + + public override TSource? TryGetLast(out bool found) => + TryGetElementAt(0, out found); + + public override TSource? TryGetElementAt(int index, out bool found) + { + if (_source is Iterator iterator && + iterator.GetCount(onlyIfCheap: true) is int iteratorCount && + iteratorCount >= 0) + { + if ((uint)index < (uint)Math.Min(_takeCount, iteratorCount)) + { + return iterator.TryGetElementAt(Random.Shared.Next(0, iteratorCount), out found); + } + } + else if (_source is IList list) + { + int count = list.Count; + if ((uint)index < (uint)Math.Min(_takeCount, count)) + { + found = true; + return list[Random.Shared.Next(0, count)]; + } + } + else if (index >= 0) + { + TSource[] array = _source.ToArray(); + if (index < Math.Min(_takeCount, array.Length)) + { + found = true; + return array[Random.Shared.Next(0, array.Length)]; + } + } + + found = false; + return default; + } + + public override Iterator? Take(int count) => + _takeCount <= count ? this : new TakeShuffleIterator(_source, count); } } } diff --git a/src/libraries/System.Linq/tests/ShuffleTests.cs b/src/libraries/System.Linq/tests/ShuffleTests.cs index 71be4327db5934..66e0ad68da30c6 100644 --- a/src/libraries/System.Linq/tests/ShuffleTests.cs +++ b/src/libraries/System.Linq/tests/ShuffleTests.cs @@ -29,6 +29,11 @@ public void Count_ExpectedCountReturned(int length) IEnumerable shuffled = source.Shuffle(); Assert.Equal(length, shuffled.Count()); Assert.Equal(length, shuffled.Count()); + + Assert.Equal(length, source.Shuffle().Take(length).Count()); + Assert.Equal(Math.Min(length, 1), source.Shuffle().Take(1).Count()); + Assert.Equal(Math.Min(length, 1), source.Shuffle().Take(2).Take(1).Count()); + Assert.Equal(Math.Min(length, 1), source.Shuffle().Take(1).Take(2).Count()); }); } @@ -42,12 +47,16 @@ public void Enumeration_AllElementsReturned(int length) { Assert.All(CreateSources(Enumerable.Range(0, length)), source => { - List shuffled = []; - foreach (int i in source.Shuffle()) - { - shuffled.Add(i); - } + List shuffled; + + shuffled = []; + foreach (int i in source.Shuffle()) shuffled.Add(i); + Assert.Equal(length, shuffled.Count); + shuffled.Sort(); + Assert.Equal(Enumerable.Range(0, length), shuffled); + shuffled = []; + foreach (int i in source.Shuffle().Take(length)) shuffled.Add(i); Assert.Equal(length, shuffled.Count); shuffled.Sort(); Assert.Equal(Enumerable.Range(0, length), shuffled); @@ -64,7 +73,14 @@ public void ToArray_AllElementsReturned(int length) { Assert.All(CreateSources(Enumerable.Range(0, length)), source => { - int[] shuffled = source.Shuffle().ToArray(); + int[] shuffled; + + shuffled = source.Shuffle().ToArray(); + Assert.Equal(length, shuffled.Length); + Array.Sort(shuffled); + Assert.Equal(Enumerable.Range(0, length), shuffled); + + shuffled = source.Shuffle().Take(length).ToArray(); Assert.Equal(length, shuffled.Length); Array.Sort(shuffled); Assert.Equal(Enumerable.Range(0, length), shuffled); @@ -81,7 +97,14 @@ public void ToList_AllElementsReturned(int length) { Assert.All(CreateSources(Enumerable.Range(0, length)), source => { - List shuffled = source.Shuffle().ToList(); + List shuffled; + + shuffled = source.Shuffle().ToList(); + Assert.Equal(length, shuffled.Count); + shuffled.Sort(); + Assert.Equal(Enumerable.Range(0, length), shuffled); + + shuffled = source.Shuffle().Take(length).ToList(); Assert.Equal(length, shuffled.Count); shuffled.Sort(); Assert.Equal(Enumerable.Range(0, length), shuffled); @@ -95,19 +118,34 @@ public void Enumeration_ElementsAreRandomized() int length = 1000; Assert.All(CreateSources(Enumerable.Range(0, length)), source => { - List first = [], second = []; - foreach (int i in source.Shuffle()) - { - first.Add(i); - } - foreach (int i in source.Shuffle()) - { - second.Add(i); - } + List first, second; + first = []; + second = []; + foreach (int i in source.Shuffle()) first.Add(i); + foreach (int i in source.Shuffle()) second.Add(i); Assert.Equal(length, first.Count); Assert.Equal(length, second.Count); Assert.NotEqual(first, second); + + foreach (int takeCount in new[] { length - 1, length + 1 }) + { + first = []; + second = []; + foreach (int i in source.Shuffle().Take(takeCount)) first.Add(i); + foreach (int i in source.Shuffle().Take(takeCount)) second.Add(i); + Assert.Equal(Math.Min(takeCount, length), first.Count); + Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.NotEqual(first, second); + + first = []; + second = []; + foreach (int i in source.Shuffle().Take(takeCount + 1).Take(takeCount)) first.Add(i); + foreach (int i in source.Shuffle().Take(takeCount + 1).Take(takeCount)) second.Add(i); + Assert.Equal(Math.Min(takeCount, length), first.Count); + Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.NotEqual(first, second); + } }); } @@ -118,10 +156,35 @@ public void ToArray_ElementsAreRandomized() int length = 1000; Assert.All(CreateSources(Enumerable.Range(0, length)), source => { - int[] first = source.Shuffle().ToArray(), second = source.Shuffle().ToArray(); - Assert.Equal(length, first.Length); - Assert.Equal(length, second.Length); - Assert.NotEqual(first, second); + try + { + int[] first, second; + + first = source.Shuffle().ToArray(); + second = source.Shuffle().ToArray(); + Assert.Equal(length, first.Length); + Assert.Equal(length, second.Length); + Assert.NotEqual(first, second); + + foreach (int takeCount in new[] { length - 1, length + 1 }) + { + first = source.Shuffle().Take(takeCount).ToArray(); + second = source.Shuffle().Take(takeCount).ToArray(); + Assert.Equal(Math.Min(takeCount, length), first.Length); + Assert.Equal(Math.Min(takeCount, length), second.Length); + Assert.NotEqual(first, second); + + first = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToArray(); + second = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToArray(); + Assert.Equal(Math.Min(takeCount, length), first.Length); + Assert.Equal(Math.Min(takeCount, length), second.Length); + Assert.NotEqual(first, second); + } + } + catch (Exception e) + { + throw new Exception(source.GetType().ToString(), e); + } }); } @@ -132,20 +195,47 @@ public void ToList_ElementsAreRandomized() int length = 1000; Assert.All(CreateSources(Enumerable.Range(0, length)), source => { - List first = source.Shuffle().ToList(), second = source.Shuffle().ToList(); + List first, second; + + first = source.Shuffle().ToList(); + second = source.Shuffle().ToList(); Assert.Equal(length, first.Count); Assert.Equal(length, second.Count); Assert.NotEqual(first, second); + + foreach (int takeCount in new[] { length - 1, length + 1 }) + { + first = source.Shuffle().Take(takeCount).ToList(); + second = source.Shuffle().Take(takeCount).ToList(); + Assert.Equal(Math.Min(takeCount, length), first.Count); + Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.NotEqual(first, second); + + first = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToList(); + second = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToList(); + Assert.Equal(Math.Min(takeCount, length), first.Count); + Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.NotEqual(first, second); + } }); } [Fact] public void ForcedToEnumeratorDoesntEnumerate() { - var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Reverse(); // Don't insist on this behaviour, but check it's correct if it happens + + var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Shuffle(); var en = iterator as IEnumerator; Assert.False(en is not null && en.MoveNext()); + + iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Shuffle().Take(1); + en = iterator as IEnumerator; + Assert.False(en is not null && en.MoveNext()); + + iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Shuffle().Take(2).Take(1); + en = iterator as IEnumerator; + Assert.False(en is not null && en.MoveNext()); } [Fact] @@ -155,11 +245,23 @@ public void First_Last_GetElement_Invalid_ExpectedExceptions() { Assert.Throws(() => source.Shuffle().First()); Assert.Throws(() => source.Shuffle().Last()); + Assert.Throws(() => source.Shuffle().Take(1).First()); + Assert.Throws(() => source.Shuffle().Take(1).Last()); Assert.Null(source.Shuffle().ElementAtOrDefault(1)); Assert.Null(source.Shuffle().ElementAtOrDefault(-1)); Assert.Null(source.Shuffle().FirstOrDefault()); Assert.Null(source.Shuffle().LastOrDefault()); + + Assert.Null(source.Shuffle().Take(1).ElementAtOrDefault(1)); + Assert.Null(source.Shuffle().Take(1).ElementAtOrDefault(-1)); + Assert.Null(source.Shuffle().Take(1).FirstOrDefault()); + Assert.Null(source.Shuffle().Take(1).LastOrDefault()); + + Assert.Null(source.Shuffle().Take(3).Take(1).ElementAtOrDefault(1)); + Assert.Null(source.Shuffle().Take(3).Take(1).ElementAtOrDefault(-1)); + Assert.Null(source.Shuffle().Take(3).Take(1).FirstOrDefault()); + Assert.Null(source.Shuffle().Take(3).Take(1).LastOrDefault()); }); } @@ -171,6 +273,14 @@ public void First_Last_GetElement_Valid_ExpectedElements() Assert.InRange(source.Shuffle().First(), 0, 9); Assert.InRange(source.Shuffle().Last(), 0, 9); Assert.InRange(source.Shuffle().ElementAt(5), 0, 9); + + Assert.InRange(source.Shuffle().Take(1).First(), 0, 9); + Assert.InRange(source.Shuffle().Take(1).Last(), 0, 9); + Assert.InRange(source.Shuffle().Take(8).ElementAt(5), 0, 9); + + Assert.InRange(source.Shuffle().Take(3).Take(2).First(), 0, 9); + Assert.InRange(source.Shuffle().Take(3).Take(2).Last(), 0, 9); + Assert.InRange(source.Shuffle().Take(8).Take(7).ElementAt(5), 0, 9); }); } @@ -182,6 +292,14 @@ public void First_Last_GetElement_ProduceRandomElements() AssertRetry(() => source.Shuffle().First() != source.Shuffle().First()); AssertRetry(() => source.Shuffle().Last() != source.Shuffle().Last()); AssertRetry(() => source.Shuffle().ElementAt(5) != source.Shuffle().ElementAt(5)); + + AssertRetry(() => source.Shuffle().Take(10).First() != source.Shuffle().Take(10).First()); + AssertRetry(() => source.Shuffle().Take(10).Last() != source.Shuffle().Take(10).Last()); + AssertRetry(() => source.Shuffle().Take(10).ElementAt(5) != source.Shuffle().Take(10).ElementAt(5)); + + AssertRetry(() => source.Shuffle().Take(10).Take(5).First() != source.Shuffle().Take(10).Take(5).First()); + AssertRetry(() => source.Shuffle().Take(10).Take(5).Last() != source.Shuffle().Take(10).Take(5).Last()); + AssertRetry(() => source.Shuffle().Take(10).Take(5).ElementAt(3) != source.Shuffle().Take(10).Take(5).ElementAt(3)); }); static void AssertRetry(Func predicate) From c8767319180c141d26360f271925e27363c34482 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Wed, 5 Feb 2025 23:02:53 -0500 Subject: [PATCH 5/5] Improve impl and tests - Avoid ToArray in TryGetElement, instead using SampleToList - Rename TakeShufferIterator to ShuffleTakeIterator - Avoid full enumeration in ShuffleTakeIterator.GetCount() - Add distribution test --- .../src/System/Linq/Shuffle.SpeedOpt.cs | 168 +++++++++--------- .../System.Linq/tests/ShuffleTests.cs | 71 ++++++++ 2 files changed, 159 insertions(+), 80 deletions(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs index daad563f2ed78b..1b4d343e890228 100644 --- a/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Shuffle.SpeedOpt.cs @@ -58,11 +58,11 @@ public override int GetCount(bool onlyIfCheap) => } else if (index >= 0) { - TSource[] array = _source.ToArray(); - if (index < array.Length) + List? sample = ShuffleTakeIterator.SampleToList(_source, 1, out long totalElementCount); + if (sample is not null && index < totalElementCount) { found = true; - return array[Random.Shared.Next(0, array.Length)]; + return sample[0]; } } @@ -80,17 +80,17 @@ public override int GetCount(bool onlyIfCheap) => // Otherwise, we either don't know how many elements are in the source, or we know it's more than count. // Try to optimize by using reservoir sampling to get a random sample of count elements. - return new TakeShuffleIterator(_source, count); + return new ShuffleTakeIterator(_source, count); } } - private sealed partial class TakeShuffleIterator : Iterator + private sealed partial class ShuffleTakeIterator : Iterator { private readonly IEnumerable _source; private readonly int _takeCount; private List? _buffer; - public TakeShuffleIterator(IEnumerable source, int takeCount) + public ShuffleTakeIterator(IEnumerable source, int takeCount) { Debug.Assert(source is not null); Debug.Assert(takeCount > 0); @@ -99,7 +99,7 @@ public TakeShuffleIterator(IEnumerable source, int takeCount) _takeCount = takeCount; } - private protected override Iterator Clone() => new TakeShuffleIterator(_source, _takeCount); + private protected override Iterator Clone() => new ShuffleTakeIterator(_source, _takeCount); public override bool MoveNext() { @@ -121,7 +121,7 @@ public override bool MoveNext() } else if (state == 1) { - List? buffer = SampleToList(_source, _takeCount); + List? buffer = SampleToList(_source, _takeCount, out _); if (buffer is not null) { _buffer = buffer; @@ -140,8 +140,60 @@ public override void Dispose() base.Dispose(); } + public override TSource[] ToArray() => SampleToList(_source, _takeCount, out _)?.ToArray() ?? []; + + public override List ToList() => SampleToList(_source, _takeCount, out _) ?? []; + + public override int GetCount(bool onlyIfCheap) => + TryGetNonEnumeratedCount(_source, out int count) ? Math.Min(_takeCount, count) : + !onlyIfCheap ? Math.Min(_takeCount, _source.Take(_takeCount).Count()) : + -1; + + public override TSource? TryGetFirst(out bool found) => + TryGetElementAt(0, out found); + + public override TSource? TryGetLast(out bool found) => + TryGetElementAt(0, out found); + + public override TSource? TryGetElementAt(int index, out bool found) + { + if (_source is Iterator iterator && + iterator.GetCount(onlyIfCheap: true) is int iteratorCount && + iteratorCount >= 0) + { + if ((uint)index < (uint)Math.Min(_takeCount, iteratorCount)) + { + return iterator.TryGetElementAt(Random.Shared.Next(0, iteratorCount), out found); + } + } + else if (_source is IList list) + { + int count = list.Count; + if ((uint)index < (uint)Math.Min(_takeCount, count)) + { + found = true; + return list[Random.Shared.Next(0, count)]; + } + } + else if (index >= 0) + { + List? sample = SampleToList(_source, 1, out long totalElementCount); + if (sample is not null && index < Math.Min(_takeCount, totalElementCount)) + { + found = true; + return sample[0]; + } + } + + found = false; + return default; + } + + public override Iterator? Take(int count) => + _takeCount <= count ? this : new ShuffleTakeIterator(_source, count); + /// Uses reservoir sampling to randomly select elements from . - private static List? SampleToList(IEnumerable source, int takeCount) + internal static List? SampleToList(IEnumerable source, int takeCount, out long totalElementCount) { List? reservoir = null; @@ -151,38 +203,39 @@ public override void Dispose() Debug.Assert(listCount > takeCount, "Known listCount <= takeCount should have been handled by Iterator.Take override"); reservoir = new(takeCount); - for (int i = 0; i < listCount; i++) + + // Fill the reservoir with the first takeCount elements from the source. + for (int i = 0; i < takeCount; i++) { - if (i < takeCount) + reservoir.Add(list[i]); + } + + // For each subsequent element in the source, randomly replace an element in the + // reservoir with a decreasing probability. + for (int i = takeCount; i < listCount; i++) + { + int r = Random.Shared.Next(i + 1); + if (r < takeCount) { - // Fill the reservoir with the first takeCount elements from the source. - reservoir.Add(list[i]); - } - else - { - // For each subsequent element in the source, randomly replace an element in the - // reservoir with a decreasing probability. - int r = Random.Shared.Next(i + 1); - if (r < takeCount) - { - reservoir[r] = list[i]; - } + reservoir[r] = list[i]; } } + + totalElementCount = listCount; } else { using IEnumerator e = source.GetEnumerator(); if (e.MoveNext()) { - reservoir = [e.Current]; - // Fill the reservoir with the first takeCount elements from the source. // If we can't fill it, just return what we get. + reservoir = new List(Math.Min(takeCount, 4)) { e.Current }; while (reservoir.Count < takeCount) { if (!e.MoveNext()) { + totalElementCount = reservoir.Count; goto ReturnReservoir; } @@ -191,15 +244,22 @@ public override void Dispose() // For each subsequent element in the source, randomly replace an element in the // reservoir with a decreasing probability. - long totalElementsSeen = reservoir.Count; + long i = takeCount; while (e.MoveNext()) { - long r = Random.Shared.NextInt64(totalElementsSeen); - if (r < reservoir.Count) + i++; + long r = Random.Shared.NextInt64(i); + if (r < takeCount) { reservoir[(int)r] = e.Current; } } + + totalElementCount = i; + } + else + { + totalElementCount = 0; } } @@ -215,58 +275,6 @@ public override void Dispose() return reservoir; } - - public override TSource[] ToArray() => SampleToList(_source, _takeCount)?.ToArray() ?? []; - - public override List ToList() => SampleToList(_source, _takeCount) ?? []; - - public override int GetCount(bool onlyIfCheap) => - !onlyIfCheap ? Math.Min(_takeCount, _source.Count()) : - TryGetNonEnumeratedCount(_source, out int count) ? Math.Min(_takeCount, count) : - -1; - - public override TSource? TryGetFirst(out bool found) => - TryGetElementAt(0, out found); - - public override TSource? TryGetLast(out bool found) => - TryGetElementAt(0, out found); - - public override TSource? TryGetElementAt(int index, out bool found) - { - if (_source is Iterator iterator && - iterator.GetCount(onlyIfCheap: true) is int iteratorCount && - iteratorCount >= 0) - { - if ((uint)index < (uint)Math.Min(_takeCount, iteratorCount)) - { - return iterator.TryGetElementAt(Random.Shared.Next(0, iteratorCount), out found); - } - } - else if (_source is IList list) - { - int count = list.Count; - if ((uint)index < (uint)Math.Min(_takeCount, count)) - { - found = true; - return list[Random.Shared.Next(0, count)]; - } - } - else if (index >= 0) - { - TSource[] array = _source.ToArray(); - if (index < Math.Min(_takeCount, array.Length)) - { - found = true; - return array[Random.Shared.Next(0, array.Length)]; - } - } - - found = false; - return default; - } - - public override Iterator? Take(int count) => - _takeCount <= count ? this : new TakeShuffleIterator(_source, count); } } } diff --git a/src/libraries/System.Linq/tests/ShuffleTests.cs b/src/libraries/System.Linq/tests/ShuffleTests.cs index 66e0ad68da30c6..ade04ecee7a984 100644 --- a/src/libraries/System.Linq/tests/ShuffleTests.cs +++ b/src/libraries/System.Linq/tests/ShuffleTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Runtime.InteropServices; using Xunit; namespace System.Linq.Tests @@ -126,6 +127,8 @@ public void Enumeration_ElementsAreRandomized() foreach (int i in source.Shuffle()) second.Add(i); Assert.Equal(length, first.Count); Assert.Equal(length, second.Count); + Assert.Equal(length, first.Distinct().Count()); + Assert.Equal(length, second.Distinct().Count()); Assert.NotEqual(first, second); foreach (int takeCount in new[] { length - 1, length + 1 }) @@ -136,6 +139,8 @@ public void Enumeration_ElementsAreRandomized() foreach (int i in source.Shuffle().Take(takeCount)) second.Add(i); Assert.Equal(Math.Min(takeCount, length), first.Count); Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.Equal(Math.Min(takeCount, length), first.Distinct().Count()); + Assert.Equal(Math.Min(takeCount, length), second.Distinct().Count()); Assert.NotEqual(first, second); first = []; @@ -144,6 +149,8 @@ public void Enumeration_ElementsAreRandomized() foreach (int i in source.Shuffle().Take(takeCount + 1).Take(takeCount)) second.Add(i); Assert.Equal(Math.Min(takeCount, length), first.Count); Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.Equal(Math.Min(takeCount, length), first.Distinct().Count()); + Assert.Equal(Math.Min(takeCount, length), second.Distinct().Count()); Assert.NotEqual(first, second); } }); @@ -164,6 +171,8 @@ public void ToArray_ElementsAreRandomized() second = source.Shuffle().ToArray(); Assert.Equal(length, first.Length); Assert.Equal(length, second.Length); + Assert.Equal(length, first.Distinct().Count()); + Assert.Equal(length, second.Distinct().Count()); Assert.NotEqual(first, second); foreach (int takeCount in new[] { length - 1, length + 1 }) @@ -172,12 +181,16 @@ public void ToArray_ElementsAreRandomized() second = source.Shuffle().Take(takeCount).ToArray(); Assert.Equal(Math.Min(takeCount, length), first.Length); Assert.Equal(Math.Min(takeCount, length), second.Length); + Assert.Equal(Math.Min(takeCount, length), first.Distinct().Count()); + Assert.Equal(Math.Min(takeCount, length), second.Distinct().Count()); Assert.NotEqual(first, second); first = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToArray(); second = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToArray(); Assert.Equal(Math.Min(takeCount, length), first.Length); Assert.Equal(Math.Min(takeCount, length), second.Length); + Assert.Equal(Math.Min(takeCount, length), first.Distinct().Count()); + Assert.Equal(Math.Min(takeCount, length), second.Distinct().Count()); Assert.NotEqual(first, second); } } @@ -201,6 +214,8 @@ public void ToList_ElementsAreRandomized() second = source.Shuffle().ToList(); Assert.Equal(length, first.Count); Assert.Equal(length, second.Count); + Assert.Equal(length, first.Distinct().Count()); + Assert.Equal(length, second.Distinct().Count()); Assert.NotEqual(first, second); foreach (int takeCount in new[] { length - 1, length + 1 }) @@ -209,12 +224,16 @@ public void ToList_ElementsAreRandomized() second = source.Shuffle().Take(takeCount).ToList(); Assert.Equal(Math.Min(takeCount, length), first.Count); Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.Equal(Math.Min(takeCount, length), first.Distinct().Count()); + Assert.Equal(Math.Min(takeCount, length), second.Distinct().Count()); Assert.NotEqual(first, second); first = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToList(); second = source.Shuffle().Take(takeCount + 1).Take(takeCount).ToList(); Assert.Equal(Math.Min(takeCount, length), first.Count); Assert.Equal(Math.Min(takeCount, length), second.Count); + Assert.Equal(Math.Min(takeCount, length), first.Distinct().Count()); + Assert.Equal(Math.Min(takeCount, length), second.Distinct().Count()); Assert.NotEqual(first, second); } }); @@ -312,5 +331,57 @@ static void AssertRetry(Func predicate) Assert.Fail("Predicate was true for 10 iterations"); } } + + [OuterLoop] + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void ValidateShuffleTakeRandomDistribution(int mode) + { + const int InputLength = 10; + const int Iterations = 100_000; + const double Expected = Iterations / (double)InputLength; + + IEnumerable ints = Enumerable.Range(0, InputLength); + IEnumerable[] sources = [ints, ints.ToList(), ForceNotCollection(ints)]; + + foreach (IEnumerable source in sources) + { + IEnumerable selected = source.Shuffle().Take(1); + + Dictionary counts = new(); + for (int i = 0; i < Iterations; i++) + { + int value = 0; + switch (mode) + { + case 0: + using (IEnumerator e = selected.GetEnumerator()) + { + e.MoveNext(); + value = e.Current; + } + break; + + case 1: + value = selected.First(); + break; + + case 2: + value = selected.ToArray()[0]; + break; + + default: + Assert.Fail("Invalid mode"); + break; + } + + CollectionsMarshal.GetValueRefOrAddDefault(counts, value, out _)++; + } + + Assert.All(counts, kvp => Assert.InRange(kvp.Value, Expected * 0.85, Expected * 1.15)); + } + } } }