From f35b4b3c9a934404010797aa950ca3d4b48d9aa4 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 17 Aug 2022 17:53:31 +0200 Subject: [PATCH] Vectorize {Last}IndexOf{Any}{Except} without code duplication (#73768) Co-authored-by: Jan Kotas --- .../src/System/StubHelpers.cs | 4 +- .../Runtime/CompilerHelpers/InteropHelpers.cs | 8 +- .../System.Memory/tests/Span/IndexOf.T.cs | 55 + .../src/System/Array.cs | 24 +- .../System.Private.CoreLib/src/System/Enum.cs | 2 +- .../src/System/Globalization/Ordinal.cs | 4 +- .../src/System/MemoryExtensions.cs | 641 +++++-- .../src/System/SpanHelpers.Byte.cs | 1318 ++----------- .../src/System/SpanHelpers.Char.cs | 1262 +------------ .../src/System/SpanHelpers.T.cs | 1631 +++++++++++++++-- .../src/System/SpanHelpers.cs | 2 - .../src/System/String.Manipulation.cs | 2 +- .../src/System/String.Searching.cs | 18 +- .../src/System/String.cs | 34 +- 14 files changed, 2126 insertions(+), 2879 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs index 595f5482fca5b..bc731b674d2e9 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs @@ -176,7 +176,7 @@ internal static unsafe void ConvertFixedToNative(int flags, string strManaged, I internal static unsafe string ConvertFixedToManaged(IntPtr cstr, int length) { - int end = SpanHelpers.IndexOf(ref *(byte*)cstr, 0, length); + int end = new ReadOnlySpan((byte*)cstr, length).IndexOf((byte)0); if (end >= 0) { length = end; @@ -450,7 +450,7 @@ internal static unsafe void ConvertToNative(string? strManaged, IntPtr nativeHom internal static unsafe string ConvertToManaged(IntPtr nativeHome, int length) { - int end = SpanHelpers.IndexOf(ref *(char*)nativeHome, '\0', length); + int end = new ReadOnlySpan((char*)nativeHome, length).IndexOf('\0'); if (end >= 0) { length = end; diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs index 4e8bfcd492dde..8e383b44d8503 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs @@ -55,8 +55,8 @@ internal static unsafe void StringToByValAnsiString(string str, byte* pNative, i public static unsafe string ByValAnsiStringToString(byte* buffer, int length) { - int end = SpanHelpers.IndexOf(ref *(byte*)buffer, 0, length); - if (end != -1) + int end = new ReadOnlySpan(buffer, length).IndexOf((byte)0); + if (end >= 0) { length = end; } @@ -77,8 +77,8 @@ internal static unsafe void StringToUnicodeFixedArray(string str, ushort* buffer internal static unsafe string UnicodeToStringFixedArray(ushort* buffer, int length) { - int end = SpanHelpers.IndexOf(ref *(char*)buffer, '\0', length); - if (end != -1) + int end = new ReadOnlySpan(buffer, length).IndexOf('\0'); + if (end >= 0) { length = end; } diff --git a/src/libraries/System.Memory/tests/Span/IndexOf.T.cs b/src/libraries/System.Memory/tests/Span/IndexOf.T.cs index 67b6e896bd59f..e2fb8a0e64e48 100644 --- a/src/libraries/System.Memory/tests/Span/IndexOf.T.cs +++ b/src/libraries/System.Memory/tests/Span/IndexOf.T.cs @@ -192,5 +192,60 @@ public static void IndexOfNull_String(string[] spanInput, int expected) Span theStrings = spanInput; Assert.Equal(expected, theStrings.IndexOf((string)null)); } + + [Fact] + public static void NotBitwiseEquatableUsesCustomIEquatableImplementationForActualComparison() + { + const byte Ten = 10, NotTen = 11; + for (int length = 1; length < 100; length++) + { + TwoBytes[] array = new TwoBytes[length]; + for (int i = 0; i < length; i++) + { + array[i] = new TwoBytes(Ten, (byte)i); + } + + Span span = new Span(array); + ReadOnlySpan ros = new ReadOnlySpan(array); + + ReadOnlySpan noMatch2 = new TwoBytes[2] { new TwoBytes(10, NotTen), new TwoBytes(10, NotTen) }; + Assert.Equal(-1, span.IndexOfAny(noMatch2)); + Assert.Equal(-1, ros.IndexOfAny(noMatch2)); + Assert.Equal(-1, span.LastIndexOfAny(noMatch2)); + Assert.Equal(-1, ros.LastIndexOfAny(noMatch2)); + + ReadOnlySpan noMatch3 = new TwoBytes[3] { new TwoBytes(10, NotTen), new TwoBytes(10, NotTen), new TwoBytes(10, NotTen) }; + Assert.Equal(-1, span.IndexOfAny(noMatch3)); + Assert.Equal(-1, ros.IndexOfAny(noMatch3)); + Assert.Equal(-1, span.LastIndexOfAny(noMatch3)); + Assert.Equal(-1, ros.LastIndexOfAny(noMatch3)); + + ReadOnlySpan match2 = new TwoBytes[2] { new TwoBytes(0, Ten), new TwoBytes(0, Ten) }; + Assert.Equal(0, span.IndexOfAny(match2)); + Assert.Equal(0, ros.IndexOfAny(match2)); + Assert.Equal(0, span.LastIndexOfAny(match2)); + Assert.Equal(0, ros.LastIndexOfAny(match2)); + + ReadOnlySpan match3 = new TwoBytes[3] { new TwoBytes(0, Ten), new TwoBytes(0, Ten), new TwoBytes(0, Ten) }; + Assert.Equal(0, span.IndexOfAny(match3)); + Assert.Equal(0, ros.IndexOfAny(match3)); + Assert.Equal(0, span.LastIndexOfAny(match3)); + Assert.Equal(0, ros.LastIndexOfAny(match3)); + } + } + + private readonly struct TwoBytes : IEquatable + { + private readonly byte _first, _second; + + public TwoBytes(byte first, byte second) + { + _first = first; + _second = second; + } + + // it compares different fields on purpose + public bool Equals(TwoBytes other) => _first == other._second && _second == other._first; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs index 32435ec4367ec..b665d3094e1dc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Array.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs @@ -1328,17 +1328,17 @@ public static int IndexOf(T[] array, T value, int startIndex, int count) { if (Unsafe.SizeOf() == sizeof(byte)) { - int result = SpanHelpers.IndexOf( + int result = SpanHelpers.IndexOfValueType( ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), startIndex), Unsafe.As(ref value), count); return (result >= 0 ? startIndex : 0) + result; } - else if (Unsafe.SizeOf() == sizeof(char)) + else if (Unsafe.SizeOf() == sizeof(short)) { - int result = SpanHelpers.IndexOf( - ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), startIndex), - Unsafe.As(ref value), + int result = SpanHelpers.IndexOfValueType( + ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), startIndex), + Unsafe.As(ref value), count); return (result >= 0 ? startIndex : 0) + result; } @@ -1586,19 +1586,19 @@ public static int LastIndexOf(T[] array, T value, int startIndex, int count) if (Unsafe.SizeOf() == sizeof(byte)) { int endIndex = startIndex - count + 1; - int result = SpanHelpers.LastIndexOf( + int result = SpanHelpers.LastIndexOfValueType( ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), endIndex), Unsafe.As(ref value), count); return (result >= 0 ? endIndex : 0) + result; } - else if (Unsafe.SizeOf() == sizeof(char)) + else if (Unsafe.SizeOf() == sizeof(short)) { int endIndex = startIndex - count + 1; - int result = SpanHelpers.LastIndexOf( - ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), endIndex), - Unsafe.As(ref value), + int result = SpanHelpers.LastIndexOfValueType( + ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), endIndex), + Unsafe.As(ref value), count); return (result >= 0 ? endIndex : 0) + result; @@ -1606,7 +1606,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)) else if (Unsafe.SizeOf() == sizeof(int)) { int endIndex = startIndex - count + 1; - int result = SpanHelpers.LastIndexOf( + int result = SpanHelpers.LastIndexOfValueType( ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), endIndex), Unsafe.As(ref value), count); @@ -1616,7 +1616,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), else if (Unsafe.SizeOf() == sizeof(long)) { int endIndex = startIndex - count + 1; - int result = SpanHelpers.LastIndexOf( + int result = SpanHelpers.LastIndexOfValueType( ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)), endIndex), Unsafe.As(ref value), count); diff --git a/src/libraries/System.Private.CoreLib/src/System/Enum.cs b/src/libraries/System.Private.CoreLib/src/System/Enum.cs index 89404415b5e17..cd4535867b033 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Enum.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Enum.cs @@ -436,7 +436,7 @@ private static int FindDefinedIndex(ulong[] ulValues, ulong ulValue) int ulValuesLength = ulValues.Length; ref ulong start = ref MemoryMarshal.GetArrayDataReference(ulValues); return ulValuesLength <= NumberOfValuesThreshold ? - SpanHelpers.IndexOf(ref start, ulValue, ulValuesLength) : + SpanHelpers.IndexOfValueType(ref Unsafe.As(ref start), (long)ulValue, ulValuesLength) : SpanHelpers.BinarySearch(ref start, ulValuesLength, ulValue); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index ef51de122f717..76d2b06b20b90 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -264,8 +264,8 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly { // Do a quick search for the first element of "value". int relativeIndex = isLetter ? - SpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : - SpanHelpers.IndexOf(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength); + SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : + SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength); if (relativeIndex < 0) { break; diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 658a464d6d3d2..5ae66529eb20d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -7,7 +7,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; namespace System { @@ -269,28 +268,33 @@ public static bool Contains(this Span span, T value) where T : IEquatable< if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.Contains( + { + return SpanHelpers.ContainsValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.Contains( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value), + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.ContainsValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), span.Length); - - if (Unsafe.SizeOf() == sizeof(int)) - return 0 <= SpanHelpers.IndexOfValueType( + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + return SpanHelpers.ContainsValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); - - if (Unsafe.SizeOf() == sizeof(long)) - return 0 <= SpanHelpers.IndexOfValueType( + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + return SpanHelpers.ContainsValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); + } } return SpanHelpers.Contains(ref MemoryMarshal.GetReference(span), value, span.Length); @@ -308,28 +312,33 @@ public static bool Contains(this ReadOnlySpan span, T value) where T : IEq if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.Contains( + { + return SpanHelpers.ContainsValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.Contains( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value), + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.ContainsValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), span.Length); - - if (Unsafe.SizeOf() == sizeof(int)) - return 0 <= SpanHelpers.IndexOfValueType( + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + return SpanHelpers.ContainsValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); - - if (Unsafe.SizeOf() == sizeof(long)) - return 0 <= SpanHelpers.IndexOfValueType( + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + return SpanHelpers.ContainsValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); + } } return SpanHelpers.Contains(ref MemoryMarshal.GetReference(span), value, span.Length); @@ -346,15 +355,15 @@ public static int IndexOf(this Span span, T value) where T : IEquatable if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.IndexOf( + return SpanHelpers.IndexOfValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.IndexOf( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value), + if (Unsafe.SizeOf() == sizeof(short)) + return SpanHelpers.IndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), span.Length); if (Unsafe.SizeOf() == sizeof(int)) @@ -412,16 +421,33 @@ public static int LastIndexOf(this Span span, T value) where T : IEquatabl if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.LastIndexOf( + { + return SpanHelpers.LastIndexOfValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.LastIndexOf( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value), + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + return SpanHelpers.LastIndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), span.Length); + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + return SpanHelpers.LastIndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } } return SpanHelpers.LastIndexOf(ref MemoryMarshal.GetReference(span), value, span.Length); @@ -503,7 +529,7 @@ public static int IndexOfAnyExcept(this Span span, ReadOnlySpan values) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int IndexOfAnyExcept(this ReadOnlySpan span, T value) where T : IEquatable? { - if (RuntimeHelpers.IsBitwiseEquatable()) + if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) { if (Unsafe.SizeOf() == sizeof(byte)) { @@ -512,36 +538,34 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); } - - if (Unsafe.SizeOf() == sizeof(short)) + else if (Unsafe.SizeOf() == sizeof(short)) { return SpanHelpers.IndexOfAnyExceptValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); } - - if (Unsafe.SizeOf() == sizeof(int)) + else if (Unsafe.SizeOf() == sizeof(int)) { return SpanHelpers.IndexOfAnyExceptValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); } - - if (Unsafe.SizeOf() == sizeof(long)) + else { + Debug.Assert(Unsafe.SizeOf() == sizeof(long)); + return SpanHelpers.IndexOfAnyExceptValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); } } - - return SpanHelpers.IndexOfAnyExcept( - ref MemoryMarshal.GetReference(span), - value, - span.Length); + else + { + return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length); + } } /// Searches for the first index of any value other than the specified or . @@ -553,18 +577,30 @@ ref MemoryMarshal.GetReference(span), /// The index in the span of the first occurrence of any value other than and . /// If all of the values are or , returns -1. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1) where T : IEquatable? { - for (int i = 0; i < span.Length; i++) + if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) { - if (!EqualityComparer.Default.Equals(span[i], value0) && - !EqualityComparer.Default.Equals(span[i], value1)) + if (Unsafe.SizeOf() == sizeof(byte)) { - return i; + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); } } - return -1; + return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, span.Length); } /// Searches for the first index of any value other than the specified , , or . @@ -577,25 +613,38 @@ public static int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T val /// The index in the span of the first occurrence of any value other than , , and . /// If all of the values are , , and , returns -1. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2) where T : IEquatable? { - for (int i = 0; i < span.Length; i++) + if (RuntimeHelpers.IsBitwiseEquatable()) { - if (!EqualityComparer.Default.Equals(span[i], value0) && - !EqualityComparer.Default.Equals(span[i], value1) && - !EqualityComparer.Default.Equals(span[i], value2)) + if (Unsafe.SizeOf() == sizeof(byte)) { - return i; + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.IndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); } } - return -1; + return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2, T value3) where T : IEquatable? + private static int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2, T value3) where T : IEquatable? { - if (RuntimeHelpers.IsBitwiseEquatable()) + if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) { if (Unsafe.SizeOf() == sizeof(byte)) { @@ -617,40 +666,9 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value3), span.Length); } - else if (Unsafe.SizeOf() == sizeof(int)) - { - return SpanHelpers.IndexOfAnyExceptValueType( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - Unsafe.As(ref value2), - Unsafe.As(ref value3), - span.Length); - } - else if (Unsafe.SizeOf() == sizeof(long)) - { - return SpanHelpers.IndexOfAnyExceptValueType( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - Unsafe.As(ref value2), - Unsafe.As(ref value3), - span.Length); - } - } - - for (int i = 0; i < span.Length; i++) - { - if (!EqualityComparer.Default.Equals(span[i], value0) && - !EqualityComparer.Default.Equals(span[i], value1) && - !EqualityComparer.Default.Equals(span[i], value2) && - !EqualityComparer.Default.Equals(span[i], value3)) - { - return i; - } } - return -1; + return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, value3, span.Length); } /// Searches for the first index of any value other than the specified . @@ -680,7 +698,7 @@ public static int IndexOfAnyExcept(this ReadOnlySpan span, ReadOnlySpan case 3: return IndexOfAnyExcept(span, values[0], values[1], values[2]); - case 4: // common for searching whitespaces + case 4: return IndexOfAnyExcept(span, values[0], values[1], values[2], values[3]); default: @@ -751,17 +769,46 @@ public static int LastIndexOfAnyExcept(this Span span, ReadOnlySpan val /// The index in the span of the last occurrence of any value other than . /// If all of the values are , returns -1. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int LastIndexOfAnyExcept(this ReadOnlySpan span, T value) where T : IEquatable? { - for (int i = span.Length - 1; i >= 0; i--) + if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) { - if (!EqualityComparer.Default.Equals(span[i], value)) + if (Unsafe.SizeOf() == sizeof(byte)) { - return i; + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); } - } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + else + { + Debug.Assert(Unsafe.SizeOf() == sizeof(long)); - return -1; + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + } + else + { + return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length); + } } /// Searches for the last index of any value other than the specified or . @@ -773,18 +820,30 @@ public static int LastIndexOfAnyExcept(this ReadOnlySpan span, T value) wh /// The index in the span of the last occurrence of any value other than and . /// If all of the values are or , returns -1. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1) where T : IEquatable? { - for (int i = span.Length - 1; i >= 0; i--) + if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) { - if (!EqualityComparer.Default.Equals(span[i], value0) && - !EqualityComparer.Default.Equals(span[i], value1)) + if (Unsafe.SizeOf() == sizeof(byte)) { - return i; + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); } } - return -1; + return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, span.Length); } /// Searches for the last index of any value other than the specified , , or . @@ -797,19 +856,62 @@ public static int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T /// The index in the span of the last occurrence of any value other than , , and . /// If all of the values are , , and , returns -1. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2) where T : IEquatable? { - for (int i = span.Length - 1; i >= 0; i--) + if (RuntimeHelpers.IsBitwiseEquatable()) { - if (!EqualityComparer.Default.Equals(span[i], value0) && - !EqualityComparer.Default.Equals(span[i], value1) && - !EqualityComparer.Default.Equals(span[i], value2)) + if (Unsafe.SizeOf() == sizeof(byte)) { - return i; + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); } } - return -1; + return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2, T value3) where T : IEquatable? + { + if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + Unsafe.As(ref value3), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyExceptValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + Unsafe.As(ref value3), + span.Length); + } + } + + return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, value3, span.Length); } /// Searches for the last index of any value other than the specified . @@ -840,6 +942,9 @@ public static int LastIndexOfAnyExcept(this ReadOnlySpan span, ReadOnlySpa case 3: return LastIndexOfAnyExcept(span, values[0], values[1], values[2]); + case 4: + return LastIndexOfAnyExcept(span, values[0], values[1], values[2], values[3]); + default: for (int i = span.Length - 1; i >= 0; i--) { @@ -912,15 +1017,27 @@ public static int IndexOf(this ReadOnlySpan span, T value) where T : IEqua if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.IndexOf( + return SpanHelpers.IndexOfValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value), span.Length); - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.IndexOf( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value), + if (Unsafe.SizeOf() == sizeof(short)) + return SpanHelpers.IndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + + if (Unsafe.SizeOf() == sizeof(int)) + return SpanHelpers.IndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + + if (Unsafe.SizeOf() == sizeof(long)) + return SpanHelpers.IndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), span.Length); } @@ -966,16 +1083,33 @@ public static int LastIndexOf(this ReadOnlySpan span, T value) where T : I if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.LastIndexOf( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value), - span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.LastIndexOf( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value), + { + return SpanHelpers.LastIndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), span.Length); + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + return SpanHelpers.LastIndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + return SpanHelpers.LastIndexOfValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value), + span.Length); + } } return SpanHelpers.LastIndexOf(ref MemoryMarshal.GetReference(span), value, span.Length); @@ -1024,18 +1158,21 @@ public static int IndexOfAny(this Span span, T value0, T value1) where T : if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.IndexOfAny( + { + return SpanHelpers.IndexOfAnyValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value0), Unsafe.As(ref value1), span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.IndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.IndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), span.Length); + } } return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length); @@ -1054,20 +1191,23 @@ public static int IndexOfAny(this Span span, T value0, T value1, T value2) if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.IndexOfAny( + { + return SpanHelpers.IndexOfAnyValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value0), Unsafe.As(ref value1), Unsafe.As(ref value2), span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.IndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - Unsafe.As(ref value2), + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.IndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), span.Length); + } } return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length); @@ -1094,18 +1234,21 @@ public static int IndexOfAny(this ReadOnlySpan span, T value0, T value1) w if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.IndexOfAny( + { + return SpanHelpers.IndexOfAnyValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value0), Unsafe.As(ref value1), span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.IndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.IndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), span.Length); + } } return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length); @@ -1124,20 +1267,23 @@ public static int IndexOfAny(this ReadOnlySpan span, T value0, T value1, T if (RuntimeHelpers.IsBitwiseEquatable()) { if (Unsafe.SizeOf() == sizeof(byte)) - return SpanHelpers.IndexOfAny( + { + return SpanHelpers.IndexOfAnyValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref value0), Unsafe.As(ref value1), Unsafe.As(ref value2), span.Length); - - if (Unsafe.SizeOf() == sizeof(char)) - return SpanHelpers.IndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - Unsafe.As(ref value2), + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.IndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), span.Length); + } } return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length); @@ -1158,7 +1304,7 @@ public static int IndexOfAny(this ReadOnlySpan span, ReadOnlySpan value ref byte valueRef = ref Unsafe.As(ref MemoryMarshal.GetReference(values)); if (values.Length == 2) { - return SpanHelpers.IndexOfAny( + return SpanHelpers.IndexOfAnyValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), valueRef, Unsafe.Add(ref valueRef, 1), @@ -1166,7 +1312,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), } else if (values.Length == 3) { - return SpanHelpers.IndexOfAny( + return SpanHelpers.IndexOfAnyValueType( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), valueRef, Unsafe.Add(ref valueRef, 1), @@ -1175,30 +1321,27 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), } } - if (Unsafe.SizeOf() == sizeof(char)) + if (Unsafe.SizeOf() == sizeof(short)) { - ref char spanRef = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); - ref char valueRef = ref Unsafe.As(ref MemoryMarshal.GetReference(values)); + ref short spanRef = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); + ref short valueRef = ref Unsafe.As(ref MemoryMarshal.GetReference(values)); switch (values.Length) { case 0: return -1; case 1: - return SpanHelpers.IndexOf( - ref spanRef, - valueRef, - span.Length); + return SpanHelpers.IndexOfValueType(ref spanRef, valueRef, span.Length); case 2: - return SpanHelpers.IndexOfAny( + return SpanHelpers.IndexOfAnyValueType( ref spanRef, valueRef, Unsafe.Add(ref valueRef, 1), span.Length); case 3: - return SpanHelpers.IndexOfAny( + return SpanHelpers.IndexOfAnyValueType( ref spanRef, valueRef, Unsafe.Add(ref valueRef, 1), @@ -1206,7 +1349,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); case 4: - return SpanHelpers.IndexOfAny( + return SpanHelpers.IndexOfAnyValueType( ref spanRef, valueRef, Unsafe.Add(ref valueRef, 1), @@ -1215,7 +1358,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); case 5: - return SpanHelpers.IndexOfAny( + return SpanHelpers.IndexOfAnyValueType( ref spanRef, valueRef, Unsafe.Add(ref valueRef, 1), @@ -1225,7 +1368,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); default: - return IndexOfAnyProbabilistic(ref spanRef, span.Length, ref valueRef, values.Length); + return IndexOfAnyProbabilistic(ref Unsafe.As(ref spanRef), span.Length, ref Unsafe.As(ref valueRef), values.Length); } } } @@ -1272,12 +1415,25 @@ private static unsafe int IndexOfAnyProbabilistic(ref char searchSpace, int sear [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int LastIndexOfAny(this Span span, T value0, T value1) where T : IEquatable? { - if (Unsafe.SizeOf() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable()) - return SpanHelpers.LastIndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - span.Length); + if (RuntimeHelpers.IsBitwiseEquatable()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); + } + } return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length); } @@ -1292,13 +1448,27 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int LastIndexOfAny(this Span span, T value0, T value1, T value2) where T : IEquatable? { - if (Unsafe.SizeOf() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable()) - return SpanHelpers.LastIndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - Unsafe.As(ref value2), - span.Length); + if (RuntimeHelpers.IsBitwiseEquatable()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); + } + } return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length); } @@ -1330,12 +1500,25 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int LastIndexOfAny(this ReadOnlySpan span, T value0, T value1) where T : IEquatable? { - if (Unsafe.SizeOf() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable()) - return SpanHelpers.LastIndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - span.Length); + if (RuntimeHelpers.IsBitwiseEquatable()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + span.Length); + } + } return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length); } @@ -1350,13 +1533,27 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int LastIndexOfAny(this ReadOnlySpan span, T value0, T value1, T value2) where T : IEquatable? { - if (Unsafe.SizeOf() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable()) - return SpanHelpers.LastIndexOfAny( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref value0), - Unsafe.As(ref value1), - Unsafe.As(ref value2), - span.Length); + if (RuntimeHelpers.IsBitwiseEquatable()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref value0), + Unsafe.As(ref value1), + Unsafe.As(ref value2), + span.Length); + } + } return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length); } @@ -1371,28 +1568,66 @@ public static int LastIndexOfAny(this ReadOnlySpan span, ReadOnlySpan v { if (RuntimeHelpers.IsBitwiseEquatable()) { - if (Unsafe.SizeOf() == sizeof(char)) + if (Unsafe.SizeOf() == sizeof(byte)) { + ref byte valueRef = ref Unsafe.As(ref MemoryMarshal.GetReference(values)); + if (values.Length == 2) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + valueRef, + Unsafe.Add(ref valueRef, 1), + span.Length); + } + else if (values.Length == 3) + { + return SpanHelpers.LastIndexOfAnyValueType( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + valueRef, + Unsafe.Add(ref valueRef, 1), + Unsafe.Add(ref valueRef, 2), + span.Length); + } + } + + if (Unsafe.SizeOf() == sizeof(short)) + { + ref short spanRef = ref Unsafe.As(ref MemoryMarshal.GetReference(span)); + ref short valueRef = ref Unsafe.As(ref MemoryMarshal.GetReference(values)); switch (values.Length) { case 0: return -1; case 1: - return LastIndexOf(span, values[0]); + return SpanHelpers.LastIndexOfValueType(ref spanRef, valueRef, span.Length); case 2: - return LastIndexOfAny(span, values[0], values[1]); + return SpanHelpers.LastIndexOfAnyValueType( + ref spanRef, + valueRef, + Unsafe.Add(ref valueRef, 1), + span.Length); case 3: - return LastIndexOfAny(span, values[0], values[1], values[2]); + return SpanHelpers.LastIndexOfAnyValueType( + ref spanRef, + valueRef, + Unsafe.Add(ref valueRef, 1), + Unsafe.Add(ref valueRef, 2), + span.Length); + + case 4: + return SpanHelpers.LastIndexOfAnyValueType( + ref spanRef, + valueRef, + Unsafe.Add(ref valueRef, 1), + Unsafe.Add(ref valueRef, 2), + Unsafe.Add(ref valueRef, 3), + span.Length); default: - return LastIndexOfAnyProbabilistic( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - span.Length, - ref Unsafe.As(ref MemoryMarshal.GetReference(values)), - values.Length); + return LastIndexOfAnyProbabilistic(ref Unsafe.As(ref spanRef), span.Length, ref Unsafe.As(ref valueRef), values.Length); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 5029e01a3161d..61565cfd27db3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2,10 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; namespace System @@ -22,7 +22,7 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte int valueTailLength = valueLength - 1; if (valueTailLength == 0) - return IndexOf(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain IndexOf + return IndexOfValueType(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain IndexOf nint offset = 0; byte valueHead = value; @@ -38,7 +38,7 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte while (remainingSearchSpaceLength > 0) { // Do a quick search for the first element of "value". - int relativeIndex = IndexOf(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength); + int relativeIndex = IndexOfValueType(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength); if (relativeIndex < 0) break; @@ -197,7 +197,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b int valueTailLength = valueLength - 1; if (valueTailLength == 0) - return LastIndexOf(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf + return LastIndexOfValueType(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf int offset = 0; byte valueHead = value; @@ -217,7 +217,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b break; // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there. // Do a quick search for the first element of "value". - int relativeIndex = LastIndexOf(ref searchSpace, valueHead, remainingSearchSpaceLength); + int relativeIndex = LastIndexOfValueType(ref searchSpace, valueHead, remainingSearchSpaceLength); if (relativeIndex < 0) break; @@ -333,828 +333,52 @@ ref Unsafe.Add(ref searchSpace, offset + bitPos), } } - // Adapted from IndexOf(...) - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static bool Contains(ref byte searchSpace, byte value, int length) + [DoesNotReturn] + private static void ThrowMustBeNullTerminatedString() { - Debug.Assert(length >= 0); - - uint uValue = value; // Use uint for comparisons to avoid unnecessary 8->32 extensions - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (uint)length; - - if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) - { - lengthToExamine = UnalignedCountVector(ref searchSpace); - } - - while (lengthToExamine >= 8) - { - lengthToExamine -= 8; - ref byte start = ref Unsafe.AddByteOffset(ref searchSpace, offset); - - if (uValue == Unsafe.AddByteOffset(ref start, 0) || - uValue == Unsafe.AddByteOffset(ref start, 1) || - uValue == Unsafe.AddByteOffset(ref start, 2) || - uValue == Unsafe.AddByteOffset(ref start, 3) || - uValue == Unsafe.AddByteOffset(ref start, 4) || - uValue == Unsafe.AddByteOffset(ref start, 5) || - uValue == Unsafe.AddByteOffset(ref start, 6) || - uValue == Unsafe.AddByteOffset(ref start, 7)) - { - goto Found; - } - - offset += 8; - } - - if (lengthToExamine >= 4) - { - lengthToExamine -= 4; - ref byte start = ref Unsafe.AddByteOffset(ref searchSpace, offset); - - if (uValue == Unsafe.AddByteOffset(ref start, 0) || - uValue == Unsafe.AddByteOffset(ref start, 1) || - uValue == Unsafe.AddByteOffset(ref start, 2) || - uValue == Unsafe.AddByteOffset(ref start, 3)) - { - goto Found; - } - - offset += 4; - } - - while (lengthToExamine > 0) - { - lengthToExamine--; - - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) - goto Found; - - offset++; - } - - if (Vector.IsHardwareAccelerated && (offset < (uint)length)) - { - lengthToExamine = ((uint)length - offset) & (nuint)~(Vector.Count - 1); - - Vector values = new(value); - Vector matches; - - while (offset < lengthToExamine) - { - matches = Vector.Equals(values, LoadVector(ref searchSpace, offset)); - if (matches == Vector.Zero) - { - offset += (nuint)Vector.Count; - continue; - } - - goto Found; - } - - // The total length is at least Vector.Count, so instead of falling back to a - // sequential scan for the remainder, we check the vector read from the end -- note: unaligned read necessary. - // We do this only if at least one element is left. - if (offset < (uint)length) - { - offset = (uint)(length - Vector.Count); - matches = Vector.Equals(values, LoadVector(ref searchSpace, offset)); - if (matches != Vector.Zero) - { - goto Found; - } - } - } - - return false; - - Found: - return true; + throw new ArgumentException(SR.Arg_MustBeNullTerminatedString); } + // IndexOfNullByte processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator. + // This behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it. [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe int IndexOf(ref byte searchSpace, byte value, int length) + internal static unsafe int IndexOfNullByte(ref byte searchSpace) { - Debug.Assert(length >= 0); + const int Length = int.MaxValue; - uint uValue = value; // Use uint for comparisons to avoid unnecessary 8->32 extensions + const uint uValue = 0; // Use uint for comparisons to avoid unnecessary 8->32 extensions nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; + nuint lengthToExamine = (nuint)(uint)Length; if (Vector128.IsHardwareAccelerated) { // Avx2 branch also operates on Sse2 sizes, so check is combined. - if (length >= Vector128.Count * 2) - { - lengthToExamine = UnalignedCountVector128(ref searchSpace); - } - } - else if (Vector.IsHardwareAccelerated) - { - if (length >= Vector.Count * 2) - { - lengthToExamine = UnalignedCountVector(ref searchSpace); - } - } - SequentialScan: - while (lengthToExamine >= 8) - { - lengthToExamine -= 8; - - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) - goto Found; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1)) - goto Found1; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2)) - goto Found2; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3)) - goto Found3; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 4)) - goto Found4; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 5)) - goto Found5; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 6)) - goto Found6; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 7)) - goto Found7; - - offset += 8; - } - - if (lengthToExamine >= 4) - { - lengthToExamine -= 4; - - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) - goto Found; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1)) - goto Found1; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2)) - goto Found2; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3)) - goto Found3; - - offset += 4; - } - - while (lengthToExamine > 0) - { - lengthToExamine -= 1; - - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) - goto Found; - - offset += 1; - } - - // We get past SequentialScan only if IsHardwareAccelerated is true; and remain length is greater than Vector length. - // However, we still have the redundant check to allow the JIT to see that the code is unreachable and eliminate it when the platform does not - // have hardware accelerated. After processing Vector lengths we return to SequentialScan to finish any remaining. - if (Vector256.IsHardwareAccelerated) - { - if (offset < (nuint)(uint)length) - { - if ((((nuint)(uint)Unsafe.AsPointer(ref searchSpace) + offset) & (nuint)(Vector256.Count - 1)) != 0) - { - // Not currently aligned to Vector256 (is aligned to Vector128); this can cause a problem for searches - // with no upper bound e.g. String.strlen. - // Start with a check on Vector128 to align to Vector256, before moving to processing Vector256. - // This ensures we do not fault across memory pages while searching for an end of string. - Vector128 values = Vector128.Create(value); - Vector128 search = Vector128.LoadUnsafe(ref searchSpace, offset); - - // Same method as below - uint matches = Vector128.Equals(values, search).ExtractMostSignificantBits(); - if (matches == 0) - { - // Zero flags set so no matches - offset += (nuint)Vector128.Count; - } - else - { - // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); - } - } - - lengthToExamine = GetByteVector256SpanLength(offset, length); - if (lengthToExamine > offset) - { - Vector256 values = Vector256.Create(value); - do - { - Vector256 search = Vector256.LoadUnsafe(ref searchSpace, offset); - uint matches = Vector256.Equals(values, search).ExtractMostSignificantBits(); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) - { - // Zero flags set so no matches - offset += (nuint)Vector256.Count; - continue; - } - - // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); - } while (lengthToExamine > offset); - } - - lengthToExamine = GetByteVector128SpanLength(offset, length); - if (lengthToExamine > offset) - { - Vector128 values = Vector128.Create(value); - Vector128 search = Vector128.LoadUnsafe(ref searchSpace, offset); - - // Same method as above - uint matches = Vector128.Equals(values, search).ExtractMostSignificantBits(); - if (matches == 0) - { - // Zero flags set so no matches - offset += (nuint)Vector128.Count; - } - else - { - // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); - } - } - - if (offset < (nuint)(uint)length) - { - lengthToExamine = ((nuint)(uint)length - offset); - goto SequentialScan; - } - } - } - else if (Vector128.IsHardwareAccelerated) - { - if (offset < (nuint)(uint)length) - { - lengthToExamine = GetByteVector128SpanLength(offset, length); - - Vector128 values = Vector128.Create(value); - while (lengthToExamine > offset) - { - Vector128 search = Vector128.LoadUnsafe(ref searchSpace, offset); - - // Same method as above - Vector128 compareResult = Vector128.Equals(values, search); - if (compareResult == Vector128.Zero) - { - // Zero flags set so no matches - offset += (nuint)Vector128.Count; - continue; - } - - // Find bitflag offset of first match and add to current offset - uint matches = compareResult.ExtractMostSignificantBits(); - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); - } - - if (offset < (nuint)(uint)length) - { - lengthToExamine = ((nuint)(uint)length - offset); - goto SequentialScan; - } - } + lengthToExamine = UnalignedCountVector128(ref searchSpace); } else if (Vector.IsHardwareAccelerated) { - if (offset < (nuint)(uint)length) - { - lengthToExamine = GetByteVectorSpanLength(offset, length); - - Vector values = new Vector(value); - - while (lengthToExamine > offset) - { - var matches = Vector.Equals(values, LoadVector(ref searchSpace, offset)); - if (Vector.Zero.Equals(matches)) - { - offset += (nuint)Vector.Count; - continue; - } - - // Find offset of first match and add to current offset - return (int)offset + LocateFirstFoundByte(matches); - } - - if (offset < (nuint)(uint)length) - { - lengthToExamine = ((nuint)(uint)length - offset); - goto SequentialScan; - } - } - } - return -1; - Found: // Workaround for https://github.com/dotnet/runtime/issues/8795 - return (int)offset; - Found1: - return (int)(offset + 1); - Found2: - return (int)(offset + 2); - Found3: - return (int)(offset + 3); - Found4: - return (int)(offset + 4); - Found5: - return (int)(offset + 5); - Found6: - return (int)(offset + 6); - Found7: - return (int)(offset + 7); - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static int LastIndexOf(ref byte searchSpace, byte value, int length) - { - Debug.Assert(length >= 0); - - uint uValue = value; // Use uint for comparisons to avoid unnecessary 8->32 extensions - nuint offset = (nuint)(uint)length; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) - { - lengthToExamine = UnalignedCountVectorFromEnd(ref searchSpace, length); + lengthToExamine = UnalignedCountVector(ref searchSpace); } SequentialScan: while (lengthToExamine >= 8) { lengthToExamine -= 8; - offset -= 8; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 7)) - goto Found7; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 6)) - goto Found6; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 5)) - goto Found5; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 4)) - goto Found4; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3)) - goto Found3; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2)) - goto Found2; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1)) - goto Found1; if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) goto Found; - } - - if (lengthToExamine >= 4) - { - lengthToExamine -= 4; - offset -= 4; - - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3)) - goto Found3; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2)) - goto Found2; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1)) - goto Found1; - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) - goto Found; - } - - while (lengthToExamine > 0) - { - lengthToExamine -= 1; - offset -= 1; - - if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) - goto Found; - } - - if (Vector.IsHardwareAccelerated && (offset > 0)) - { - lengthToExamine = (offset & (nuint)~(Vector.Count - 1)); - - Vector values = new Vector(value); - - while (lengthToExamine > (nuint)(Vector.Count - 1)) - { - var matches = Vector.Equals(values, LoadVector(ref searchSpace, offset - (nuint)Vector.Count)); - if (Vector.Zero.Equals(matches)) - { - offset -= (nuint)Vector.Count; - lengthToExamine -= (nuint)Vector.Count; - continue; - } - - // Find offset of first match and add to current offset - return (int)(offset) - Vector.Count + LocateLastFoundByte(matches); - } - if (offset > 0) - { - lengthToExamine = offset; - goto SequentialScan; - } - } - return -1; - Found: // Workaround for https://github.com/dotnet/runtime/issues/8795 - return (int)offset; - Found1: - return (int)(offset + 1); - Found2: - return (int)(offset + 2); - Found3: - return (int)(offset + 3); - Found4: - return (int)(offset + 4); - Found5: - return (int)(offset + 5); - Found6: - return (int)(offset + 6); - Found7: - return (int)(offset + 7); - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, int length) - { - Debug.Assert(length >= 0); - - uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions - uint uValue1 = value1; // Use uint for comparisons to avoid unnecessary 8->32 extensions - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Sse2.IsSupported || AdvSimd.Arm64.IsSupported) - { - // Avx2 branch also operates on Sse2 sizes, so check is combined. - nint vectorDiff = (nint)length - Vector128.Count; - if (vectorDiff >= 0) - { - // >= Sse2 intrinsics are supported, and length is enough to use them so use that path. - // We jump forward to the intrinsics at the end of the method so a naive branch predict - // will choose the non-intrinsic path so short lengths which don't gain anything aren't - // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths - // more than make this back from the intrinsics. - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - else if (Vector.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, as it is used later - nint vectorDiff = (nint)length - Vector.Count; - if (vectorDiff >= 0) - { - // Similar as above for Vector version - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - - uint lookUp; - while (lengthToExamine >= 8) - { - lengthToExamine -= 8; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found3; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found4; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found5; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found6; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found7; - - offset += 8; - } - - if (lengthToExamine >= 4) - { - lengthToExamine -= 4; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found3; - - offset += 4; - } - - while (lengthToExamine > 0) - { - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found; - - offset += 1; - lengthToExamine -= 1; - } - - NotFound: - return -1; - Found: // Workaround for https://github.com/dotnet/runtime/issues/8795 - return (int)offset; - Found1: - return (int)(offset + 1); - Found2: - return (int)(offset + 2); - Found3: - return (int)(offset + 3); - Found4: - return (int)(offset + 4); - Found5: - return (int)(offset + 5); - Found6: - return (int)(offset + 6); - Found7: - return (int)(offset + 7); - - IntrinsicsCompare: - // When we move into a Vectorized block, we process everything of Vector size; - // and then for any remainder we do a final compare of Vector size but starting at - // the end and forwards, which may overlap on an earlier compare. - - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (Sse2.IsSupported) - { - int matches; - if (Avx2.IsSupported) - { - Vector256 search; - // Guard as we may only have a valid size for Vector128; when we will move to the Sse2 - // We have already subtracted Vector128.Count from lengthToExamine so compare against that - // to see if we have double the size for Vector256.Count - if (lengthToExamine >= (nuint)Vector128.Count) - { - Vector256 values0 = Vector256.Create(value0); - Vector256 values1 = Vector256.Create(value1); - - // Subtract Vector128.Count so we have now subtracted Vector256.Count - lengthToExamine -= (nuint)Vector128.Count; - // First time this checks again against 0, however we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector256(ref searchSpace, offset); - // Bitwise Or to combine the flagged matches for the second value to our match flags - matches = Avx2.MoveMask( - Avx2.Or( - Avx2.CompareEqual(values0, search), - Avx2.CompareEqual(values1, search))); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) - { - // None matched - offset += (nuint)Vector256.Count; - continue; - } - - goto IntrinsicsMatch; - } - - // Move to Vector length from end for final compare - search = LoadVector256(ref searchSpace, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = Avx2.MoveMask( - Avx2.Or( - Avx2.CompareEqual(values0, search), - Avx2.CompareEqual(values1, search))); - if (matches == 0) - { - // None matched - goto NotFound; - } - - goto IntrinsicsMatch; - } - } - - // Initial size check was done on method entry. - Debug.Assert(length >= Vector128.Count); - { - Vector128 search; - Vector128 values0 = Vector128.Create(value0); - Vector128 values1 = Vector128.Create(value1); - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector128(ref searchSpace, offset); - - matches = Sse2.MoveMask( - Sse2.Or( - Sse2.CompareEqual(values0, search), - Sse2.CompareEqual(values1, search)) - .AsByte()); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) - { - // None matched - offset += (nuint)Vector128.Count; - continue; - } - - goto IntrinsicsMatch; - } - // Move to Vector length from end for final compare - search = LoadVector128(ref searchSpace, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = Sse2.MoveMask( - Sse2.Or( - Sse2.CompareEqual(values0, search), - Sse2.CompareEqual(values1, search))); - if (matches == 0) - { - // None matched - goto NotFound; - } - } - - IntrinsicsMatch: - // Find bitflag offset of first difference and add to current offset - offset += (nuint)BitOperations.TrailingZeroCount(matches); - goto Found; - } - else if (AdvSimd.Arm64.IsSupported) - { - Vector128 search; - Vector128 matches; - Vector128 values0 = Vector128.Create(value0); - Vector128 values1 = Vector128.Create(value1); - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector128(ref searchSpace, offset); - - matches = AdvSimd.Or( - AdvSimd.CompareEqual(values0, search), - AdvSimd.CompareEqual(values1, search)); - - if (matches == Vector128.Zero) - { - offset += (nuint)Vector128.Count; - continue; - } - - // Find bitflag offset of first match and add to current offset - offset += FindFirstMatchedLane(matches); - - goto Found; - } - - // Move to Vector length from end for final compare - search = LoadVector128(ref searchSpace, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = AdvSimd.Or( - AdvSimd.CompareEqual(values0, search), - AdvSimd.CompareEqual(values1, search)); - - if (matches == Vector128.Zero) - { - // None matched - goto NotFound; - } - - // Find bitflag offset of first match and add to current offset - offset += FindFirstMatchedLane(matches); - - goto Found; - } - else if (Vector.IsHardwareAccelerated) - { - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - - Vector search; - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector(ref searchSpace, offset); - search = Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)); - if (Vector.Zero.Equals(search)) - { - // None matched - offset += (nuint)Vector.Count; - continue; - } - - goto VectorMatch; - } - - // Move to Vector length from end for final compare - search = LoadVector(ref searchSpace, lengthToExamine); - offset = lengthToExamine; - search = Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)); - if (Vector.Zero.Equals(search)) - { - // None matched - goto NotFound; - } - - VectorMatch: - offset += (nuint)LocateFirstFoundByte(search); - goto Found; - } - - Debug.Fail("Unreachable"); - goto NotFound; - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, byte value2, int length) - { - Debug.Assert(length >= 0); - - uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions - uint uValue1 = value1; // Use uint for comparisons to avoid unnecessary 8->32 extensions - uint uValue2 = value2; // Use uint for comparisons to avoid unnecessary 8->32 extensions - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector128.IsHardwareAccelerated) - { - // Avx2 branch also operates on Sse2 sizes, so check is combined. - nint vectorDiff = (nint)length - Vector128.Count; - if (vectorDiff >= 0) - { - // >= Vector128 is accelerated, and length is enough to use them so use that path. - // We jump forward to the intrinsics at the end of the method so a naive branch predict - // will choose the non-intrinsic path so short lengths which don't gain anything aren't - // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths - // more than make this back from the intrinsics. - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - else if (Vector.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, as it is used later - nint vectorDiff = (nint)length - Vector.Count; - if (vectorDiff >= 0) - { - // Similar as above for Vector version - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - - uint lookUp; - while (lengthToExamine >= 8) - { - lengthToExamine -= 8; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1)) goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2)) goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3)) goto Found3; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 4)) goto Found4; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 5)) goto Found5; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 6)) goto Found6; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 7)) goto Found7; offset += 8; @@ -1164,17 +388,13 @@ public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, byt { lengthToExamine -= 4; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) goto Found; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1)) goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2)) goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3)) goto Found3; offset += 4; @@ -1182,417 +402,149 @@ public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, byt while (lengthToExamine > 0) { - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) + lengthToExamine -= 1; + + if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset)) goto Found; offset += 1; - lengthToExamine -= 1; } - NotFound: - return -1; - Found: // Workaround for https://github.com/dotnet/runtime/issues/8795 - return (int)offset; - Found1: - return (int)(offset + 1); - Found2: - return (int)(offset + 2); - Found3: - return (int)(offset + 3); - Found4: - return (int)(offset + 4); - Found5: - return (int)(offset + 5); - Found6: - return (int)(offset + 6); - Found7: - return (int)(offset + 7); - - IntrinsicsCompare: - // When we move into a Vectorized block, we process everything of Vector size; - // and then for any remainder we do a final compare of Vector size but starting at - // the end and forwards, which may overlap on an earlier compare. - - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (Vector128.IsHardwareAccelerated) + // We get past SequentialScan only if IsHardwareAccelerated is true; and remain length is greater than Vector length. + // However, we still have the redundant check to allow the JIT to see that the code is unreachable and eliminate it when the platform does not + // have hardware accelerated. After processing Vector lengths we return to SequentialScan to finish any remaining. + if (Vector256.IsHardwareAccelerated) { - uint matches; - if (Vector256.IsHardwareAccelerated) + if (offset < (nuint)(uint)Length) { - Vector256 search; - // Guard as we may only have a valid size for Vector128; when we will move to the Sse2 - // We have already subtracted Vector128.Count from lengthToExamine so compare against that - // to see if we have double the size for Vector256.Count - if (lengthToExamine >= (nuint)Vector128.Count) + if ((((nuint)(uint)Unsafe.AsPointer(ref searchSpace) + offset) & (nuint)(Vector256.Count - 1)) != 0) + { + // Not currently aligned to Vector256 (is aligned to Vector128); this can cause a problem for searches + // with no upper bound e.g. String.strlen. + // Start with a check on Vector128 to align to Vector256, before moving to processing Vector256. + // This ensures we do not fault across memory pages while searching for an end of string. + Vector128 search = Vector128.LoadUnsafe(ref searchSpace, offset); + + // Same method as below + uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); + if (matches == 0) + { + // Zero flags set so no matches + offset += (nuint)Vector128.Count; + } + else + { + // Find bitflag offset of first match and add to current offset + return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + } + } + + lengthToExamine = GetByteVector256SpanLength(offset, Length); + if (lengthToExamine > offset) { - Vector256 values0 = Vector256.Create(value0); - Vector256 values1 = Vector256.Create(value1); - Vector256 values2 = Vector256.Create(value2); - - // Subtract Vector128.Count so we have now subtracted Vector256.Count - lengthToExamine -= (nuint)Vector128.Count; - // First time this checks again against 0, however we will move into final compare if it fails. - while (lengthToExamine > offset) + do { - search = Vector256.LoadUnsafe(ref searchSpace, offset); - // Bitwise Or to combine the flagged matches for the second value to our match flags - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search)).ExtractMostSignificantBits(); - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, + Vector256 search = Vector256.LoadUnsafe(ref searchSpace, offset); + uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); + // Note that MoveMask has converted the equal vector elements into a set of bit flags, // So the bit position in 'matches' corresponds to the element offset. if (matches == 0) { - // None matched + // Zero flags set so no matches offset += (nuint)Vector256.Count; continue; } - goto IntrinsicsMatch; - } + // Find bitflag offset of first match and add to current offset + return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + } while (lengthToExamine > offset); + } + + lengthToExamine = GetByteVector128SpanLength(offset, Length); + if (lengthToExamine > offset) + { + Vector128 search = Vector128.LoadUnsafe(ref searchSpace, offset); - // Move to Vector length from end for final compare - search = Vector256.LoadUnsafe(ref searchSpace, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search)).ExtractMostSignificantBits(); + // Same method as above + uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); if (matches == 0) { - // None matched - goto NotFound; + // Zero flags set so no matches + offset += (nuint)Vector128.Count; + } + else + { + // Find bitflag offset of first match and add to current offset + return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); } + } - goto IntrinsicsMatch; + if (offset < (nuint)(uint)Length) + { + lengthToExamine = ((nuint)(uint)Length - offset); + goto SequentialScan; } } - - // Initial size check was done on method entry. - Debug.Assert(length >= Vector128.Count); + } + else if (Vector128.IsHardwareAccelerated) + { + if (offset < (nuint)(uint)Length) { - Vector128 search, compareResult; - Vector128 values0 = Vector128.Create(value0); - Vector128 values1 = Vector128.Create(value1); - Vector128 values2 = Vector128.Create(value2); - // First time this checks against 0 and we will move into final compare if it fails. + lengthToExamine = GetByteVector128SpanLength(offset, Length); + while (lengthToExamine > offset) { - search = Vector128.LoadUnsafe(ref searchSpace, offset); + Vector128 search = Vector128.LoadUnsafe(ref searchSpace, offset); - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search); + // Same method as above + Vector128 compareResult = Vector128.Equals(Vector128.Zero, search); if (compareResult == Vector128.Zero) { - // None matched + // Zero flags set so no matches offset += (nuint)Vector128.Count; continue; } - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - matches = compareResult.ExtractMostSignificantBits(); - goto IntrinsicsMatch; + // Find bitflag offset of first match and add to current offset + uint matches = compareResult.ExtractMostSignificantBits(); + return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); } - // Move to Vector length from end for final compare - search = Vector128.LoadUnsafe(ref searchSpace, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search); - if (compareResult == Vector128.Zero) + + if (offset < (nuint)(uint)Length) { - // None matched - goto NotFound; + lengthToExamine = ((nuint)(uint)Length - offset); + goto SequentialScan; } - matches = compareResult.ExtractMostSignificantBits(); } - - IntrinsicsMatch: - // Find bitflag offset of first difference and add to current offset - offset += (nuint)BitOperations.TrailingZeroCount(matches); - goto Found; } else if (Vector.IsHardwareAccelerated) { - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - Vector values2 = new Vector(value2); - - Vector search; - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector(ref searchSpace, offset); - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)); - if (Vector.Zero.Equals(search)) - { - // None matched - offset += (nuint)Vector.Count; - continue; - } - - goto VectorMatch; - } - - // Move to Vector length from end for final compare - search = LoadVector(ref searchSpace, lengthToExamine); - offset = lengthToExamine; - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)); - if (Vector.Zero.Equals(search)) + if (offset < (nuint)(uint)Length) { - // None matched - goto NotFound; - } - - VectorMatch: - offset += (nuint)LocateFirstFoundByte(search); - goto Found; - } - - Debug.Fail("Unreachable"); - goto NotFound; - } - - public static int LastIndexOfAny(ref byte searchSpace, byte value0, byte value1, int length) - { - Debug.Assert(length >= 0); - - uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions - uint uValue1 = value1; - nuint offset = (nuint)(uint)length; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) - { - lengthToExamine = UnalignedCountVectorFromEnd(ref searchSpace, length); - } - SequentialScan: - uint lookUp; - while (lengthToExamine >= 8) - { - lengthToExamine -= 8; - offset -= 8; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found7; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found6; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found5; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found4; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found3; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found; - } - - if (lengthToExamine >= 4) - { - lengthToExamine -= 4; - offset -= 4; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found3; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found; - } - - while (lengthToExamine > 0) - { - lengthToExamine -= 1; - offset -= 1; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp) - goto Found; - } + lengthToExamine = GetByteVectorSpanLength(offset, Length); - if (Vector.IsHardwareAccelerated && (offset > 0)) - { - lengthToExamine = (offset & (nuint)~(Vector.Count - 1)); - - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - - while (lengthToExamine > (nuint)(Vector.Count - 1)) - { - Vector search = LoadVector(ref searchSpace, offset - (nuint)Vector.Count); - var matches = Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)); - if (Vector.Zero.Equals(matches)) + while (lengthToExamine > offset) { - offset -= (nuint)Vector.Count; - lengthToExamine -= (nuint)Vector.Count; - continue; - } - - // Find offset of first match and add to current offset - return (int)(offset) - Vector.Count + LocateLastFoundByte(matches); - } - - if (offset > 0) - { - lengthToExamine = offset; - goto SequentialScan; - } - } - return -1; - Found: // Workaround for https://github.com/dotnet/runtime/issues/8795 - return (int)offset; - Found1: - return (int)(offset + 1); - Found2: - return (int)(offset + 2); - Found3: - return (int)(offset + 3); - Found4: - return (int)(offset + 4); - Found5: - return (int)(offset + 5); - Found6: - return (int)(offset + 6); - Found7: - return (int)(offset + 7); - } - - public static int LastIndexOfAny(ref byte searchSpace, byte value0, byte value1, byte value2, int length) - { - Debug.Assert(length >= 0); - - uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions - uint uValue1 = value1; - uint uValue2 = value2; - nuint offset = (nuint)(uint)length; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) - { - lengthToExamine = UnalignedCountVectorFromEnd(ref searchSpace, length); - } - SequentialScan: - uint lookUp; - while (lengthToExamine >= 8) - { - lengthToExamine -= 8; - offset -= 8; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found7; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found6; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found5; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found4; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found3; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found; - } - - if (lengthToExamine >= 4) - { - lengthToExamine -= 4; - offset -= 4; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found3; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found2; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found1; - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found; - } - - while (lengthToExamine > 0) - { - lengthToExamine -= 1; - offset -= 1; - - lookUp = Unsafe.AddByteOffset(ref searchSpace, offset); - if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp) - goto Found; - } - - if (Vector.IsHardwareAccelerated && (offset > 0)) - { - lengthToExamine = (offset & (nuint)~(Vector.Count - 1)); - - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - Vector values2 = new Vector(value2); - - while (lengthToExamine > (nuint)(Vector.Count - 1)) - { - Vector search = LoadVector(ref searchSpace, offset - (nuint)Vector.Count); + var matches = Vector.Equals(Vector.Zero, LoadVector(ref searchSpace, offset)); + if (Vector.Zero.Equals(matches)) + { + offset += (nuint)Vector.Count; + continue; + } - var matches = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)); + // Find offset of first match and add to current offset + return (int)offset + LocateFirstFoundByte(matches); + } - if (Vector.Zero.Equals(matches)) + if (offset < (nuint)(uint)Length) { - offset -= (nuint)Vector.Count; - lengthToExamine -= (nuint)Vector.Count; - continue; + lengthToExamine = ((nuint)(uint)Length - offset); + goto SequentialScan; } - - // Find offset of first match and add to current offset - return (int)(offset) - Vector.Count + LocateLastFoundByte(matches); - } - - if (offset > 0) - { - lengthToExamine = offset; - goto SequentialScan; } } - return -1; + + ThrowMustBeNullTerminatedString(); Found: // Workaround for https://github.com/dotnet/runtime/issues/8795 return (int)offset; Found1: @@ -2201,34 +1153,6 @@ private static unsafe nuint UnalignedCountVector128(ref byte searchSpace) return (nuint)(uint)((Vector128.Count - unaligned) & (Vector128.Count - 1)); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe nuint UnalignedCountVectorFromEnd(ref byte searchSpace, int length) - { - nint unaligned = (nint)Unsafe.AsPointer(ref searchSpace) & (Vector.Count - 1); - return (nuint)(uint)(((length & (Vector.Count - 1)) + unaligned) & (Vector.Count - 1)); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint FindFirstMatchedLane(Vector128 compareResult) - { - Debug.Assert(AdvSimd.Arm64.IsSupported); - - // Mask to help find the first lane in compareResult that is set. - // MSB 0x10 corresponds to 1st lane, 0x01 corresponds to 0th lane and so forth. - Vector128 mask = Vector128.Create((ushort)0x1001).AsByte(); - - // Find the first lane that is set inside compareResult. - Vector128 maskedSelectedLanes = AdvSimd.And(compareResult, mask); - Vector128 pairwiseSelectedLane = AdvSimd.Arm64.AddPairwise(maskedSelectedLanes, maskedSelectedLanes); - ulong selectedLanes = pairwiseSelectedLane.AsUInt64().ToScalar(); - - // It should be handled by compareResult != Vector.Zero - Debug.Assert(selectedLanes != 0); - - // Find the first lane that is set inside compareResult. - return (uint)BitOperations.TrailingZeroCount(selectedLanes) >> 2; - } - public static void Reverse(ref byte buf, nuint length) { if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index 29a5906643a2a..8a44db8cc3aed 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -5,7 +5,6 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; namespace System @@ -24,7 +23,7 @@ public static int IndexOf(ref char searchSpace, int searchSpaceLength, ref char if (valueTailLength == 0) { // for single-char values use plain IndexOf - return IndexOf(ref searchSpace, value, searchSpaceLength); + return IndexOfChar(ref searchSpace, value, searchSpaceLength); } nint offset = 0; @@ -41,7 +40,7 @@ public static int IndexOf(ref char searchSpace, int searchSpaceLength, ref char while (remainingSearchSpaceLength > 0) { // Do a quick search for the first element of "value". - int relativeIndex = IndexOf(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength); + int relativeIndex = IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength); if (relativeIndex < 0) break; @@ -68,6 +67,7 @@ ref Unsafe.As(ref Unsafe.Add(ref searchSpace, offset + 1)), // Based on http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd "Algorithm 1: Generic SIMD" by Wojciech Muła // Some details about the implementation can also be found in https://github.com/dotnet/runtime/pull/63285 SEARCH_TWO_CHARS: + ref ushort ushortSearchSpace = ref Unsafe.As(ref searchSpace); if (Vector256.IsHardwareAccelerated && searchSpaceMinusValueTailLength - Vector256.Count >= 0) { // Find the last unique (which is not equal to ch1) character @@ -88,8 +88,8 @@ ref Unsafe.As(ref Unsafe.Add(ref searchSpace, offset + 1)), // Make sure we don't go out of bounds Debug.Assert(offset + ch1ch2Distance + Vector256.Count <= searchSpaceLength); - Vector256 cmpCh2 = Vector256.Equals(ch2, LoadVector256(ref searchSpace, offset + ch1ch2Distance)); - Vector256 cmpCh1 = Vector256.Equals(ch1, LoadVector256(ref searchSpace, offset)); + Vector256 cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance))); + Vector256 cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)offset)); Vector256 cmpAnd = (cmpCh1 & cmpCh2).AsByte(); // Early out: cmpAnd is all zeros @@ -155,8 +155,8 @@ ref Unsafe.As(ref value), (nuint)(uint)valueLength * 2)) // Make sure we don't go out of bounds Debug.Assert(offset + ch1ch2Distance + Vector128.Count <= searchSpaceLength); - Vector128 cmpCh2 = Vector128.Equals(ch2, LoadVector128(ref searchSpace, offset + ch1ch2Distance)); - Vector128 cmpCh1 = Vector128.Equals(ch1, LoadVector128(ref searchSpace, offset)); + Vector128 cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance))); + Vector128 cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset)); Vector128 cmpAnd = (cmpCh1 & cmpCh2).AsByte(); // Early out: cmpAnd is all zeros @@ -214,7 +214,7 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c int valueTailLength = valueLength - 1; if (valueTailLength == 0) - return LastIndexOf(ref searchSpace, value, searchSpaceLength); // for single-char values use plain LastIndexOf + return LastIndexOfValueType(ref Unsafe.As(ref searchSpace), (short)value, searchSpaceLength); // for single-char values use plain LastIndexOf int offset = 0; char valueHead = value; @@ -234,7 +234,7 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c break; // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there. // Do a quick search for the first element of "value". - int relativeIndex = LastIndexOf(ref searchSpace, valueHead, remainingSearchSpaceLength); + int relativeIndex = LastIndexOfValueType(ref Unsafe.As(ref searchSpace), (short)valueHead, remainingSearchSpaceLength); if (relativeIndex == -1) break; @@ -253,6 +253,7 @@ ref Unsafe.As(ref Unsafe.Add(ref searchSpace, relativeIndex + 1)), // Based on http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd "Algorithm 1: Generic SIMD" by Wojciech Muła // Some details about the implementation can also be found in https://github.com/dotnet/runtime/pull/63285 SEARCH_TWO_CHARS: + ref ushort ushortSearchSpace = ref Unsafe.As(ref searchSpace); if (Vector256.IsHardwareAccelerated && searchSpaceMinusValueTailLength >= Vector256.Count) { offset = searchSpaceMinusValueTailLength - Vector256.Count; @@ -270,8 +271,8 @@ ref Unsafe.As(ref Unsafe.Add(ref searchSpace, relativeIndex + 1)), do { - Vector256 cmpCh1 = Vector256.Equals(ch1, LoadVector256(ref searchSpace, (nuint)offset)); - Vector256 cmpCh2 = Vector256.Equals(ch2, LoadVector256(ref searchSpace, (nuint)(offset + ch1ch2Distance))); + Vector256 cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)offset)); + Vector256 cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance))); Vector256 cmpAnd = (cmpCh1 & cmpCh2).AsByte(); // Early out: cmpAnd is all zeros @@ -319,8 +320,8 @@ ref Unsafe.As(ref value), (nuint)(uint)valueLength * 2)) do { - Vector128 cmpCh1 = Vector128.Equals(ch1, LoadVector128(ref searchSpace, (nuint)offset)); - Vector128 cmpCh2 = Vector128.Equals(ch2, LoadVector128(ref searchSpace, (nuint)(offset + ch1ch2Distance))); + Vector128 cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset)); + Vector128 cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance))); Vector128 cmpAnd = (cmpCh1 & cmpCh2).AsByte(); // Early out: cmpAnd is all zeros @@ -420,104 +421,13 @@ public static unsafe int SequenceCompareTo(ref char first, int firstLength, ref return lengthDelta; } - // Adapted from IndexOf(...) + // IndexOfNullCharacter processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator. + // This behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it. [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe bool Contains(ref char searchSpace, char value, int length) + public static unsafe int IndexOfNullCharacter(ref char searchSpace) { - Debug.Assert(length >= 0); - - fixed (char* pChars = &searchSpace) - { - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (uint)length; - - if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) - { - // Figure out how many characters to read sequentially until we are vector aligned - // This is equivalent to: - // unaligned = ((int)pCh % Unsafe.SizeOf>()) / ElementsPerByte - // length = (Vector.Count - unaligned) % Vector.Count - const int ElementsPerByte = sizeof(ushort) / sizeof(byte); - int unaligned = (int)((uint)((int)pChars & (Unsafe.SizeOf>() - 1)) / ElementsPerByte); - lengthToExamine = (uint)((Vector.Count - unaligned) & (Vector.Count - 1)); - } - - while (lengthToExamine >= 4) - { - lengthToExamine -= 4; - char* pStart = pChars + offset; - - if (value == pStart[0] || - value == pStart[1] || - value == pStart[2] || - value == pStart[3]) - { - goto Found; - } - - offset += 4; - } - - while (lengthToExamine > 0) - { - lengthToExamine--; - - if (value == pChars[offset]) - goto Found; - - offset++; - } - - // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow - // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware acceleration. - if (Vector.IsHardwareAccelerated && (offset < (uint)length)) - { - // Get the highest multiple of Vector.Count that is within the search space. - // That will be how many times we iterate in the loop below. - // This is equivalent to: lengthToExamine = Vector.Count + ((uint)length - offset) / Vector.Count) - lengthToExamine = ((uint)length - offset) & (nuint)~(Vector.Count - 1); - - Vector values = new(value); - Vector matches; - - while (offset < lengthToExamine) - { - // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned - Debug.Assert(((int)(pChars + offset) % Unsafe.SizeOf>()) == 0); - matches = Vector.Equals(values, Unsafe.Read>(pChars + offset)); - if (matches == Vector.Zero) - { - offset += (nuint)Vector.Count; - continue; - } - - goto Found; - } - - // The total length is at least Vector.Count, so instead of falling back to a - // sequential scan for the remainder, we check the vector read from the end -- note: unaligned read necessary. - // We do this only if at least one element is left. - if (offset < (uint)length) - { - matches = Vector.Equals(values, Unsafe.ReadUnaligned>(pChars + (uint)length - (uint)Vector.Count)); - if (matches != Vector.Zero) - { - goto Found; - } - } - } - - return false; - - Found: - return true; - } - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe int IndexOf(ref char searchSpace, char value, int length) - { - Debug.Assert(length >= 0); + const char value = '\0'; + const int length = int.MaxValue; nint offset = 0; nint lengthToExamine = length; @@ -530,18 +440,12 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) { // Avx2 branch also operates on Sse2 sizes, so check is combined. // Needs to be double length to allow us to align the data first. - if (length >= Vector128.Count * 2) - { - lengthToExamine = UnalignedCountVector128(ref searchSpace); - } + lengthToExamine = UnalignedCountVector128(ref searchSpace); } else if (Vector.IsHardwareAccelerated) { // Needs to be double length to allow us to align the data first. - if (length >= Vector.Count * 2) - { - lengthToExamine = UnalignedCountVector(ref searchSpace); - } + lengthToExamine = UnalignedCountVector(ref searchSpace); } SequentialScan: @@ -600,11 +504,10 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) // method, so the alignment only acts as best endeavour. The GC cost is likely to dominate over // the misalignment that may occur after; to we default to giving the GC a free hand to relocate and // its up to the caller whether they are operating over fixed data. - Vector128 values = Vector128.Create((ushort)value); Vector128 search = Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset); // Same method as below - uint matches = Vector128.Equals(values, search).AsByte().ExtractMostSignificantBits(); + uint matches = Vector128.Equals(Vector128.Zero, search).AsByte().ExtractMostSignificantBits(); if (matches == 0) { // Zero flags set so no matches @@ -620,13 +523,12 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) lengthToExamine = GetCharVector256SpanLength(offset, length); if (lengthToExamine > 0) { - Vector256 values = Vector256.Create((ushort)value); do { Debug.Assert(lengthToExamine >= Vector256.Count); Vector256 search = Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)offset); - uint matches = Vector256.Equals(values, search).AsByte().ExtractMostSignificantBits(); + uint matches = Vector256.Equals(Vector256.Zero, search).AsByte().ExtractMostSignificantBits(); // Note that MoveMask has converted the equal vector elements into a set of bit flags, // So the bit position in 'matches' corresponds to the element offset. if (matches == 0) @@ -648,11 +550,10 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) { Debug.Assert(lengthToExamine >= Vector128.Count); - Vector128 values = Vector128.Create((ushort)value); Vector128 search = Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset); // Same method as above - uint matches = Vector128.Equals(values, search).AsByte().ExtractMostSignificantBits(); + uint matches = Vector128.Equals(Vector128.Zero, search).AsByte().ExtractMostSignificantBits(); if (matches == 0) { // Zero flags set so no matches @@ -684,7 +585,6 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) lengthToExamine = GetCharVector128SpanLength(offset, length); if (lengthToExamine > 0) { - Vector128 values = Vector128.Create((ushort)value); do { Debug.Assert(lengthToExamine >= Vector128.Count); @@ -692,7 +592,7 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) Vector128 search = Vector128.LoadUnsafe(ref ushortSearchSpace, (uint)offset); // Same method as above - Vector128 compareResult = Vector128.Equals(values, search); + Vector128 compareResult = Vector128.Equals(Vector128.Zero, search); if (compareResult == Vector128.Zero) { // Zero flags set so no matches @@ -725,12 +625,11 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) if (lengthToExamine > 0) { - Vector values = new Vector((ushort)value); do { Debug.Assert(lengthToExamine >= Vector.Count); - var matches = Vector.Equals(values, LoadVector(ref searchSpace, offset)); + var matches = Vector.Equals(Vector.Zero, LoadVector(ref searchSpace, offset)); if (Vector.Zero.Equals(matches)) { offset += Vector.Count; @@ -750,789 +649,8 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) } } } - return -1; - Found3: - return (int)(offset + 3); - Found2: - return (int)(offset + 2); - Found1: - return (int)(offset + 1); - Found: - return (int)(offset); - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, int length) - { - Debug.Assert(length >= 0); - - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector128.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector128.Count; - if (vectorDiff >= 0) - { - // >= Vector128 is accelerated and length is enough to use them, so use that path. - // We jump forward to the intrinsics at the end of them method so a naive branch predict - // will choose the non-intrinsic path so short lengths which don't gain anything aren't - // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths - // more than make this back from the intrinsics. - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - else if (Vector.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector.Count; - if (vectorDiff >= 0) - { - // Similar as above for Vector version - lengthToExamine = (nuint)vectorDiff; - goto VectorCompare; - } - } - - int lookUp; - while (lengthToExamine >= 4) - { - ref char current = ref Add(ref searchStart, offset); - - lookUp = current; - if (value0 == lookUp || value1 == lookUp) - goto Found; - lookUp = Unsafe.Add(ref current, 1); - if (value0 == lookUp || value1 == lookUp) - goto Found1; - lookUp = Unsafe.Add(ref current, 2); - if (value0 == lookUp || value1 == lookUp) - goto Found2; - lookUp = Unsafe.Add(ref current, 3); - if (value0 == lookUp || value1 == lookUp) - goto Found3; - - offset += 4; - lengthToExamine -= 4; - } - - while (lengthToExamine > 0) - { - lookUp = Add(ref searchStart, offset); - if (value0 == lookUp || value1 == lookUp) - goto Found; - - offset += 1; - lengthToExamine -= 1; - } - - NotFound: - return -1; - Found3: - return (int)(offset + 3); - Found2: - return (int)(offset + 2); - Found1: - return (int)(offset + 1); - Found: - return (int)offset; - - IntrinsicsCompare: - // When we move into a Vectorized block, we process everything of Vector size; - // and then for any remainder we do a final compare of Vector size but starting at - // the end and forwards, which may overlap on an earlier compare. - - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (Vector128.IsHardwareAccelerated) - { - uint matches; - ref ushort ushortSearchStart = ref Unsafe.As(ref searchStart); - if (Vector256.IsHardwareAccelerated) - { - Vector256 search; - // Guard as we may only have a valid size for Vector128; when we will move to the Sse2 - // We have already subtracted Vector128.Count from lengthToExamine so compare against that - // to see if we have double the size for Vector256.Count - if (lengthToExamine >= (nuint)Vector128.Count) - { - Vector256 values0 = Vector256.Create((ushort)value0); - Vector256 values1 = Vector256.Create((ushort)value1); - - // Subtract Vector128.Count so we have now subtracted Vector256.Count - lengthToExamine -= (nuint)Vector128.Count; - // First time this checks again against 0, however we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector256.LoadUnsafe(ref ushortSearchStart, offset); - // Bitwise Or to combine the flagged matches for the second value to our match flags - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search)) - .AsByte().ExtractMostSignificantBits(); - - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) - { - // None matched - offset += (nuint)Vector256.Count; - continue; - } - - goto IntrinsicsMatch; - } - - // Move to Vector length from end for final compare - search = Vector256.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search)) - .AsByte().ExtractMostSignificantBits(); - if (matches == 0) - { - // None matched - goto NotFound; - } - - goto IntrinsicsMatch; - } - } - - // Initial size check was done on method entry. - Debug.Assert(length >= Vector128.Count); - { - Vector128 search, compareResult; - Vector128 values0 = Vector128.Create((ushort)value0); - Vector128 values1 = Vector128.Create((ushort)value1); - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector128.LoadUnsafe(ref ushortSearchStart, offset); - - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search); - if (compareResult == Vector128.Zero) - { - // None matched - offset += (nuint)Vector128.Count; - continue; - } - - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - matches = compareResult.AsByte().ExtractMostSignificantBits(); - goto IntrinsicsMatch; - } - // Move to Vector length from end for final compare - search = Vector128.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search); - if (compareResult == Vector128.Zero) - { - // None matched - goto NotFound; - } - matches = compareResult.AsByte().ExtractMostSignificantBits(); - } - - IntrinsicsMatch: - // Find bitflag offset of first difference and add to current offset, - // flags are in bytes so divide by 2 for chars (shift right by 1) - offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1; - goto Found; - } - - VectorCompare: - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (!Vector128.IsHardwareAccelerated && Vector.IsHardwareAccelerated) - { - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - - Vector search; - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector(ref searchStart, offset); - search = Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)); - if (Vector.Zero.Equals(search)) - { - // None matched - offset += (nuint)Vector.Count; - continue; - } - - goto VectorMatch; - } - - // Move to Vector length from end for final compare - search = LoadVector(ref searchStart, lengthToExamine); - offset = lengthToExamine; - search = Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)); - if (Vector.Zero.Equals(search)) - { - // None matched - goto NotFound; - } - - VectorMatch: - offset += (nuint)(uint)LocateFirstFoundChar(search); - goto Found; - } - - Debug.Fail("Unreachable"); - goto NotFound; - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, int length) - { - Debug.Assert(length >= 0); - - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector128.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector128.Count; - if (vectorDiff >= 0) - { - // >= Vector128 is accelerated and length is enough to use them, so use that path. - // We jump forward to the intrinsics at the end of them method so a naive branch predict - // will choose the non-intrinsic path so short lengths which don't gain anything aren't - // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths - // more than make this back from the intrinsics. - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - else if (Vector.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector.Count; - if (vectorDiff >= 0) - { - // Similar as above for Vector version - lengthToExamine = (nuint)vectorDiff; - goto VectorCompare; - } - } - - int lookUp; - while (lengthToExamine >= 4) - { - ref char current = ref Add(ref searchStart, offset); - - lookUp = current; - if (value0 == lookUp || value1 == lookUp || value2 == lookUp) - goto Found; - lookUp = Unsafe.Add(ref current, 1); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp) - goto Found1; - lookUp = Unsafe.Add(ref current, 2); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp) - goto Found2; - lookUp = Unsafe.Add(ref current, 3); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp) - goto Found3; - - offset += 4; - lengthToExamine -= 4; - } - - while (lengthToExamine > 0) - { - lookUp = Add(ref searchStart, offset); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp) - goto Found; - - offset += 1; - lengthToExamine -= 1; - } - - NotFound: - return -1; - Found3: - return (int)(offset + 3); - Found2: - return (int)(offset + 2); - Found1: - return (int)(offset + 1); - Found: - return (int)offset; - - IntrinsicsCompare: - // When we move into a Vectorized block, we process everything of Vector size; - // and then for any remainder we do a final compare of Vector size but starting at - // the end and forwards, which may overlap on an earlier compare. - - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (Vector128.IsHardwareAccelerated) - { - uint matches; - ref ushort ushortSearchStart = ref Unsafe.As(ref searchStart); - if (Vector256.IsHardwareAccelerated) - { - Vector256 search; - // Guard as we may only have a valid size for Vector128; when we will move to the Sse2 - // We have already subtracted Vector128.Count from lengthToExamine so compare against that - // to see if we have double the size for Vector256.Count - if (lengthToExamine >= (nuint)Vector128.Count) - { - Vector256 values0 = Vector256.Create((ushort)value0); - Vector256 values1 = Vector256.Create((ushort)value1); - Vector256 values2 = Vector256.Create((ushort)value2); - - // Subtract Vector128.Count so we have now subtracted Vector256.Count - lengthToExamine -= (nuint)Vector128.Count; - // First time this checks again against 0, however we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector256.LoadUnsafe(ref ushortSearchStart, offset); - // Bitwise Or to combine the flagged matches for the second value to our match flags - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search)) - .AsByte().ExtractMostSignificantBits(); - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) - { - // None matched - offset += (nuint)Vector256.Count; - continue; - } - - goto IntrinsicsMatch; - } - - // Move to Vector length from end for final compare - search = Vector256.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search)) - .AsByte().ExtractMostSignificantBits(); - if (matches == 0) - { - // None matched - goto NotFound; - } - - goto IntrinsicsMatch; - } - } - - // Initial size check was done on method entry. - Debug.Assert(length >= Vector128.Count); - { - Vector128 search, compareResult; - Vector128 values0 = Vector128.Create((ushort)value0); - Vector128 values1 = Vector128.Create((ushort)value1); - Vector128 values2 = Vector128.Create((ushort)value2); - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector128.LoadUnsafe(ref ushortSearchStart, offset); - - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search); - if (compareResult == Vector128.Zero) - { - // None matched - offset += (nuint)Vector128.Count; - continue; - } - - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - matches = compareResult.AsByte().ExtractMostSignificantBits(); - goto IntrinsicsMatch; - } - // Move to Vector length from end for final compare - search = Vector128.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search); - if (compareResult == Vector128.Zero) - { - // None matched - goto NotFound; - } - matches = compareResult.AsByte().ExtractMostSignificantBits(); - } - - IntrinsicsMatch: - // Find bitflag offset of first difference and add to current offset, - // flags are in bytes so divide by 2 for chars (shift right by 1) - offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1; - goto Found; - } - - VectorCompare: - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (!Vector128.IsHardwareAccelerated && Vector.IsHardwareAccelerated) - { - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - Vector values2 = new Vector(value2); - - Vector search; - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector(ref searchStart, offset); - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)); - if (Vector.Zero.Equals(search)) - { - // None matched - offset += (nuint)Vector.Count; - continue; - } - - goto VectorMatch; - } - - // Move to Vector length from end for final compare - search = LoadVector(ref searchStart, lengthToExamine); - offset = lengthToExamine; - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)); - if (Vector.Zero.Equals(search)) - { - // None matched - goto NotFound; - } - - VectorMatch: - offset += (nuint)(uint)LocateFirstFoundChar(search); - goto Found; - } - - Debug.Fail("Unreachable"); - goto NotFound; - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, char value3, int length) - { - Debug.Assert(length >= 0); - - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector128.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector128.Count; - if (vectorDiff >= 0) - { - // >= Vector128 is accelerated and length is enough to use them, so use that path. - // We jump forward to the intrinsics at the end of them method so a naive branch predict - // will choose the non-intrinsic path so short lengths which don't gain anything aren't - // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths - // more than make this back from the intrinsics. - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - else if (Vector.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector.Count; - if (vectorDiff >= 0) - { - // Similar as above for Vector version - lengthToExamine = (nuint)vectorDiff; - goto VectorCompare; - } - } - - int lookUp; - while (lengthToExamine >= 4) - { - ref char current = ref Add(ref searchStart, offset); - - lookUp = current; - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp) - goto Found; - lookUp = Unsafe.Add(ref current, 1); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp) - goto Found1; - lookUp = Unsafe.Add(ref current, 2); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp) - goto Found2; - lookUp = Unsafe.Add(ref current, 3); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp) - goto Found3; - - offset += 4; - lengthToExamine -= 4; - } - - while (lengthToExamine > 0) - { - lookUp = Add(ref searchStart, offset); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp) - goto Found; - - offset += 1; - lengthToExamine -= 1; - } - - NotFound: - return -1; - Found3: - return (int)(offset + 3); - Found2: - return (int)(offset + 2); - Found1: - return (int)(offset + 1); - Found: - return (int)offset; - - IntrinsicsCompare: - // When we move into a Vectorized block, we process everything of Vector size; - // and then for any remainder we do a final compare of Vector size but starting at - // the end and forwards, which may overlap on an earlier compare. - - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (Vector128.IsHardwareAccelerated) - { - uint matches; - ref ushort ushortSearchStart = ref Unsafe.As(ref searchStart); - if (Vector256.IsHardwareAccelerated) - { - Vector256 search; - // Guard as we may only have a valid size for Vector128; when we will move to the Sse2 - // We have already subtracted Vector128.Count from lengthToExamine so compare against that - // to see if we have double the size for Vector256.Count - if (lengthToExamine >= (nuint)Vector128.Count) - { - Vector256 values0 = Vector256.Create((ushort)value0); - Vector256 values1 = Vector256.Create((ushort)value1); - Vector256 values2 = Vector256.Create((ushort)value2); - Vector256 values3 = Vector256.Create((ushort)value3); - - // Subtract Vector128.Count so we have now subtracted Vector256.Count - lengthToExamine -= (nuint)Vector128.Count; - // First time this checks again against 0, however we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector256.LoadUnsafe(ref ushortSearchStart, offset); - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) - | Vector256.Equals(values2, search) | Vector256.Equals(values3, search)) - .AsByte().ExtractMostSignificantBits(); - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) - { - // None matched - offset += (nuint)Vector256.Count; - continue; - } - - goto IntrinsicsMatch; - } - - // Move to Vector length from end for final compare - search = Vector256.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) - | Vector256.Equals(values2, search) | Vector256.Equals(values3, search)) - .AsByte().ExtractMostSignificantBits(); - if (matches == 0) - { - // None matched - goto NotFound; - } - - goto IntrinsicsMatch; - } - } - - // Initial size check was done on method entry. - Debug.Assert(length >= Vector128.Count); - { - Vector128 search, compareResult; - Vector128 values0 = Vector128.Create((ushort)value0); - Vector128 values1 = Vector128.Create((ushort)value1); - Vector128 values2 = Vector128.Create((ushort)value2); - Vector128 values3 = Vector128.Create((ushort)value3); - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector128.LoadUnsafe(ref ushortSearchStart, offset); - - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) - | Vector128.Equals(values2, search) | Vector128.Equals(values3, search); - if (compareResult == Vector128.Zero) - { - // None matched - offset += (nuint)Vector128.Count; - continue; - } - - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - matches = compareResult.AsByte().ExtractMostSignificantBits(); - goto IntrinsicsMatch; - } - // Move to Vector length from end for final compare - search = Vector128.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) - | Vector128.Equals(values2, search) | Vector128.Equals(values3, search); - if (compareResult == Vector128.Zero) - { - // None matched - goto NotFound; - } - matches = compareResult.AsByte().ExtractMostSignificantBits(); - } - - IntrinsicsMatch: - // Find bitflag offset of first difference and add to current offset, - // flags are in bytes so divide by 2 for chars (shift right by 1) - offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1; - goto Found; - } - - VectorCompare: - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (!Vector128.IsHardwareAccelerated && Vector.IsHardwareAccelerated) - { - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - Vector values2 = new Vector(value2); - Vector values3 = new Vector(value3); - - Vector search; - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector(ref searchStart, offset); - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)), - Vector.Equals(search, values3)); - if (Vector.Zero.Equals(search)) - { - // None matched - offset += (nuint)Vector.Count; - continue; - } - - goto VectorMatch; - } - - // Move to Vector length from end for final compare - search = LoadVector(ref searchStart, lengthToExamine); - offset = lengthToExamine; - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)), - Vector.Equals(search, values3)); - if (Vector.Zero.Equals(search)) - { - // None matched - goto NotFound; - } - - VectorMatch: - offset += (nuint)(uint)LocateFirstFoundChar(search); - goto Found; - } - - Debug.Fail("Unreachable"); - goto NotFound; - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, char value3, char value4, int length) - { - Debug.Assert(length >= 0); - - nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations - nuint lengthToExamine = (nuint)(uint)length; - - if (Vector128.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector128.Count; - if (vectorDiff >= 0) - { - // >= Vector128 is accelerated and length is enough to use them, so use that path. - // We jump forward to the intrinsics at the end of them method so a naive branch predict - // will choose the non-intrinsic path so short lengths which don't gain anything aren't - // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths - // more than make this back from the intrinsics. - lengthToExamine = (nuint)vectorDiff; - goto IntrinsicsCompare; - } - } - else if (Vector.IsHardwareAccelerated) - { - // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice. - nint vectorDiff = (nint)length - Vector.Count; - if (vectorDiff >= 0) - { - // Similar as above for Vector version - lengthToExamine = (nuint)vectorDiff; - goto VectorCompare; - } - } - - int lookUp; - while (lengthToExamine >= 4) - { - ref char current = ref Add(ref searchStart, offset); - - lookUp = current; - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp) - goto Found; - lookUp = Unsafe.Add(ref current, 1); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp) - goto Found1; - lookUp = Unsafe.Add(ref current, 2); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp) - goto Found2; - lookUp = Unsafe.Add(ref current, 3); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp) - goto Found3; - - offset += 4; - lengthToExamine -= 4; - } - - while (lengthToExamine > 0) - { - lookUp = Add(ref searchStart, offset); - if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp) - goto Found; - - offset += 1; - lengthToExamine -= 1; - } - NotFound: - return -1; + ThrowMustBeNullTerminatedString(); Found3: return (int)(offset + 3); Found2: @@ -1540,268 +658,7 @@ public static unsafe int IndexOfAny(ref char searchStart, char value0, char valu Found1: return (int)(offset + 1); Found: - return (int)offset; - - IntrinsicsCompare: - // When we move into a Vectorized block, we process everything of Vector size; - // and then for any remainder we do a final compare of Vector size but starting at - // the end and forwards, which may overlap on an earlier compare. - - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (Vector128.IsHardwareAccelerated) - { - uint matches; - ref ushort ushortSearchStart = ref Unsafe.As(ref searchStart); - if (Vector256.IsHardwareAccelerated) - { - Vector256 search; - // Guard as we may only have a valid size for Vector128; when we will move to the Sse2 - // We have already subtracted Vector128.Count from lengthToExamine so compare against that - // to see if we have double the size for Vector256.Count - if (lengthToExamine >= (nuint)Vector128.Count) - { - Vector256 values0 = Vector256.Create((ushort)value0); - Vector256 values1 = Vector256.Create((ushort)value1); - Vector256 values2 = Vector256.Create((ushort)value2); - Vector256 values3 = Vector256.Create((ushort)value3); - Vector256 values4 = Vector256.Create((ushort)value4); - - // Subtract Vector128.Count so we have now subtracted Vector256.Count - lengthToExamine -= (nuint)Vector128.Count; - // First time this checks again against 0, however we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector256.LoadUnsafe(ref ushortSearchStart, offset); - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search) - | Vector256.Equals(values3, search) | Vector256.Equals(values4, search)) - .AsByte().ExtractMostSignificantBits(); - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) - { - // None matched - offset += (nuint)Vector256.Count; - continue; - } - - goto IntrinsicsMatch; - } - - // Move to Vector length from end for final compare - search = Vector256.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search) - | Vector256.Equals(values3, search) | Vector256.Equals(values4, search)) - .AsByte().ExtractMostSignificantBits(); - if (matches == 0) - { - // None matched - goto NotFound; - } - - goto IntrinsicsMatch; - } - } - - // Initial size check was done on method entry. - Debug.Assert(length >= Vector128.Count); - { - Vector128 search, compareResult; - Vector128 values0 = Vector128.Create((ushort)value0); - Vector128 values1 = Vector128.Create((ushort)value1); - Vector128 values2 = Vector128.Create((ushort)value2); - Vector128 values3 = Vector128.Create((ushort)value3); - Vector128 values4 = Vector128.Create((ushort)value4); - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = Vector128.LoadUnsafe(ref ushortSearchStart, offset); - - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search) - | Vector128.Equals(values3, search) | Vector128.Equals(values4, search); - if (compareResult == Vector128.Zero) - { - // None matched - offset += (nuint)Vector128.Count; - continue; - } - - // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - matches = compareResult.AsByte().ExtractMostSignificantBits(); - goto IntrinsicsMatch; - } - // Move to Vector length from end for final compare - search = Vector128.LoadUnsafe(ref ushortSearchStart, lengthToExamine); - offset = lengthToExamine; - // Same as method as above - compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search) - | Vector128.Equals(values3, search) | Vector128.Equals(values4, search); - if (compareResult == Vector128.Zero) - { - // None matched - goto NotFound; - } - matches = compareResult.AsByte().ExtractMostSignificantBits(); - } - - IntrinsicsMatch: - // Find bitflag offset of first difference and add to current offset, - // flags are in bytes so divide by 2 for chars (shift right by 1) - offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1; - goto Found; - } - - VectorCompare: - // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported. - if (!Vector128.IsHardwareAccelerated && Vector.IsHardwareAccelerated) - { - Vector values0 = new Vector(value0); - Vector values1 = new Vector(value1); - Vector values2 = new Vector(value2); - Vector values3 = new Vector(value3); - Vector values4 = new Vector(value4); - - Vector search; - // First time this checks against 0 and we will move into final compare if it fails. - while (lengthToExamine > offset) - { - search = LoadVector(ref searchStart, offset); - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)), - Vector.Equals(search, values3)), - Vector.Equals(search, values4)); - if (Vector.Zero.Equals(search)) - { - // None matched - offset += (nuint)Vector.Count; - continue; - } - - goto VectorMatch; - } - - // Move to Vector length from end for final compare - search = LoadVector(ref searchStart, lengthToExamine); - offset = lengthToExamine; - search = Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.BitwiseOr( - Vector.Equals(search, values0), - Vector.Equals(search, values1)), - Vector.Equals(search, values2)), - Vector.Equals(search, values3)), - Vector.Equals(search, values4)); - if (Vector.Zero.Equals(search)) - { - // None matched - goto NotFound; - } - - VectorMatch: - offset += (nuint)(uint)LocateFirstFoundChar(search); - goto Found; - } - - Debug.Fail("Unreachable"); - goto NotFound; - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] - public static unsafe int LastIndexOf(ref char searchSpace, char value, int length) - { - Debug.Assert(length >= 0); - - fixed (char* pChars = &searchSpace) - { - char* pCh = pChars + length; - char* pEndCh = pChars; - - if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) - { - // Figure out how many characters to read sequentially from the end until we are vector aligned - // This is equivalent to: length = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte - const int elementsPerByte = sizeof(ushort) / sizeof(byte); - length = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; - } - - SequentialScan: - while (length >= 4) - { - length -= 4; - pCh -= 4; - - if (*(pCh + 3) == value) - goto Found3; - if (*(pCh + 2) == value) - goto Found2; - if (*(pCh + 1) == value) - goto Found1; - if (*pCh == value) - goto Found; - } - - while (length > 0) - { - length--; - pCh--; - - if (*pCh == value) - goto Found; - } - - // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow - // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. - if (Vector.IsHardwareAccelerated && pCh > pEndCh) - { - // Get the highest multiple of Vector.Count that is within the search space. - // That will be how many times we iterate in the loop below. - // This is equivalent to: length = Vector.Count * ((int)(pCh - pEndCh) / Vector.Count) - length = (int)((pCh - pEndCh) & ~(Vector.Count - 1)); - - // Get comparison Vector - Vector vComparison = new Vector(value); - - while (length > 0) - { - char* pStart = pCh - Vector.Count; - // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh (and hence pSart) is always vector aligned - Debug.Assert(((int)pStart & (Unsafe.SizeOf>() - 1)) == 0); - Vector vMatches = Vector.Equals(vComparison, Unsafe.Read>(pStart)); - if (Vector.Zero.Equals(vMatches)) - { - pCh -= Vector.Count; - length -= Vector.Count; - continue; - } - // Find offset of last match - return (int)(pStart - pEndCh) + LocateLastFoundChar(vMatches); - } - - if (pCh > pEndCh) - { - length = (int)(pCh - pEndCh); - goto SequentialScan; - } - } - - return -1; - Found: - return (int)(pCh - pEndCh); - Found1: - return (int)(pCh - pEndCh) + 1; - Found2: - return (int)(pCh - pEndCh) + 2; - Found3: - return (int)(pCh - pEndCh) + 3; - } + return (int)(offset); } // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138 @@ -1829,35 +686,6 @@ private static int LocateFirstFoundChar(Vector match) private static int LocateFirstFoundChar(ulong match) => BitOperations.TrailingZeroCount(match) >> 4; - // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138 - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int LocateLastFoundChar(Vector match) - { - var vector64 = Vector.AsVectorUInt64(match); - ulong candidate = 0; - int i = Vector.Count - 1; - - // This pattern is only unrolled by the Jit if the limit is Vector.Count - // As such, we need a dummy iteration variable for that condition to be satisfied - for (int j = 0; j < Vector.Count; j++) - { - candidate = vector64[i]; - if (candidate != 0) - { - break; - } - - i--; - } - - // Single LEA instruction with jitted const (using function result) - return i * 4 + LocateLastFoundChar(candidate); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int LocateLastFoundChar(ulong match) - => BitOperations.Log2(match) >> 4; - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector LoadVector(ref char start, nint offset) => Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref start, offset))); @@ -1866,25 +694,6 @@ private static Vector LoadVector(ref char start, nint offset) private static Vector LoadVector(ref char start, nuint offset) => Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref start, (nint)offset))); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 LoadVector128(ref char start, nint offset) - => Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref start, offset))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 LoadVector128(ref char start, nuint offset) - => Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref start, (nint)offset))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 LoadVector256(ref char start, nint offset) - => Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref start, offset))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 LoadVector256(ref char start, nuint offset) - => Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref start, (nint)offset))); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ref char Add(ref char start, nuint offset) => ref Unsafe.Add(ref start, (nint)offset); - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static nint GetCharVectorSpanLength(nint offset, nint length) => (length - offset) & ~(Vector.Count - 1); @@ -1922,21 +731,6 @@ private static unsafe nint UnalignedCountVector128(ref char searchSpace) return (nint)(uint)(-(int)Unsafe.AsPointer(ref searchSpace) / ElementsPerByte) & (Vector128.Count - 1); } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int FindFirstMatchedLane(Vector128 compareResult) - { - Debug.Assert(AdvSimd.Arm64.IsSupported); - - Vector128 pairwiseSelectedLane = AdvSimd.Arm64.AddPairwise(compareResult.AsByte(), compareResult.AsByte()); - ulong selectedLanes = pairwiseSelectedLane.AsUInt64().ToScalar(); - - // It should be handled by compareResult != Vector.Zero - Debug.Assert(selectedLanes != 0); - - return BitOperations.TrailingZeroCount(selectedLanes) >> 3; - } - public static void Reverse(ref char buf, nuint length) { if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 709a423680af0..b093cb11a29e6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -293,115 +293,6 @@ public static unsafe bool Contains(ref T searchSpace, T value, int length) wh return true; } - internal static unsafe int IndexOfValueType(ref T searchSpace, T value, int length) where T : struct, IEquatable - { - Debug.Assert(length >= 0); - - nint index = 0; // Use nint for arithmetic to avoid unnecessary 64->32->64 truncations - if (Vector.IsHardwareAccelerated && Vector.IsSupported && (Vector.Count * 2) <= length) - { - Vector valueVector = new Vector(value); - Vector compareVector; - Vector matchVector; - if ((uint)length % (uint)Vector.Count != 0) - { - // Number of elements is not a multiple of Vector.Count, so do one - // check and shift only enough for the remaining set to be a multiple - // of Vector.Count. - compareVector = Unsafe.As>(ref Unsafe.Add(ref searchSpace, index)); - matchVector = Vector.Equals(valueVector, compareVector); - if (matchVector != Vector.Zero) - { - goto VectorMatch; - } - index += length % Vector.Count; - length -= length % Vector.Count; - } - while (length > 0) - { - compareVector = Unsafe.As>(ref Unsafe.Add(ref searchSpace, index)); - matchVector = Vector.Equals(valueVector, compareVector); - if (matchVector != Vector.Zero) - { - goto VectorMatch; - } - index += Vector.Count; - length -= Vector.Count; - } - goto NotFound; - VectorMatch: - for (int i = 0; i < Vector.Count; i++) - if (compareVector[i].Equals(value)) - return (int)(index + i); - } - - while (length >= 8) - { - if (value.Equals(Unsafe.Add(ref searchSpace, index))) - goto Found; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 1))) - goto Found1; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 2))) - goto Found2; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 3))) - goto Found3; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 4))) - goto Found4; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 5))) - goto Found5; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 6))) - goto Found6; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 7))) - goto Found7; - - length -= 8; - index += 8; - } - - while (length >= 4) - { - if (value.Equals(Unsafe.Add(ref searchSpace, index))) - goto Found; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 1))) - goto Found1; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 2))) - goto Found2; - if (value.Equals(Unsafe.Add(ref searchSpace, index + 3))) - goto Found3; - - length -= 4; - index += 4; - } - - while (length > 0) - { - if (value.Equals(Unsafe.Add(ref searchSpace, index))) - goto Found; - - index += 1; - length--; - } - NotFound: - return -1; - - Found: // Workaround for https://github.com/dotnet/runtime/issues/8795 - return (int)index; - Found1: - return (int)(index + 1); - Found2: - return (int)(index + 2); - Found3: - return (int)(index + 3); - Found4: - return (int)(index + 4); - Found5: - return (int)(index + 5); - Found6: - return (int)(index + 6); - Found7: - return (int)(index + 7); - } - public static unsafe int IndexOf(ref T searchSpace, T value, int length) where T : IEquatable? { Debug.Assert(length >= 0); @@ -1162,7 +1053,7 @@ public static int LastIndexOfAny(ref T searchSpace, int searchSpaceLength, re return -1; // not found } - public static int IndexOfAnyExcept(ref T searchSpace, T value0, int length) + internal static int IndexOfAnyExcept(ref T searchSpace, T value0, int length) { Debug.Assert(length >= 0, "Expected non-negative length"); @@ -1177,113 +1068,125 @@ public static int IndexOfAnyExcept(ref T searchSpace, T value0, int length) return -1; } - public static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, int length) where T : struct, IEquatable + internal static int LastIndexOfAnyExcept(ref T searchSpace, T value0, int length) { Debug.Assert(length >= 0, "Expected non-negative length"); - Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); - if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + for (int i = length -1; i >= 0; i--) { - for (int i = 0; i < length; i++) + if (!EqualityComparer.Default.Equals(Unsafe.Add(ref searchSpace, i), value0)) { - if (!Unsafe.Add(ref searchSpace, i).Equals(value0)) - { - return i; - } + return i; } } - else - { - Vector128 notEquals, value0Vector = Vector128.Create(value0); - ref T current = ref searchSpace; - ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); - // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do - { - notEquals = ~Vector128.Equals(value0Vector, Vector128.LoadUnsafe(ref current)); - if (notEquals != Vector128.Zero) - { - return ComputeIndex(ref searchSpace, ref current, notEquals); - } + return -1; + } - current = ref Unsafe.Add(ref current, Vector128.Count); - } - while (!Unsafe.IsAddressGreaterThan(ref current, ref oneVectorAwayFromEnd)); + internal static int IndexOfAnyExcept(ref T searchSpace, T value0, T value1, int length) + { + Debug.Assert(length >= 0, "Expected non-negative length"); - // If any elements remain, process the last vector in the search space. - if ((uint)length % Vector128.Count != 0) + for (int i = 0; i < length; i++) + { + ref T current = ref Unsafe.Add(ref searchSpace, i); + if (!EqualityComparer.Default.Equals(current, value0) && !EqualityComparer.Default.Equals(current, value1)) { - notEquals = ~Vector128.Equals(value0Vector, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)); - if (notEquals != Vector128.Zero) - { - return ComputeIndex(ref searchSpace, ref oneVectorAwayFromEnd, notEquals); - } + return i; } } return -1; } - internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, IEquatable + internal static int LastIndexOfAnyExcept(ref T searchSpace, T value0, T value1, int length) { Debug.Assert(length >= 0, "Expected non-negative length"); - Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); - if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + for (int i = length - 1; i >= 0; i--) { - for (int i = 0; i < length; i++) + ref T current = ref Unsafe.Add(ref searchSpace, i); + if (!EqualityComparer.Default.Equals(current, value0) && !EqualityComparer.Default.Equals(current, value1)) { - T current = Unsafe.Add(ref searchSpace, i); - if (!current.Equals(value0) && !current.Equals(value1) && !current.Equals(value2) && !current.Equals(value3)) - { - return i; - } + return i; } } - else - { - Vector128 notEquals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3); - ref T currentSearchSpace = ref searchSpace; - ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); - // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + return -1; + } + + internal static int IndexOfAnyExcept(ref T searchSpace, T value0, T value1, T value2, int length) + { + Debug.Assert(length >= 0, "Expected non-negative length"); + + for (int i = 0; i < length; i++) + { + ref T current = ref Unsafe.Add(ref searchSpace, i); + if (!EqualityComparer.Default.Equals(current, value0) + && !EqualityComparer.Default.Equals(current, value1) + && !EqualityComparer.Default.Equals(current, value2)) { - current = Vector128.LoadUnsafe(ref currentSearchSpace); - notEquals = ~(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) - | Vector128.Equals(values2, current) | Vector128.Equals(values3, current)); - if (notEquals != Vector128.Zero) - { - return ComputeIndex(ref searchSpace, ref currentSearchSpace, notEquals); - } + return i; + } + } + + return -1; + } + + internal static int LastIndexOfAnyExcept(ref T searchSpace, T value0, T value1, T value2, int length) + { + Debug.Assert(length >= 0, "Expected non-negative length"); - currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128.Count); + for (int i = length - 1; i >= 0; i--) + { + ref T current = ref Unsafe.Add(ref searchSpace, i); + if (!EqualityComparer.Default.Equals(current, value0) + && !EqualityComparer.Default.Equals(current, value1) + && !EqualityComparer.Default.Equals(current, value2)) + { + return i; } - while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + } - // If any elements remain, process the last vector in the search space. - if ((uint)length % Vector128.Count != 0) + return -1; + } + + internal static int IndexOfAnyExcept(ref T searchSpace, T value0, T value1, T value2, T value3, int length) + { + Debug.Assert(length >= 0, "Expected non-negative length"); + + for (int i = 0; i < length; i++) + { + ref T current = ref Unsafe.Add(ref searchSpace, i); + if (!EqualityComparer.Default.Equals(current, value0) + && !EqualityComparer.Default.Equals(current, value1) + && !EqualityComparer.Default.Equals(current, value2) + && !EqualityComparer.Default.Equals(current, value3)) { - current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - notEquals = ~(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) - | Vector128.Equals(values2, current) | Vector128.Equals(values3, current)); - if (notEquals != Vector128.Zero) - { - return ComputeIndex(ref searchSpace, ref oneVectorAwayFromEnd, notEquals); - } + return i; } } return -1; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int ComputeIndex(ref T searchSpace, ref T current, Vector128 notEquals) where T : struct, IEquatable + internal static int LastIndexOfAnyExcept(ref T searchSpace, T value0, T value1, T value2, T value3, int length) { - uint notEqualsElements = notEquals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()); + Debug.Assert(length >= 0, "Expected non-negative length"); + + for (int i = length - 1; i >= 0; i--) + { + ref T current = ref Unsafe.Add(ref searchSpace, i); + if (!EqualityComparer.Default.Equals(current, value0) + && !EqualityComparer.Default.Equals(current, value1) + && !EqualityComparer.Default.Equals(current, value2) + && !EqualityComparer.Default.Equals(current, value3)) + { + return i; + } + } + + return -1; } public static bool SequenceEqual(ref T first, ref T second, int length) where T : IEquatable? @@ -1395,5 +1298,1373 @@ public static int SequenceCompareTo(ref T first, int firstLength, ref T secon } return firstLength.CompareTo(secondLength); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool CanVectorizeAndBenefit(int length) where T : IEquatable? + { + if (Vector128.IsHardwareAccelerated && RuntimeHelpers.IsBitwiseEquatable()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + return length >= Vector128.Count; + } + else if (Unsafe.SizeOf() == sizeof(short)) + { + return length >= Vector128.Count; + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + return length >= Vector128.Count; + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + return length >= Vector128.Count; + } + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = 0; + + while (length >= 8) + { + length -= 8; + + if (Unsafe.Add(ref searchSpace, offset) == value + || Unsafe.Add(ref searchSpace, offset + 1) == value + || Unsafe.Add(ref searchSpace, offset + 2) == value + || Unsafe.Add(ref searchSpace, offset + 3) == value + || Unsafe.Add(ref searchSpace, offset + 4) == value + || Unsafe.Add(ref searchSpace, offset + 5) == value + || Unsafe.Add(ref searchSpace, offset + 6) == value + || Unsafe.Add(ref searchSpace, offset + 7) == value) + { + return true; + } + + offset += 8; + } + + if (length >= 4) + { + length -= 4; + + if (Unsafe.Add(ref searchSpace, offset) == value + || Unsafe.Add(ref searchSpace, offset + 1) == value + || Unsafe.Add(ref searchSpace, offset + 2) == value + || Unsafe.Add(ref searchSpace, offset + 3) == value) + { + return true; + } + + offset += 4; + } + + while (length > 0) + { + length -= 1; + + if (Unsafe.Add(ref searchSpace, offset) == value) return true; + + offset += 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, values = Vector256.Create(value); + ref T currentSearchSpace = ref searchSpace; + ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256.Count); + continue; + } + + return true; + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the last vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd)); + if (equals != Vector256.Zero) + { + return true; + } + } + } + else + { + Vector128 equals, values = Vector128.Create(value); + ref T currentSearchSpace = ref searchSpace; + ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128.Count); + continue; + } + + return true; + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)); + if (equals != Vector128.Zero) + { + return true; + } + } + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfChar(ref char searchSpace, char value, int length) + => IndexOfValueType(ref Unsafe.As(ref searchSpace), (short)value, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfValueType(ref T searchSpace, T value, int length) where T : struct, INumber + => IndexOfValueType>(ref searchSpace, value, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value, int length) where T : struct, INumber + => IndexOfValueType>(ref searchSpace, value, length); + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int IndexOfValueType(ref TValue searchSpace, TValue value, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = 0; + + while (length >= 8) + { + length -= 8; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 4) == value)) return (int)offset + 4; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 5) == value)) return (int)offset + 5; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 6) == value)) return (int)offset + 6; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 7) == value)) return (int)offset + 7; + + offset += 8; + } + + if (length >= 4) + { + length -= 4; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3; + + offset += 4; + } + + while (length > 0) + { + length -= 1; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset; + + offset += 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, values = Vector256.Create(value); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace))); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the last vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd))); + if (equals != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + else + { + Vector128 equals, values = Vector128.Create(value); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace))); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd))); + if (equals != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyChar(ref char searchSpace, char value0, char value1, int length) + => IndexOfAnyValueType(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber + => IndexOfAnyValueType>(ref searchSpace, value0, value1, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber + => IndexOfAnyValueType>(ref searchSpace, value0, value1, length); + + // having INumber constraint here allows to use == operator and get better perf compared to .Equals + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = 0; + TValue lookUp; + + if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte + { + while (length >= 8) + { + length -= 8; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + lookUp = Unsafe.Add(ref current, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1; + lookUp = Unsafe.Add(ref current, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2; + lookUp = Unsafe.Add(ref current, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3; + lookUp = Unsafe.Add(ref current, 4); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 4; + lookUp = Unsafe.Add(ref current, 5); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 5; + lookUp = Unsafe.Add(ref current, 6); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 6; + lookUp = Unsafe.Add(ref current, 7); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 7; + + offset += 8; + } + } + + while (length >= 4) + { + length -= 4; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + lookUp = Unsafe.Add(ref current, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1; + lookUp = Unsafe.Add(ref current, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2; + lookUp = Unsafe.Add(ref current, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3; + + offset += 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + + offset += 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector256.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the last vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)); + if (equals != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + else + { + Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector128.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)); + if (equals != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber + => IndexOfAnyValueType>(ref searchSpace, value0, value1, value2, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber + => IndexOfAnyValueType>(ref searchSpace, value0, value1, value2, length); + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = 0; + TValue lookUp; + + if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte + { + while (length >= 8) + { + length -= 8; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + lookUp = Unsafe.Add(ref current, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1; + lookUp = Unsafe.Add(ref current, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2; + lookUp = Unsafe.Add(ref current, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3; + lookUp = Unsafe.Add(ref current, 4); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 4; + lookUp = Unsafe.Add(ref current, 5); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 5; + lookUp = Unsafe.Add(ref current, 6); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 6; + lookUp = Unsafe.Add(ref current, 7); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 7; + + offset += 8; + } + } + + while (length >= 4) + { + length -= 4; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + lookUp = Unsafe.Add(ref current, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1; + lookUp = Unsafe.Add(ref current, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2; + lookUp = Unsafe.Add(ref current, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3; + + offset += 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + + offset += 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector256.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current)); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the last vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current)); + if (equals != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + else + { + Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector128.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current)); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current)); + if (equals != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber + => IndexOfAnyValueType>(ref searchSpace, value0, value1, value2, value3, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber + => IndexOfAnyValueType>(ref searchSpace, value0, value1, value2, value3, length); + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, TValue value3, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = 0; + TValue lookUp; + + while (length >= 4) + { + length -= 4; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset; + lookUp = Unsafe.Add(ref current, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 1; + lookUp = Unsafe.Add(ref current, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 2; + lookUp = Unsafe.Add(ref current, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 3; + + offset += 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset; + + offset += 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector256.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) + | Vector256.Equals(values2, current) | Vector256.Equals(values3, current)); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the last vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) + | Vector256.Equals(values2, current) | Vector256.Equals(values3, current)); + if (equals != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + else + { + Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3); + ref TValue currentSearchSpace = ref searchSpace; + ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector128.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) + | Vector128.Equals(values2, current) | Vector128.Equals(values3, current)); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) + | Vector128.Equals(values2, current) | Vector128.Equals(values3, current)); + if (equals != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1, T value2, T value3, T value4, int length) + where T : struct, INumber + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = 0; + T lookUp; + + while (length >= 4) + { + length -= 4; + + ref T current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset; + lookUp = Unsafe.Add(ref current, 1); + if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset + 1; + lookUp = Unsafe.Add(ref current, 2); + if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset + 2; + lookUp = Unsafe.Add(ref current, 3); + if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset + 3; + + offset += 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset; + + offset += 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), + values2 = Vector256.Create(value2), values3 = Vector256.Create(value3), values4 = Vector256.Create(value4); + ref T currentSearchSpace = ref searchSpace; + ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector256.LoadUnsafe(ref currentSearchSpace); + equals = Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current) + | Vector256.Equals(values3, current) | Vector256.Equals(values4, current); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the last vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current) + | Vector256.Equals(values3, current) | Vector256.Equals(values4, current); + if (equals != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + else + { + Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), + values2 = Vector128.Create(value2), values3 = Vector128.Create(value3), values4 = Vector128.Create(value4); + ref T currentSearchSpace = ref searchSpace; + ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector128.LoadUnsafe(ref currentSearchSpace); + equals = Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current) + | Vector128.Equals(values3, current) | Vector128.Equals(values4, current); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + equals = Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current) + | Vector128.Equals(values3, current) | Vector128.Equals(values4, current); + if (equals != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfValueType(ref T searchSpace, T value, int length) where T : struct, INumber + => LastIndexOfValueType>(ref searchSpace, value, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnyExceptValueType(ref T searchSpace, T value, int length) where T : struct, INumber + => LastIndexOfValueType>(ref searchSpace, value, length); + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int LastIndexOfValueType(ref TValue searchSpace, TValue value, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = (nuint)length - 1; + + while (length >= 8) + { + length -= 8; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 4) == value)) return (int)offset - 4; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 5) == value)) return (int)offset - 5; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 6) == value)) return (int)offset - 6; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 7) == value)) return (int)offset - 7; + + offset -= 8; + } + + if (length >= 4) + { + length -= 4; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3; + + offset -= 4; + } + + while (length > 0) + { + length -= 1; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset; + + offset -= 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, values = Vector256.Create(value); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace))); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace))); + if (equals != Vector256.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + else + { + Vector128 equals, values = Vector128.Create(value); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace))); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace))); + if (equals != Vector128.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnyValueType(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber + => LastIndexOfAnyValueType>(ref searchSpace, value0, value1, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber + => LastIndexOfAnyValueType>(ref searchSpace, value0, value1, length); + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int LastIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = (nuint)length - 1; + TValue lookUp; + + if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte + { + while (length >= 8) + { + length -= 8; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + lookUp = Unsafe.Add(ref current, -1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1; + lookUp = Unsafe.Add(ref current, -2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2; + lookUp = Unsafe.Add(ref current, -3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3; + lookUp = Unsafe.Add(ref current, -4); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 4; + lookUp = Unsafe.Add(ref current, -5); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 5; + lookUp = Unsafe.Add(ref current, -6); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 6; + lookUp = Unsafe.Add(ref current, -7); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 7; + + offset -= 8; + } + } + + while (length >= 4) + { + length -= 4; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + lookUp = Unsafe.Add(ref current, -1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1; + lookUp = Unsafe.Add(ref current, -2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2; + lookUp = Unsafe.Add(ref current, -3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3; + + offset -= 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + + offset -= 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector256.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + current = Vector256.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)); + if (equals != Vector256.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + else + { + Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector128.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + current = Vector128.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)); + if (equals != Vector128.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnyValueType(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber + => LastIndexOfAnyValueType>(ref searchSpace, value0, value1, value2, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber + => LastIndexOfAnyValueType>(ref searchSpace, value0, value1, value2, length); + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int LastIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = (nuint)length - 1; + TValue lookUp; + + if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte + { + while (length >= 8) + { + length -= 8; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + lookUp = Unsafe.Add(ref current, -1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1; + lookUp = Unsafe.Add(ref current, -2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2; + lookUp = Unsafe.Add(ref current, -3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3; + lookUp = Unsafe.Add(ref current, -4); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 4; + lookUp = Unsafe.Add(ref current, -5); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 5; + lookUp = Unsafe.Add(ref current, -6); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 6; + lookUp = Unsafe.Add(ref current, -7); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 7; + + offset -= 8; + } + } + + while (length >= 4) + { + length -= 4; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + lookUp = Unsafe.Add(ref current, -1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1; + lookUp = Unsafe.Add(ref current, -2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2; + lookUp = Unsafe.Add(ref current, -3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3; + + offset -= 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + + offset -= 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector256.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2)); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + current = Vector256.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2)); + if (equals != Vector256.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + else + { + Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector128.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2)); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + current = Vector128.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2)); + if (equals != Vector128.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnyValueType(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber + => LastIndexOfAnyValueType>(ref searchSpace, value0, value1, value2, value3, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber + => LastIndexOfAnyValueType>(ref searchSpace, value0, value1, value2, value3, length); + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private static int LastIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, TValue value3, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + Debug.Assert(length >= 0, "Expected non-negative length"); + Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types"); + + if (!Vector128.IsHardwareAccelerated || length < Vector128.Count) + { + nuint offset = (nuint)length - 1; + TValue lookUp; + + while (length >= 4) + { + length -= 4; + + ref TValue current = ref Unsafe.Add(ref searchSpace, offset); + lookUp = current; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset; + lookUp = Unsafe.Add(ref current, -1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 1; + lookUp = Unsafe.Add(ref current, -2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 2; + lookUp = Unsafe.Add(ref current, -3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 3; + + offset -= 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset; + + offset -= 1; + } + } + else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) + { + Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector256.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) + | Vector256.Equals(current, values2) | Vector256.Equals(current, values3)); + if (equals == Vector256.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector256.Count != 0) + { + current = Vector256.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) + | Vector256.Equals(current, values2) | Vector256.Equals(current, values3)); + if (equals != Vector256.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + else + { + Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3); + ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + current = Vector128.LoadUnsafe(ref currentSearchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) + | Vector128.Equals(current, values2) | Vector128.Equals(current, values3)); + if (equals == Vector128.Zero) + { + currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + continue; + } + + return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + } + while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); + + // If any elements remain, process the first vector in the search space. + if ((uint)length % Vector128.Count != 0) + { + current = Vector128.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) + | Vector128.Equals(current, values2) | Vector128.Equals(current, values3)); + if (equals != Vector128.Zero) + { + return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndex(ref T searchSpace, ref T current, Vector128 equals) where T : struct + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int index = BitOperations.TrailingZeroCount(notEqualsElements); + return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndex(ref T searchSpace, ref T current, Vector256 equals) where T : struct + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int index = BitOperations.TrailingZeroCount(notEqualsElements); + return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeLastIndex(ref T searchSpace, ref T current, Vector128 equals) where T : struct + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) + return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()) + index; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeLastIndex(ref T searchSpace, ref T current, Vector256 equals) where T : struct + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) + return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()) + index; + } + + private interface INegator where T : struct + { + static abstract bool NegateIfNeeded(bool equals); + static abstract Vector128 NegateIfNeeded(Vector128 equals); + static abstract Vector256 NegateIfNeeded(Vector256 equals); + } + + private readonly struct DontNegate : INegator where T : struct + { + public static bool NegateIfNeeded(bool equals) => equals; + public static Vector128 NegateIfNeeded(Vector128 equals) => equals; + public static Vector256 NegateIfNeeded(Vector256 equals) => equals; + } + + private readonly struct Negate : INegator where T : struct + { + public static bool NegateIfNeeded(bool equals) => !equals; + public static Vector128 NegateIfNeeded(Vector128 equals) => ~equals; + public static Vector256 NegateIfNeeded(Vector256 equals) => ~equals; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index d35b5e2d2034e..c46f5ec8ae425 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -3,9 +3,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; namespace System diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 675a8e188477b..50b30cb5b01ec 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -1088,7 +1088,7 @@ public string Replace(string oldValue, string? newValue) int i = 0; while (true) { - int pos = SpanHelpers.IndexOf(ref Unsafe.Add(ref _firstChar, i), c, Length - i); + int pos = SpanHelpers.IndexOfChar(ref Unsafe.Add(ref _firstChar, i), c, Length - i); if (pos < 0) { break; diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index 945959f104a7e..df95295a116fa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -27,7 +27,8 @@ public bool Contains(string value, StringComparison comparisonType) #pragma warning restore CA2249 } - public bool Contains(char value) => SpanHelpers.Contains(ref _firstChar, value, Length); + public bool Contains(char value) + => SpanHelpers.ContainsValueType(ref Unsafe.As(ref _firstChar), (short)value, Length); public bool Contains(char value, StringComparison comparisonType) { @@ -36,8 +37,7 @@ public bool Contains(char value, StringComparison comparisonType) // Returns the index of the first occurrence of a specified character in the current instance. // The search starts at startIndex and runs thorough the next count characters. - // - public int IndexOf(char value) => SpanHelpers.IndexOf(ref _firstChar, value, Length); + public int IndexOf(char value) => SpanHelpers.IndexOfChar(ref _firstChar, value, Length); public int IndexOf(char value, int startIndex) { @@ -78,10 +78,10 @@ private int IndexOfCharOrdinalIgnoreCase(char value) { char valueUc = (char)(value | 0x20); char valueLc = (char)(value & ~0x20); - return SpanHelpers.IndexOfAny(ref _firstChar, valueLc, valueUc, Length); + return SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); } - return SpanHelpers.IndexOf(ref _firstChar, value, Length); + return SpanHelpers.IndexOfChar(ref _firstChar, value, Length); } public unsafe int IndexOf(char value, int startIndex, int count) @@ -96,7 +96,7 @@ public unsafe int IndexOf(char value, int startIndex, int count) ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count, ExceptionResource.ArgumentOutOfRange_Count); } - int result = SpanHelpers.IndexOf(ref Unsafe.Add(ref _firstChar, startIndex), value, count); + int result = SpanHelpers.IndexOfChar(ref Unsafe.Add(ref _firstChar, startIndex), value, count); return result < 0 ? result : result + startIndex; } @@ -280,8 +280,8 @@ public int IndexOf(string value, int startIndex, int count, StringComparison com // The search starts at startIndex and runs backwards to startIndex - count + 1. // The character at position startIndex is included in the search. startIndex is the larger // index within the string. - // - public int LastIndexOf(char value) => SpanHelpers.LastIndexOf(ref _firstChar, value, Length); + public int LastIndexOf(char value) + => SpanHelpers.LastIndexOfValueType(ref Unsafe.As(ref _firstChar), (short)value, Length); public int LastIndexOf(char value, int startIndex) { @@ -306,7 +306,7 @@ public unsafe int LastIndexOf(char value, int startIndex, int count) } int startSearchAt = startIndex + 1 - count; - int result = SpanHelpers.LastIndexOf(ref Unsafe.Add(ref _firstChar, startSearchAt), value, count); + int result = SpanHelpers.LastIndexOfValueType(ref Unsafe.As(ref Unsafe.Add(ref _firstChar, startSearchAt)), (short)value, count); return result < 0 ? result : result + startSearchAt; } diff --git a/src/libraries/System.Private.CoreLib/src/System/String.cs b/src/libraries/System.Private.CoreLib/src/System/String.cs index 6fdbdfdc449e5..39d9153f1d172 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.cs @@ -594,39 +594,9 @@ public StringRuneEnumerator EnumerateRunes() return new StringRuneEnumerator(this); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe int wcslen(char* ptr) - { - // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator. - // This IndexOf behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it. - int length = SpanHelpers.IndexOf(ref *ptr, '\0', int.MaxValue); - if (length < 0) - { - ThrowMustBeNullTerminatedString(); - } + internal static unsafe int wcslen(char* ptr) => SpanHelpers.IndexOfNullCharacter(ref *ptr); - return length; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe int strlen(byte* ptr) - { - // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator. - // This IndexOf behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it. - int length = SpanHelpers.IndexOf(ref *ptr, (byte)'\0', int.MaxValue); - if (length < 0) - { - ThrowMustBeNullTerminatedString(); - } - - return length; - } - - [DoesNotReturn] - private static void ThrowMustBeNullTerminatedString() - { - throw new ArgumentException(SR.Arg_MustBeNullTerminatedString); - } + internal static unsafe int strlen(byte* ptr) => SpanHelpers.IndexOfNullByte(ref *ptr); // // IConvertible implementation