New ASCII APIs (#75012)

Co-authored-by: Levi Broderick <levib@microsoft.com> Co-authored-by: Stephen Toub <stoub@microsoft.com> Co-authored-by: Miha Zupan <mihazupan.zupan1@gmail.com>
dotnet · Dec 21, 2022 · 66e64e5 · 66e64e5
1 parent 5fdc188
commit 66e64e5
Show file tree

Hide file tree

Showing 49 changed files with 2,248 additions and 933 deletions.
diff --git a/...clr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/PInvokeMarshal.cs b/...clr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/PInvokeMarshal.cs
@@ -10,6 +10,8 @@
 using Internal.Runtime.Augments;
 using Internal.Runtime.CompilerHelpers;
 using Internal.Runtime.CompilerServices;
+using System.Text;
+using System.Buffers;
 
 namespace System.Runtime.InteropServices
 {
@@ -502,17 +504,7 @@ public static unsafe char AnsiCharToWideChar(byte nativeValue)
         internal static unsafe byte* StringToAnsiString(char* pManaged, int lenUnicode, byte* pNative, bool terminateWithNull,
             bool bestFit, bool throwOnUnmappableChar)
         {
-            bool allAscii = true;
-
-            for (int i = 0; i < lenUnicode; i++)
-            {
-                if (pManaged[i] >= 128)
-                {
-                    allAscii = false;
-                    break;
-                }
-            }
-
+            bool allAscii = Ascii.IsValid(new ReadOnlySpan<char>(pManaged, lenUnicode));
             int length;
 
             if (allAscii) // If all ASCII, map one UNICODE character to one ANSI char
@@ -530,17 +522,8 @@ public static unsafe char AnsiCharToWideChar(byte nativeValue)
             }
             if (allAscii) // ASCII conversion
             {
-                byte* pDst = pNative;
-                char* pSrc = pManaged;
-
-                while (lenUnicode > 0)
-                {
-                    unchecked
-                    {
-                        *pDst++ = (byte)(*pSrc++);
-                        lenUnicode--;
-                    }
-                }
+                OperationStatus conversionStatus = Ascii.FromUtf16(new ReadOnlySpan<char>(pManaged, length), new Span<byte>(pNative, length), out _);
+                Debug.Assert(conversionStatus == OperationStatus.Done);
             }
             else // Let OS convert
             {
@@ -566,26 +549,9 @@ public static unsafe char AnsiCharToWideChar(byte nativeValue)
         /// </summary>
         private static unsafe bool CalculateStringLength(byte* pchBuffer, out int ansiBufferLen, out int unicodeBufferLen)
         {
-            ansiBufferLen = 0;
-
-            bool allAscii = true;
-
-            {
-                byte* p = pchBuffer;
-                byte b = *p++;
-
-                while (b != 0)
-                {
-                    if (b >= 128)
-                    {
-                        allAscii = false;
-                    }
-
-                    ansiBufferLen++;
-
-                    b = *p++;
-                }
-            }
+            ReadOnlySpan<byte> span = MemoryMarshal.CreateReadOnlySpanFromNullTerminated(pchBuffer);
+            ansiBufferLen = span.Length;
+            bool allAscii = Ascii.IsValid(span);
 
             if (allAscii)
             {

diff --git a/src/libraries/Common/tests/System/Net/Http/Http2LoopbackConnection.cs b/src/libraries/Common/tests/System/Net/Http/Http2LoopbackConnection.cs
@@ -91,7 +91,7 @@ private async Task ReadPrefixAsync()
                 throw new Exception("Connection stream closed while attempting to read connection preface.");
             }
 
-            if (Text.Encoding.ASCII.GetString(_prefix).Contains("HTTP/1.1"))
+            if (_prefix.AsSpan().IndexOf("HTTP/1.1"u8) >= 0)
             {
                 // Tests that use HttpAgnosticLoopbackServer will attempt to send an HTTP/1.1 request to an HTTP/2 server.
                 // This is invalid and we should terminate the connection.

diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/Headers/ContentDispositionHeaderValue.cs b/src/libraries/System.Net.Http/src/System/Net/Http/Headers/ContentDispositionHeaderValue.cs
@@ -422,7 +422,7 @@ private static string EncodeAndQuoteMime(string input)
                 throw new ArgumentException(SR.Format(CultureInfo.InvariantCulture,
                     SR.net_http_headers_invalid_value, input));
             }
-            else if (HeaderUtilities.ContainsNonAscii(result))
+            else if (!Ascii.IsValid(result))
             {
                 needsQuotes = true; // Encoded data must always be quoted, the equals signs are invalid in tokens.
                 result = EncodeMime(result); // =?utf-8?B?asdfasdfaesdf?=

diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/Headers/HeaderUtilities.cs b/src/libraries/System.Net.Http/src/System/Net/Http/Headers/HeaderUtilities.cs
@@ -63,20 +63,6 @@ internal static void SetQuality(UnvalidatedObjectCollection<NameValueHeaderValue
             }
         }
 
-        internal static bool ContainsNonAscii(string input)
-        {
-            Debug.Assert(input != null);
-
-            foreach (char c in input)
-            {
-                if ((int)c > 0x7f)
-                {
-                    return true;
-                }
-            }
-            return false;
-        }
-
         // Encode a string using RFC 5987 encoding.
         // encoding'lang'PercentEncodedSpecials
         internal static string Encode5987(string input)

diff --git a/...ies/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/AuthenticationHelper.Digest.cs b/...ies/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/AuthenticationHelper.Digest.cs
@@ -88,7 +88,7 @@ internal static partial class AuthenticationHelper
             }
             else
             {
-                if (HeaderUtilities.ContainsNonAscii(credential.UserName))
+                if (!Ascii.IsValid(credential.UserName))
                 {
                     string usernameStar = HeaderUtilities.Encode5987(credential.UserName);
                     sb.AppendKeyValue(UsernameStar, usernameStar, includeQuotes: false);

diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs
@@ -1582,12 +1582,10 @@ private Task WriteAsciiStringAsync(string s, bool async)
             int offset = _writeOffset;
             if (s.Length <= _writeBuffer.Length - offset)
             {
-                byte[] writeBuffer = _writeBuffer;
-                foreach (char c in s)
-                {
-                    writeBuffer[offset++] = (byte)c;
-                }
-                _writeOffset = offset;
+                OperationStatus operationStatus = Ascii.FromUtf16(s, _writeBuffer.AsSpan(offset), out int bytesWritten);
+                Debug.Assert(operationStatus == OperationStatus.Done);
+                _writeOffset = offset + bytesWritten;
+
                 return Task.CompletedTask;
             }
 
@@ -1598,14 +1596,14 @@ private Task WriteAsciiStringAsync(string s, bool async)
 
         private async Task WriteStringAsyncSlow(string s, bool async)
         {
+            if (!Ascii.IsValid(s))
+            {
+                throw new HttpRequestException(SR.net_http_request_invalid_char_encoding);
+            }
+
             for (int i = 0; i < s.Length; i++)
             {
-                char c = s[i];
-                if ((c & 0xFF80) != 0)
-                {
-                    throw new HttpRequestException(SR.net_http_request_invalid_char_encoding);
-                }
-                await WriteByteAsync((byte)c, async).ConfigureAwait(false);
+                await WriteByteAsync((byte)s[i], async).ConfigureAwait(false);
             }
         }
 

diff --git a/src/libraries/System.Net.HttpListener/src/System.Net.HttpListener.csproj b/src/libraries/System.Net.HttpListener/src/System.Net.HttpListener.csproj
@@ -71,8 +71,6 @@
              Link="Common\System\Net\CookieFields.cs" />
     <Compile Include="$(CommonPath)System\Net\CookieParser.cs"
              Link="Common\System\Net\CookieParser.cs" />
-    <Compile Include="$(CommonPath)System\Net\CaseInsensitiveAscii.cs"
-             Link="Common\System\Net\CaseInsensitiveAscii.cs" />
     <Compile Include="$(CommonPath)System\Net\ExceptionCheck.cs"
              Link="Common\System\Net\ExceptionCheck.cs" />
     <Compile Include="$(CommonPath)System\Net\HttpStatusDescription.cs"

diff --git a/src/libraries/System.Net.HttpListener/src/System/Net/HttpListener.cs b/src/libraries/System.Net.HttpListener/src/System/Net/HttpListener.cs
@@ -1,7 +1,9 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Buffers;
 using System.Collections;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Security.Authentication.ExtendedProtection;
 using System.Text;
@@ -150,22 +152,7 @@ internal void AddPrefix(string uriPrefix)
                 {
                     throw new ArgumentException(SR.net_listener_slash, nameof(uriPrefix));
                 }
-                StringBuilder registeredPrefixBuilder = new StringBuilder();
-                if (uriPrefix[j] == ':')
-                {
-                    registeredPrefixBuilder.Append(uriPrefix);
-                }
-                else
-                {
-                    registeredPrefixBuilder.Append(uriPrefix, 0, j);
-                    registeredPrefixBuilder.Append(i == 7 ? ":80" : ":443");
-                    registeredPrefixBuilder.Append(uriPrefix, j, uriPrefix.Length - j);
-                }
-                for (i = 0; registeredPrefixBuilder[i] != ':'; i++)
-                {
-                    registeredPrefixBuilder[i] = (char)CaseInsensitiveAscii.AsciiToLower[(byte)registeredPrefixBuilder[i]];
-                }
-                registeredPrefix = registeredPrefixBuilder.ToString();
+                registeredPrefix = CreateRegisteredPrefix(uriPrefix, j, i);
                 if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(this, $"mapped uriPrefix: {uriPrefix} to registeredPrefix: {registeredPrefix}");
                 if (_state == State.Started)
                 {
@@ -179,6 +166,50 @@ internal void AddPrefix(string uriPrefix)
                 if (NetEventSource.Log.IsEnabled()) NetEventSource.Error(this, exception);
                 throw;
             }
+
+            static string CreateRegisteredPrefix(string uriPrefix, int j, int i)
+            {
+                int length = uriPrefix.Length;
+                if (uriPrefix[j] != ':')
+                {
+                    length += i == 7 ? ":80".Length : ":443".Length;
+                }
+
+                return string.Create(length, (uriPrefix, j, i), static (destination, state) =>
+                {
+                    if (state.uriPrefix[state.j] == ':')
+                    {
+                        state.uriPrefix.CopyTo(destination);
+                    }
+                    else
+                    {
+                        int indexOfNextCopy = state.j;
+                        state.uriPrefix.AsSpan(0, indexOfNextCopy).CopyTo(destination);
+
+                        if (state.i == 7)
+                        {
+                            ":80".CopyTo(destination.Slice(indexOfNextCopy));
+                            indexOfNextCopy += 3;
+                        }
+                        else
+                        {
+                            ":443".CopyTo(destination.Slice(indexOfNextCopy));
+                            indexOfNextCopy += 4;
+                        }
+
+                        state.uriPrefix.AsSpan(state.j).CopyTo(destination.Slice(indexOfNextCopy));
+                    }
+
+                    int toLowerLength = destination.IndexOf(':');
+                    if (toLowerLength < 0)
+                    {
+                        toLowerLength = destination.Length;
+                    }
+
+                    OperationStatus operationStatus = Ascii.ToLowerInPlace(destination.Slice(0, toLowerLength), out _);
+                    Debug.Assert(operationStatus == OperationStatus.Done);
+                });
+            }
         }
 
         internal bool ContainsPrefix(string uriPrefix) => _uriPrefixes.Contains(uriPrefix);

diff --git a/src/libraries/System.Net.Mail/src/System/Net/Mail/DomainLiteralReader.cs b/src/libraries/System.Net.Mail/src/System/Net/Mail/DomainLiteralReader.cs
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
 using System.Diagnostics;
 using System.Net.Mime;
 
@@ -70,7 +71,7 @@ internal static bool TryReadReverse(string data, int index, out int outIndex, bo
                     return true;
                 }
                 // Check for invalid characters
-                else if (data[index] > MailBnfHelper.Ascii7bitMaxValue || !MailBnfHelper.Dtext[data[index]])
+                else if (!Ascii.IsValid(data[index]) || !MailBnfHelper.Dtext[data[index]])
                 {
                     if (throwExceptionIfFail)
                     {

diff --git a/src/libraries/System.Net.Mail/src/System/Net/Mail/DotAtomReader.cs b/src/libraries/System.Net.Mail/src/System/Net/Mail/DotAtomReader.cs
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
 using System.Diagnostics;
 using System.Net.Mime;
 
@@ -43,7 +44,7 @@ internal static bool TryReadReverse(string data, int index, out int outIndex, bo
             // Scan for the first invalid chars (including whitespace)
             for (; 0 <= index; index--)
             {
-                if (data[index] <= MailBnfHelper.Ascii7bitMaxValue // Any Unicode allowed
+                if (Ascii.IsValid(data[index]) // Any ASCII allowed
                  && (data[index] != MailBnfHelper.Dot && !MailBnfHelper.Atext[data[index]])) // Invalid char
                 {
                     break;

diff --git a/src/libraries/System.Net.Mail/src/System/Net/Mail/MailBnfHelper.cs b/src/libraries/System.Net.Mail/src/System/Net/Mail/MailBnfHelper.cs
@@ -26,7 +26,6 @@ internal static class MailBnfHelper
         // characters allowed inside of comments
         internal static readonly bool[] Ctext = CreateCharactersAllowedInComments();
 
-        internal const int Ascii7bitMaxValue = 127;
         internal const char Quote = '\"';
         internal const char Space = ' ';
         internal const char Tab = '\t';
@@ -226,11 +225,11 @@ internal static void ValidateHeaderName(string data)
                 {
                     //if data contains Unicode and Unicode is permitted, then
                     //it is valid in a quoted string in a header.
-                    if (data[offset] <= Ascii7bitMaxValue && !Qtext[data[offset]])
+                    if (Ascii.IsValid(data[offset]) && !Qtext[data[offset]])
                         throw new FormatException(SR.Format(SR.MailHeaderFieldInvalidCharacter, data[offset]));
                 }
                 //not permitting Unicode, in which case Unicode is a formatting error
-                else if (data[offset] > Ascii7bitMaxValue || !Qtext[data[offset]])
+                else if (!Ascii.IsValid(data[offset]) || !Qtext[data[offset]])
                 {
                     throw new FormatException(SR.Format(SR.MailHeaderFieldInvalidCharacter, data[offset]));
                 }
@@ -256,7 +255,7 @@ internal static string ReadToken(string data, ref int offset)
             int start = offset;
             for (; offset < data.Length; offset++)
             {
-                if (data[offset] > Ascii7bitMaxValue)
+                if (!Ascii.IsValid(data[offset]))
                 {
                     throw new FormatException(SR.Format(SR.MailHeaderFieldInvalidCharacter, data[offset]));
                 }
@@ -367,7 +366,7 @@ internal static void GetTokenOrQuotedString(string data, StringBuilder builder,
 
         private static bool CheckForUnicode(char ch, bool allowUnicode)
         {
-            if (ch < Ascii7bitMaxValue)
+            if (Ascii.IsValid(ch))
             {
                 return false;
             }

diff --git a/src/libraries/System.Net.Mail/src/System/Net/Mail/QuotedPairReader.cs b/src/libraries/System.Net.Mail/src/System/Net/Mail/QuotedPairReader.cs
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
 using System.Diagnostics;
 using System.Net.Mime;
 
@@ -52,7 +53,7 @@ internal static bool TryCountQuotedChars(string data, int index, bool permitUnic
             }
             else
             {
-                if (!permitUnicodeEscaping && data[index] > MailBnfHelper.Ascii7bitMaxValue)
+                if (!permitUnicodeEscaping && !Ascii.IsValid(data[index]))
                 {
                     if (throwExceptionIfFail)
                     {

diff --git a/src/libraries/System.Net.Mail/src/System/Net/Mail/QuotedStringFormatReader.cs b/src/libraries/System.Net.Mail/src/System/Net/Mail/QuotedStringFormatReader.cs
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
 using System.Diagnostics;
 using System.Net.Mime;
 
@@ -185,7 +186,7 @@ internal static bool TryReadReverseUnQuoted(string data, int index, bool permitU
         // non-whitespace control characters as well as all remaining ASCII chars except backslash and double quote.
         private static bool IsValidQtext(bool allowUnicode, char ch)
         {
-            if (ch > MailBnfHelper.Ascii7bitMaxValue)
+            if (!Ascii.IsValid(ch))
             {
                 return allowUnicode;
             }

diff --git a/src/libraries/System.Net.Mail/src/System/Net/Mail/SmtpClient.cs b/src/libraries/System.Net.Mail/src/System/Net/Mail/SmtpClient.cs
@@ -140,7 +140,7 @@ private void Initialize()
                 for (int i = 0; i < clientDomainRaw.Length; i++)
                 {
                     ch = clientDomainRaw[i];
-                    if ((ushort)ch <= 0x7F)
+                    if (Ascii.IsValid(ch))
                         sb.Append(ch);
                 }
                 if (sb.Length > 0)

diff --git a/src/libraries/System.Net.Mail/src/System/Net/Mail/WhitespaceReader.cs b/src/libraries/System.Net.Mail/src/System/Net/Mail/WhitespaceReader.cs
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
 using System.Diagnostics;
 using System.Net.Mime;
 
@@ -166,7 +167,7 @@ internal static bool TryReadCfwsReverse(string data, int index, out int outIndex
                 }
                 // Check for valid characters within comments.  Allow Unicode, as we won't transmit any comments.
                 else if (commentDepth > 0
-                    && (data[index] > MailBnfHelper.Ascii7bitMaxValue || MailBnfHelper.Ctext[data[index]]))
+                    && (!Ascii.IsValid(data[index]) || MailBnfHelper.Ctext[data[index]]))
                 {
                     index--;
                 }