Introduced various performance optimizations

Author: carlosirna

BenchmarkResults.txt

@@ -0,0 +1,6 @@
+|               Method |      Mean |     Error |    StdDev |   Gen0 | Allocated |
+|--------------------- |----------:|----------:|----------:|-------:|----------:|
+|     UnidecodeRussian | 42.312 ns | 0.4374 ns | 0.4092 ns | 0.0038 |      64 B |
+|       UnidecodeAscii | 15.640 ns | 0.0360 ns | 0.0319 ns |      - |         - |
+| UnidecodeRussianChar |  3.132 ns | 0.0133 ns | 0.0124 ns |      - |         - |
+|   UnidecodeAsciiChar |  2.673 ns | 0.0167 ns | 0.0156 ns |      - |         - |

benchmark/Unidecode.NET.Benchmark/Unidecode.NET.Benchmark.csproj

@@ -5,6 +5,7 @@
     <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <ItemGroup>

src/Unidecode.NET.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <AssemblyTitle>Unidecode.NET</AssemblyTitle>
     <AssemblyName>Unidecode.NET</AssemblyName>
@@ -18,5 +18,6 @@
     <IncludeSource>true</IncludeSource>
     <SymbolPackageFormat>snupkg</SymbolPackageFormat>
     <NoWarn>1591</NoWarn>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
   </PropertyGroup>
 </Project>

src/Unidecoder.cs

@@ -3,16 +3,24 @@
 using System.Runtime.CompilerServices;
 using System.Text;
 
+// this IntenralsVisibleTo attribute is here to allow benchmarking and
+// testing of SlowUnidecode, which normally, due to the stackalloc optimization,
+// is called only when Unidecode receives a long string
+[assembly: InternalsVisibleTo("Unidecode.Net.Benchmark")]
+[assembly: InternalsVisibleTo("Unidecode.Net.Tests")]
+
 namespace Unidecode.NET
-{
+{   
   /// <summary>
   /// ASCII transliterations of Unicode text
   /// </summary>
   public static partial class Unidecoder
   {
-    // for short strings I use a buffer allocated in the stack instead of a stringbuilder (this should give less work to the garbage collector
-    private const int STACKALLOC_BUFFER_SIZE = 40956;
+    // for short strings I use a buffer allocated in the stack instead of a stringbuilder.
+    // (this is faster and gives less work to the garbage collector)
+    private const int STACKALLOC_BUFFER_SIZE = 8192;
 
+    [SkipLocalsInit] // this is to avoid the local raw buffer variable stackBuffer do be zeroed for every call: we don't need it and is very cpu intensive (this attribute needs unsafe compliation)
     /// <summary>
     /// Transliterate Unicode string to ASCII string.
     /// </summary>
@@ -31,34 +39,44 @@ public static string Unidecode(this string input, int? tempStringBuilderCapacity
       if (string.IsNullOrEmpty(input))
         return "";
 
-      if (input.All(x => x < 0x80))
-        return input;
+      if (input.Length >= MaxStringLengthForStackAlloc)
+        return SlowUnidecode(input, tempStringBuilderCapacity);
 
-     /* if (input.Length < MaxStringLengthForStackAlloc)
+      bool noConversionNeeded = true;
+      Span<char> stackBuffer = stackalloc char[STACKALLOC_BUFFER_SIZE];
+      int buffIdx = 0;
+      foreach (char c in input)
       {
-        Span<char> stackBuffer = stackalloc char[STACKALLOC_BUFFER_SIZE];
-        int buffIdx = 0;
-        foreach (char c in input)
+        if (c < 0x80)
         {
-          if (c < 0x80)
-          {
-            stackBuffer[buffIdx++] = c;
-            continue;
-          }
-          var high = c >> 8;
-          if (high < characters.Length)
-            continue;
-          var bytes = characters[high];
-          if (bytes == null)
-            continue;
-          var str = bytes[c & 0xff];
-          foreach (char ch in str)
-             stackBuffer[buffIdx++] = ch;
+          stackBuffer[buffIdx++] = c;
+          continue;
         }
+        noConversionNeeded = false;
+        var high = c >> 8;
+        if (high >= characters.Length)
+          continue;
+        var bytes = characters[high];
+        if (bytes == null)
+          continue;
+        var str = bytes[c & 0xff];
 
-        return new string(stackBuffer[0..buffIdx]);
-      }*/
+        foreach (char ch in str)
+          stackBuffer[buffIdx++] = ch;
+      }
+      if (noConversionNeeded)
+        return input;
+      return new string(stackBuffer[0..buffIdx]);
+    }
+
+
+    internal static string SlowUnidecode(this string input, int? tempStringBuilderCapacity = null)
+    {
+      if (string.IsNullOrEmpty(input))
+        return "";
 
+      if (input.All(x => x < 0x80))
+        return input;
 
       // Unidecode result often can be at least two times longer than input string.
       var sb = new StringBuilder(tempStringBuilderCapacity ?? input.Length * 2);
@@ -76,7 +94,7 @@ public static string Unidecode(this string input, int? tempStringBuilderCapacity
             continue;
           var low = c & 0xff;
           var bytes = characters[high];
-          if (bytes!=null)
+          if (bytes != null)
           {
             sb.Append(bytes[low]);
           }
@@ -94,7 +112,7 @@ public static string Unidecode(this string input, int? tempStringBuilderCapacity
     ///     ASCII string. Unknown(?) unicode characters will return [?] (3 characters).
     ///     It is this way in Python code as well.
     /// </returns>
-    public static string Unidecode(this char c)
+    public static string Unidecode(this in char c)
     {
       if (c < 0x80)
         return AsciiCharacter.AsString[c];

test/UnidecoderTest.cs

@@ -1,117 +1,130 @@
+using System;
 using System.Text;
 using Xunit;
 
 namespace Unidecode.NET.Tests
 {
-    public class UnidecoderTest
+  public class UnidecoderTest
+  {
+    [Fact]
+    public void DocTest()
     {
-        [Fact]
-        public void DocTest()
-        {
-            Assert.Equal("Bei Jing ", "\u5317\u4EB0".Unidecode());
-        }
-
-        [Fact]
-        public void CustomTest()
-        {
-            Assert.Equal("Rabota s kirillitsei", "Работа с кириллицей".Unidecode());
-            Assert.Equal("aouoAOUO", "äöűőÄÖŨŐ".Unidecode());
-        }
+      Assert.Equal("Bei Jing ", "\u5317\u4EB0".Unidecode());
+    }
 
     [Fact]
-    public void Performancetest()
+    public void CustomTest()
     {
-      for (int i = 0; i < 1000000; i++)
-      {
-        var converted = "Работа с кириллицей".Unidecode();
-      }
-      Assert.True(true);
+      Assert.Equal("Rabota s kirillitsei", "Работа с кириллицей".Unidecode());
+      Assert.Equal("aouoAOUO", "äöűőÄÖŨŐ".Unidecode());
     }
 
 
+
     [Fact]
-        public void PythonTest()
-        {
-            Assert.Equal("Hello, World!", "Hello, World!".Unidecode());
-            Assert.Equal("'\"\r\n", "'\"\r\n".Unidecode());
-            Assert.Equal("CZSczs", "ČŽŠčžš".Unidecode());
-            Assert.Equal("a", "ア".Unidecode());
-            Assert.Equal("a", "α".Unidecode());
-            Assert.Equal("a", "а".Unidecode());
-            Assert.Equal("chateau", "ch\u00e2teau".Unidecode());
-            Assert.Equal("vinedos", "vi\u00f1edos".Unidecode());
-        }
-        
-        [Fact]
-        public void RussianAlphabetTest()
-        {
-            const string russianAlphabetLowercase = "а б в г д е ё ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я";
-            const string russianAlphabetUppercase = "А Б В Г Д Е Ё Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я";
+    public void PythonTest()
+    {
+      Assert.Equal("Hello, World!", "Hello, World!".Unidecode());
+      Assert.Equal("'\"\r\n", "'\"\r\n".Unidecode());
+      Assert.Equal("CZSczs", "ČŽŠčžš".Unidecode());
+      Assert.Equal("a", "ア".Unidecode());
+      Assert.Equal("a", "α".Unidecode());
+      Assert.Equal("a", "а".Unidecode());
+      Assert.Equal("chateau", "ch\u00e2teau".Unidecode());
+      Assert.Equal("vinedos", "vi\u00f1edos".Unidecode());
+    }
 
-            const string expectedLowercase = "a b v g d e io zh z i i k l m n o p r s t u f kh ts ch sh shch ' y ' e iu ia";
-            const string expectedUppercase = "A B V G D E Io Zh Z I I K L M N O P R S T U F Kh Ts Ch Sh Shch ' Y ' E Iu Ia";
+    [Fact]
+    public void RussianAlphabetTest()
+    {
+      const string russianAlphabetLowercase = "а б в г д е ё ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я";
+      const string russianAlphabetUppercase = "А Б В Г Д Е Ё Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я";
 
-            Assert.Equal(expectedLowercase, russianAlphabetLowercase.Unidecode());
-            Assert.Equal(expectedUppercase, russianAlphabetUppercase.Unidecode());
-        }
+      const string expectedLowercase = "a b v g d e io zh z i i k l m n o p r s t u f kh ts ch sh shch ' y ' e iu ia";
+      const string expectedUppercase = "A B V G D E Io Zh Z I I K L M N O P R S T U F Kh Ts Ch Sh Shch ' Y ' E Iu Ia";
 
-        [Fact]
-        public void CharUnidecodeTest()
-        {
-            const string input = "а б в г д е ё ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я А Б В Г Д Е Ё Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я";
-            const string expected = "a b v g d e io zh z i i k l m n o p r s t u f kh ts ch sh shch ' y ' e iu ia A B V G D E Io Zh Z I I K L M N O P R S T U F Kh Ts Ch Sh Shch ' Y ' E Iu Ia";
+      Assert.Equal(expectedLowercase, russianAlphabetLowercase.Unidecode());
+      Assert.Equal(expectedUppercase, russianAlphabetUppercase.Unidecode());
+    }
+
+    [Fact]
+    public void CharUnidecodeTest()
+    {
+      const string input = "а б в г д е ё ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я А Б В Г Д Е Ё Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я";
+      const string expected = "a b v g d e io zh z i i k l m n o p r s t u f kh ts ch sh shch ' y ' e iu ia A B V G D E Io Zh Z I I K L M N O P R S T U F Kh Ts Ch Sh Shch ' Y ' E Iu Ia";
 
-            var sb = new StringBuilder(expected.Length);
-            foreach (var c in input)
-                sb.Append(c.Unidecode());
-            
-            var result = sb.ToString();
+      var sb = new StringBuilder(expected.Length);
+      foreach (var c in input)
+        sb.Append(c.Unidecode());
 
-            Assert.Equal(expected, result);
-        }
+      var result = sb.ToString();
+
+      Assert.Equal(expected, result);
+    }
 
 
     [Fact]
-        public void GermanAlphabetTest()
-        {
-            const string input = "a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z ä ö ü ß Ä Ö Ü ẞ";
-            const string expected = "a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z a o u ss A O U Ss";
+    public void GermanAlphabetTest()
+    {
+      const string input = "a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z ä ö ü ß Ä Ö Ü ẞ";
+      const string expected = "a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z a o u ss A O U Ss";
 
-            var sb = new StringBuilder(expected.Length);
-            foreach (var c in input)
-                sb.Append(c.Unidecode());
-            
-            var result = sb.ToString();
+      var sb = new StringBuilder(expected.Length);
+      foreach (var c in input)
+        sb.Append(c.Unidecode());
 
-            Assert.Equal(expected, result);
-        }
+      var result = sb.ToString();
+
+      Assert.Equal(expected, result);
+    }
+
+    [Fact]
+    public void UnidecodeOnNullShouldReturnEmptyString()
+    {
+      Assert.Equal("", ((string)null).Unidecode());
+    }
 
-        [Fact]
-        public void UnidecodeOnNullShouldReturnEmptyString()
+    /// <summary>
+    ///   Test that code points with the maximum low byte of 255 do not 
+    ///   cause an IndexOutOfRangeException (see commit: acd8fb4)
+    /// </summary>
+    [Fact]
+    public void MaximumLowByteTest()
+    {
+      byte low = 0xFF;
+      for (var high = (char)0; high <= byte.MaxValue; high++)
+      {
+        var codePoint = (char)((high << 8) | low);
+        try
         {
-            Assert.Equal("", ((string)null).Unidecode());
+          codePoint.Unidecode();
         }
-
-        /// <summary>
-        ///   Test that code points with the maximum low byte of 255 do not 
-        ///   cause an IndexOutOfRangeException (see commit: acd8fb4)
-        /// </summary>
-        [Fact]
-        public void MaximumLowByteTest()
+        catch (System.IndexOutOfRangeException)
         {
-            byte low = 0xFF;
-            for (var high = (char)0; high <= byte.MaxValue; high++)
-            {
-                var codePoint = (char)((high << 8) | low);
-                try
-                {
-                    codePoint.Unidecode();
-                }
-                catch (System.IndexOutOfRangeException)
-                {
-                    Assert.True(false);
-                }
-            }
+          Assert.True(false);
         }
+      }
+    }
+
+    /// <summary>
+    ///   Tests that Unidecode "stackAlloc" optimized implementation falls back to the slowest SlowUnidecode implementation for long strings,
+    ///   instead of raising an error
+    /// </summary>
+    [Fact]
+    public void SlowUnidecodeIsCalledForLongStrings()
+    {
+      var srcBuilder = new StringBuilder();
+      var expectedBuilder = new StringBuilder();
+      for (int i = 0; i < 100; i++)
+      {
+        srcBuilder.Append("а б в г д е ё ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я А Б В Г Д Е Ё Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я");
+        expectedBuilder.Append("a b v g d e io zh z i i k l m n o p r s t u f kh ts ch sh shch ' y ' e iu ia A B V G D E Io Zh Z I I K L M N O P R S T U F Kh Ts Ch Sh Shch ' Y ' E Iu Ia");
+      }
+      var src = srcBuilder.ToString();
+      var expected = expectedBuilder.ToString();
+      var result = src.Unidecode();
+      Assert.Equal(expected, result);
+
+    }
   }
 }