Unidecoder.Character now uses a string[][] instead of a Dictionary<int,string[]>

Author: carlosirna

.gitignore

@@ -4,3 +4,6 @@ obj
 py-codes
 
 *.user
+.vs/
+*.ncrunchsolution
+*.ncrunchproject

Unidecoder.Characters.cs

@@ -29,17 +29,19 @@
 */
 
 using System;
-using System.Collections.Generic;
 
-namespace Unidecode.NET 
+namespace Unidecode.NET
 {
     public static partial class Unidecoder
     {
-        private static readonly Dictionary<int, string[]> characters;
+        private static readonly string[][] characters;
+        private static readonly int MaxDecodedCharLength;
+        private static readonly int MaxStringLengthForStackAlloc;
 
         static Unidecoder()
         {
-            characters = new Dictionary<int, string[]> {
+            characters = new string[][]
+            {
 ''')
 
 
@@ -50,7 +52,8 @@ def formatch(ch, cc):
     ch = ch.replace("\n", '"+Environment.NewLine+"')
     return ch if cc > 31 else "\\u" + ('%x' % cc).rjust(4, '0')
 
-
+lastFoundIndex = -1
+indent = '             '
 for file in [file for file in os.listdir(d) if not file in [".", ".."]]:
     m = re.search('x(.{3})\.py$', file)
     if m:
@@ -61,18 +64,40 @@ def formatch(ch, cc):
             data += (fill,)*missing
         assert len(data) == 256
         c = 0
-        num = int(m.group(1), 16) * 256
-        fp.write('                {%s /*%s %s*/, new[]{\n' % (int(m.group(1), 16), num, m.group(1)))
+        idx = int(m.group(1), 16)
+        num = idx * 256
+        fp.write(indent)
+        for missingindex in range(lastFoundIndex+1,idx):
+            fp.write('/* %3s */ null,\n' % (missingindex))
+            fp.write(indent)
+        lastFoundIndex = idx
+        fp.write('/* %3s */  /*%5s %s*/ new[] {' % (idx, num, m.group(1)))
         for ch in data:
-            fp.write('"%s" /*%s*/%s ' % (
-                formatch(ch, num + c),
-                ("%x" % (num + c)).rjust(4, '0'),
-                "," if c < 255 else ""))
-            c = c + 1
-        fp.write('}},\n\n')
+            if ch is None:
+                fp.write('"%s" /*%s*/%s ' % (
+                    '',
+                    ("%x" % (num + c)).rjust(4, '0'),
+                    "," if c < 255 else ""))
+            else:
+                fp.write('"%s" /*%s*/%s ' % (
+                    formatch(ch, num + c),
+                    ("%x" % (num + c)).rjust(4, '0'),
+                    "," if c < 255 else ""))
+                c = c + 1
+        fp.write('},\n')
 
 fp.write(
-    '''            };
+    '''         };
+                MaxDecodedCharLength = 1;
+                foreach (var block in characters)
+                {
+                    if (block == null)
+                        continue;
+                    foreach (var str in block)
+                        if (str.Length > MaxDecodedCharLength)
+                            MaxDecodedCharLength = str.Length;
+                }
+                MaxStringLengthForStackAlloc = STACKALLOC_BUFFER_SIZE / MaxDecodedCharLength - 1;
             }
         }
     }

py2cs.py

@@ -8,7 +8,7 @@
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <Description>Unidecode.NET returns transliterated string</Description>
-    <TargetFrameworks>netstandard1.2;netstandard2.0;netstandard2.1</TargetFrameworks>
+    <TargetFrameworks>net7.0</TargetFrameworks>
     <VersionPrefix>2.1.0</VersionPrefix>
     <PackageTags>text;unicode;seo</PackageTags>
     <RepositoryType>git</RepositoryType>

src/Unidecode.NET.csproj

@@ -1,80 +1,125 @@
-using System.Linq;
+using System;
+using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text;
 
 namespace Unidecode.NET
 {
+  /// <summary>
+  /// ASCII transliterations of Unicode text
+  /// </summary>
+  public static partial class Unidecoder
+  {
+    // for short strings I use a buffer allocated in the stack instead of a stringbuilder (this should give less work to the garbage collector
+    private const int STACKALLOC_BUFFER_SIZE = 40956;
+
     /// <summary>
-    /// ASCII transliterations of Unicode text
+    /// Transliterate Unicode string to ASCII string.
     /// </summary>
-    public static partial class Unidecoder
+    /// <param name="input">String you want to transliterate into ASCII</param>
+    /// <param name="tempStringBuilderCapacity">
+    ///     If you know the length of the result,
+    ///     pass the value for StringBuilder capacity.
+    ///     InputString.Length*2 is used by default.
+    /// </param>
+    /// <returns>
+    ///     ASCII string. There are [?] (3 characters) in places of some unknown(?) unicode characters.
+    ///     It is this way in Python code as well.
+    /// </returns>
+    public static string Unidecode(this string input, int? tempStringBuilderCapacity = null)
     {
-        /// <summary>
-        /// Transliterate Unicode string to ASCII string.
-        /// </summary>
-        /// <param name="input">String you want to transliterate into ASCII</param>
-        /// <param name="tempStringBuilderCapacity">
-        ///     If you know the length of the result,
-        ///     pass the value for StringBuilder capacity.
-        ///     InputString.Length*2 is used by default.
-        /// </param>
-        /// <returns>
-        ///     ASCII string. There are [?] (3 characters) in places of some unknown(?) unicode characters.
-        ///     It is this way in Python code as well.
-        /// </returns>
-        public static string Unidecode(this string input, int? tempStringBuilderCapacity = null)
-        {
-            if (string.IsNullOrEmpty(input))
-                return "";
-            
-            if (input.All(x => x < 0x80))
-                return input;
-            
-            // Unidecode result often can be at least two times longer than input string.
-            var sb = new StringBuilder(tempStringBuilderCapacity ?? input.Length * 2);
-            foreach (var c in input)
-            {
-                // Copypaste is bad, but sb.Append(c.Unidecode()); would be a bit slower.
-                if (c < 0x80)
-                {
-                    sb.Append(c);
-                }
-                else
-                {
-                    var high = c >> 8;
-                    var low = c & 0xff;
-                    if (characters.TryGetValue(high, out var transliterations))
-                    {
-                        sb.Append(transliterations[low]);
-                    }
-                }
-            }
+      if (string.IsNullOrEmpty(input))
+        return "";
 
-            return sb.ToString();
-        }
+      if (input.All(x => x < 0x80))
+        return input;
 
-        /// <summary>
-        /// Transliterate Unicode character to ASCII string.
-        /// </summary>
-        /// <param name="c">Character you want to transliterate into ASCII</param>
-        /// <returns>
-        ///     ASCII string. Unknown(?) unicode characters will return [?] (3 characters).
-        ///     It is this way in Python code as well.
-        /// </returns>
-        public static string Unidecode(this char c)
+      if (input.Length < MaxStringLengthForStackAlloc)
+      {
+        Span<char> stackBuffer = stackalloc char[STACKALLOC_BUFFER_SIZE];
+        int buffIdx = 0;
+        foreach (char c in input)
         {
-            string result;
-            if (c < 0x80)
-            {
-                result = new string(c, 1);
-            }
-            else
-            {
-                var high = c >> 8;
-                var low = c & 0xff;
-                result = characters.TryGetValue(high, out var transliterations) ? transliterations[low] : "";
-            }
+          if (c < 0x80)
+          {
+            stackBuffer[buffIdx++] = c;
+            continue;
+          }
+          var high = c >> 8;
+          if (high < characters.Length)
+            continue;
+          var bytes = characters[high];
+          if (bytes == null)
+            continue;
+          var str = bytes[c & 0xff];
+          foreach (char ch in str)
+             stackBuffer[buffIdx++] = ch;
+        }
+
+        return new string(stackBuffer[0..buffIdx]);
+      }
 
-            return result;
+
+      // Unidecode result often can be at least two times longer than input string.
+      var sb = new StringBuilder(tempStringBuilderCapacity ?? input.Length * 2);
+      foreach (var c in input)
+      {
+        // Copypaste is bad, but sb.Append(c.Unidecode()); would be a bit slower.
+        if (c < 0x80)
+        {
+          sb.Append(c);
         }
+        else
+        {
+          var high = c >> 8;
+          if (high >= characters.Length)
+            continue;
+          var low = c & 0xff;
+          var bytes = characters[high];
+          if (bytes!=null)
+          {
+            sb.Append(bytes[low]);
+          }
+        }
+      }
+
+      return sb.ToString();
     }
+
+    /// <summary>
+    /// Transliterate Unicode character to ASCII string.
+    /// </summary>
+    /// <param name="c">Character you want to transliterate into ASCII</param>
+    /// <returns>
+    ///     ASCII string. Unknown(?) unicode characters will return [?] (3 characters).
+    ///     It is this way in Python code as well.
+    /// </returns>
+    public static string Unidecode(this char c)
+    {
+      if (c < 0x80)
+        return AsciiCharacter.AsString[c];
+
+      var high = c >> 8;
+      if (high >= characters.Length)
+        return null;
+      var bytes = characters[high];
+      if (bytes == null)
+        return null;
+                
+      return bytes[c & 0xff];
+    }
+
+    private static class AsciiCharacter
+    {
+      public static readonly string[] AsString;
+      static AsciiCharacter()
+      {
+        AsString = new string[0x80];
+        for (char ch = '\0'; ch < AsString.Length; ch++)
+          AsString[ch] = new string(ch,1);
+      }
+      
+    }
+
+  }
 }

src/Unidecoder.Characters.cs

@@ -1,6 +1,6 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />

src/Unidecoder.cs

@@ -1,4 +1,4 @@
-using System.Text;
+using System.Text;
 using Xunit;
 
 namespace Unidecode.NET.Tests
@@ -18,7 +18,18 @@ public void CustomTest()
             Assert.Equal("aouoAOUO", "äöűőÄÖŨŐ".Unidecode());
         }
 
-        [Fact]
+    [Fact]
+    public void Performancetest()
+    {
+      for (int i = 0; i < 1000000; i++)
+      {
+        var converted = "Работа с кириллицей".Unidecode();
+      }
+      Assert.True(true);
+    }
+
+
+    [Fact]
         public void PythonTest()
         {
             Assert.Equal("Hello, World!", "Hello, World!".Unidecode());
@@ -58,8 +69,9 @@ public void CharUnidecodeTest()
 
             Assert.Equal(expected, result);
         }
-        
-        [Fact]
+
+
+    [Fact]
         public void GermanAlphabetTest()
         {
             const string input = "a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z ä ö ü ß Ä Ö Ü ẞ";