Refactored tests, added lots of tests with data generated by different Unity versions

Author: francis-page

.gitignore

@@ -21,6 +21,7 @@
 **/*.gen.csproj
 **/*.gen.sln
 **/.vscode
+**/.DS_Store
 **/bin/
 **/obj/
 **/.idea/

Analyzer/Processors/ShaderProcessor.cs

@@ -17,7 +17,7 @@ public class ShaderProcessor : IProcessor, IDisposable
 
         List<int> m_Keywords = new();
         Dictionary<int, string> m_KeywordNames = new();
-        HashSet<uint> m_UniquePrograms = new();
+        HashSet<(sbyte, uint)> m_UniquePrograms = new();
 
         static Dictionary<string, int> s_Keywords = new();
         static long s_SubProgramId = 0;
@@ -34,6 +34,9 @@ public class ShaderProcessor : IProcessor, IDisposable
 
         public void Init(SQLiteConnection db)
         {
+            s_Keywords.Clear();
+            s_SubProgramId = 0;
+            
             using var command = new SQLiteCommand(db);
 
             command.CommandText = Properties.Resources.Shader;
@@ -159,7 +162,7 @@ public void Process(AnalyzerTool analyzer, long objectId, Dictionary<int, int> l
 
             int decompressedSize = 0;
 
-            if (!reader["decompressedLengths"].TypeTreeNode.Children[1].IsLeaf)
+            if (reader["decompressedLengths"][0].IsArray)
             {
                 // The decompressed lengths are stored per graphics API.
                 foreach (var apiLengths in reader["decompressedLengths"])
@@ -195,9 +198,11 @@ void ProcessProgram(RandomAccessReader subPrograms, int hwTier = -1)
 
             foreach (var subProgram in subPrograms)
             {
+                var api = subProgram["m_GpuProgramType"].GetValue<sbyte>();
+                
                 m_Keywords.Clear();
-
-                m_UniquePrograms.Add(subProgram["m_BlobIndex"].GetValue<uint>());
+                
+                m_UniquePrograms.Add((api, subProgram["m_BlobIndex"].GetValue<uint>()));
 
                 if (subProgram.HasChild("m_KeywordIndices"))
                 {
@@ -233,7 +238,7 @@ void ProcessProgram(RandomAccessReader subPrograms, int hwTier = -1)
                 m_InsertSubProgramCommand.Parameters["@id"].Value = s_SubProgramId;
                 m_InsertSubProgramCommand.Parameters["@sub_program"].Value = progNum++;
                 m_InsertSubProgramCommand.Parameters["@hw_tier"].Value = hwTier != -1 ? hwTier : subProgram["m_ShaderHardwareTier"].GetValue<sbyte>();
-                m_InsertSubProgramCommand.Parameters["@api"].Value = subProgram["m_GpuProgramType"].GetValue<sbyte>();
+                m_InsertSubProgramCommand.Parameters["@api"].Value = api;
                 m_InsertSubProgramCommand.ExecuteNonQuery();
 
                 m_InsertSubProgramKeywordsCommand.Parameters["@subprogram_id"].Value = s_SubProgramId;

Analyzer/Resources/Shader.sql

@@ -73,8 +73,8 @@ SELECT sp.shader AS shader_id, o.name, sp.sub_shader, sp.hw_tier, api.name api,
 FROM shader_subprograms sp
 CROSS JOIN objects o ON o.id = sp.shader
 CROSS JOIN shader_apis api ON api.id = sp.api
-CROSS JOIN shader_subprogram_keywords sk ON sk.subprogram_id = sp.id
-CROSS JOIN shader_keywords k ON sk.keyword_id = k.id
+LEFT JOIN shader_subprogram_keywords sk ON sk.subprogram_id = sp.id
+LEFT JOIN shader_keywords k ON sk.keyword_id = k.id
 GROUP BY sp.id;
 
 CREATE VIEW shader_keyword_ratios AS

TextDumper/TextDumperTool.cs

@@ -126,7 +126,7 @@ void RecursiveDump(TypeTreeNode node, ref long offset, int level)
             }
         }
 
-        void DumpArray(TypeTreeNode node, ref long offset, int level, bool isManagedReferenceRegistry = false)
+        void DumpArray(TypeTreeNode node, ref long offset, int level)
         {
             // First child contains array size.
             var sizeNode = node.Children[0];
@@ -178,16 +178,11 @@ void DumpArray(TypeTreeNode node, ref long offset, int level, bool isManagedRefe
                 }
                 else
                 {
+                    ++level;
+                    
                     for (int i = 0; i < arraySize; ++i)
                     {
-                        if (!isManagedReferenceRegistry)
-                        {
-                            RecursiveDump(dataNode, ref offset, level + 1);
-                        }
-                        else
-                        {
-                            DumpManagedReferenceData(dataNode, ref offset, level + 1);
-                        }
+                        RecursiveDump(dataNode, ref offset, level);
                     }
                 }
             }
@@ -199,56 +194,78 @@ void DumpManagedReferenceRegistry(TypeTreeNode node, ref long offset, int level)
                 throw new Exception("Invalid ManagedReferenceRegistry");
 
             // First child is version number.
+            var version = m_Reader.ReadInt32(offset);
             RecursiveDump(node.Children[0], ref offset, level);
-            
-            var refIdsVectorNode = node.Children[1];
 
-            if (refIdsVectorNode.Children.Count < 1 || refIdsVectorNode.Name != "RefIds")
-                throw new Exception("Invalid ManagedReferenceRegistry RefIds vector");
-            
-            m_StringBuilder.Append(' ', level * 2);
-            m_StringBuilder.Append(refIdsVectorNode.Name);
-            m_StringBuilder.Append(' ');
-            m_StringBuilder.Append(refIdsVectorNode.Type);
-            m_StringBuilder.Append(' ');
-            
-            m_Writer.WriteLine(m_StringBuilder);
-            m_StringBuilder.Clear();
+            TypeTreeNode refTypeNode;
+            TypeTreeNode refObjData;
 
-            var refIdsArrayNode = refIdsVectorNode.Children[0];
-            
-            if (refIdsArrayNode.Children.Count != 2 || !refIdsArrayNode.Flags.HasFlag(TypeTreeFlags.IsArray))
-                throw new Exception("Invalid ManagedReferenceRegistry RefIds array");
-            
-            DumpArray(refIdsArrayNode, ref offset, level + 1, true);
-        }
+            if (version == 1)
+            {
+                // Second child is the ReferencedObject.
+                var refObjNode = node.Children[1];
+                // And its children are the referenced type and data nodes.
+                refTypeNode = refObjNode.Children[0];
+                refObjData = refObjNode.Children[1];
+                
+                int i = 0;
 
-        void DumpManagedReferenceData(TypeTreeNode node, ref long offset, int level)
-        {
-            m_StringBuilder.Append(' ', level * 2);
-            m_StringBuilder.Append(node.Name);
-            m_StringBuilder.Append(' ');
-            m_StringBuilder.Append(node.Type);
-            m_StringBuilder.Append(' ');
-            
-            m_Writer.WriteLine(m_StringBuilder);
-            m_StringBuilder.Clear();
+                while (DumpManagedReferenceData(refTypeNode, refObjData, ref offset, level, i++))
+                {}
+            }
+            else if (version == 2)
+            {
+                // Second child is the RefIds vector.
+                var refIdsVectorNode = node.Children[1];
 
-            if (node.Children.Count < 3)
-                throw new Exception("Invalid ReferencedObject");
-            
-            // First child is rid.
-            RecursiveDump(node.Children[0], ref offset, level + 1);
+                if (refIdsVectorNode.Children.Count < 1 || refIdsVectorNode.Name != "RefIds")
+                    throw new Exception("Invalid ManagedReferenceRegistry RefIds vector");
 
-            // Second child is ReferencedManagedType
-            var refTypeNode = node.Children[1];
-            
+                var refIdsArrayNode = refIdsVectorNode.Children[0];
+
+                if (refIdsArrayNode.Children.Count != 2 || !refIdsArrayNode.Flags.HasFlag(TypeTreeFlags.IsArray))
+                    throw new Exception("Invalid ManagedReferenceRegistry RefIds array");
+
+                // First child is the array size.
+                int arraySize = m_Reader.ReadInt32(offset);
+                offset += 4;
+                
+                // Second child is the ReferencedObject.
+                var refObjNode = refIdsArrayNode.Children[1];
+
+                for (int i = 0; i < arraySize; ++i)
+                {
+                    // First child is the rid.
+                    long rid = m_Reader.ReadInt64(offset);
+                    offset += 8;
+                    
+                    // And the next children are the referenced type and data nodes.
+                    refTypeNode = refObjNode.Children[1];
+                    refObjData = refObjNode.Children[2];
+                    DumpManagedReferenceData(refTypeNode, refObjData, ref offset, level, rid);
+                }
+            }
+            else
+            {
+                throw new Exception("Unsupported ManagedReferenceRegistry version");
+            }
+        }
+
+        bool DumpManagedReferenceData(TypeTreeNode refTypeNode, TypeTreeNode referencedTypeDataNode, ref long offset, int level, long id)
+        {
             if (refTypeNode.Children.Count < 3)
                 throw new Exception("Invalid ReferencedManagedType");
+            
+            m_StringBuilder.Append(' ', level * 2);
+            m_StringBuilder.Append($"rid_");
+            m_StringBuilder.Append(id);
+            m_StringBuilder.Append(" ReferencedObject");
+            m_StringBuilder.AppendLine();
+            ++level;
 
             var refTypeOffset = offset;
             var stringSize = m_Reader.ReadInt32(offset);
-            var clasName = m_Reader.ReadString(offset + 4, stringSize);
+            var className = m_Reader.ReadString(offset + 4, stringSize);
             offset += stringSize + 4;
             offset = (offset + 3) & ~(3);
 
@@ -262,13 +279,12 @@ void DumpManagedReferenceData(TypeTreeNode node, ref long offset, int level)
             offset += stringSize + 4;
             offset = (offset + 3) & ~(3);
 
+            if (className == "Terminus" && namespaceName == "UnityEngine.DMAT" && assemblyName == "FAKE_ASM")
+                return false;
+
             // Not the most efficient way, but it simplifies the code.
-            RecursiveDump(refTypeNode, ref refTypeOffset, level + 1);
+            RecursiveDump(refTypeNode, ref refTypeOffset, level);
 
-            // Third child is the referenced type data.
-            var referencedTypeDataNode = node.Children[2];
-            level += 1;
-            
             m_StringBuilder.Append(' ', level * 2);
             m_StringBuilder.Append(referencedTypeDataNode.Name);
             m_StringBuilder.Append(' ');
@@ -278,13 +294,15 @@ void DumpManagedReferenceData(TypeTreeNode node, ref long offset, int level)
             m_Writer.WriteLine(m_StringBuilder);
             m_StringBuilder.Clear();
 
-            var refTypeRoot = m_SerializedFile.GetRefTypeTypeTreeRoot(clasName, namespaceName, assemblyName);
-            
+            var refTypeRoot = m_SerializedFile.GetRefTypeTypeTreeRoot(className, namespaceName, assemblyName);
+                
             // Dump the ReferencedObject using its own TypeTree, but skip the root.
             foreach (var child in refTypeRoot.Children)
             {
                 RecursiveDump(child, ref offset, level + 1);
             }
+
+            return true;
         }
 
         void OutputSerializedFile(string path)