Add separate parsers

This refactors the code to have separate parsers for Asset Bundles
and Addressables Build Layout files. The parser implements a
CanParse and Parse method to handle applicability. Currently
there's no overlap so we won't be double-parsing, but it's
possible in the future.

Author: timt-unity3d

Analyzer/Analyzer.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net9.0</TargetFramework>

Analyzer/AnalyzerTool.cs

@@ -1,11 +1,13 @@
-using Newtonsoft.Json;
+using Analyzer.SQLite.Parsers;
+using Analyzer.SQLite.Writers;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using UnityDataTools.Analyzer.Build;
-using UnityDataTools.Analyzer.SQLite;
+using UnityDataTools.Analyzer.SQLite.Handlers;
 using UnityDataTools.FileSystem;
 
 namespace UnityDataTools.Analyzer;
@@ -14,6 +16,12 @@ public class AnalyzerTool
 {
     bool m_Verbose = false;
 
+    public List<ISQLiteFileParser> parsers = new List<ISQLiteFileParser>()
+    {
+        new AddressablesBuildLayoutParser(),
+        new SerializedFileParser(),
+    };
+
     public int Analyze(
         string path,
         string databaseName,
@@ -24,11 +32,19 @@ public int Analyze(
     {
         m_Verbose = verbose;
 
-        using SQLiteWriter writer = new (databaseName, skipReferences);
+        // TODO: skipReferences needs to be passed into AssetBundleWriter
+        using SQLiteWriter writer = new (databaseName);
 
         try
         {
             writer.Begin();
+            foreach (var parser in parsers)
+            {
+                parser.Verbose = verbose;
+                parser.SkipReferences = skipReferences;
+                parser.Init(writer.Connection);
+                
+            }
         }
         catch (Exception e)
         {
@@ -47,13 +63,31 @@ public int Analyze(
         int i = 1;
         foreach (var file in files)
         {
-            if (Path.GetExtension(file) == ".json" && IsAddressablesBuildReport(file))
+            bool foundParser = false;
+            foreach(var parser in parsers)
             {
-                ProcessAddressablesBuild(file, writer, i, files.Length);
-                ++i;
-                continue;
+                if (parser.CanParse(file))
+                {
+                    foundParser = true;
+                    Console.Error.WriteLine(file);
+                    try
+                    {
+                        parser.Parse(file);
+                        ReportProgress(Path.GetRelativePath(path, file), i, files.Length);
+                    }
+                    catch (Exception e)
+                    {                        
+                        EraseProgressLine();
+                        Console.Error.WriteLine();
+                        Console.Error.WriteLine($"Error processing file: {file}");
+                        Console.WriteLine($"{e.GetType()}: {e.Message}");
+                        if (m_Verbose)
+                            Console.WriteLine(e.StackTrace);
+                    }
+                    ++i;
+                }
             }
-            if (ShouldIgnoreFile(file))
+            if (!foundParser)
             {
                 var relativePath = Path.GetRelativePath(path, file);
 
@@ -63,152 +97,25 @@ public int Analyze(
                     Console.WriteLine($"Ignoring {relativePath}");
                 }
                 ++i;
-                continue;
             }
-
-            ProcessFile(file, path, writer, i, files.Length);
-            ++i;
         }
 
         Console.WriteLine();
         Console.WriteLine("Finalizing database...");
 
         writer.End();
+        foreach (var parser in parsers)
+        {
+            parser.Dispose();
+        }
 
-        timer.Stop();
+            timer.Stop();
         Console.WriteLine();
         Console.WriteLine($"Total time: {(timer.Elapsed.TotalMilliseconds / 1000.0):F3} s");
 
         return 0;
     }
 
-    bool ShouldIgnoreFile(string file)
-    {
-        // Unfortunately there is no standard extension for AssetBundles, and SerializedFiles often have no extension at all.
-        // Also there is also no distinctive signature at the start of a SerializedFile to immediately recognize it based on its first bytes.
-        // This makes it difficult to use the "--search-pattern" argument to only pick those files.
-
-        // Hence to reduce noise in UnityDataTool output we filter out files that we have a high confidence are
-        // NOT SerializedFiles or Unity Archives.
-
-        string fileName = Path.GetFileName(file);
-        string extension = Path.GetExtension(file);
-
-        return IgnoredFileNames.Contains(fileName) || IgnoredExtensions.Contains(extension);
-    }
-
-    // These lists are based on expected output files in Player, AssetBundle, Addressables and ECS builds.
-    // However this is by no means exhaustive.
-    private static readonly HashSet<string> IgnoredFileNames = new()
-    {
-        ".DS_Store", "boot.config", "archive_dependencies.bin", "scene_info.bin", "app.info", "link.xml",
-        "catalog.bin", "catalog.hash"
-    };
-
-    private static readonly HashSet<string> IgnoredExtensions = new()
-    {
-        ".txt", ".resS", ".resource", ".json", ".dll", ".pdb", ".exe", ".manifest", ".entities", ".entityheader"
-    };
-
-    void ProcessFile(string file, string rootDirectory, SQLiteWriter writer, int fileIndex, int cntFiles)
-    {
-        try
-        {
-            UnityArchive archive = null;
-
-            try
-            {
-                archive = UnityFileSystem.MountArchive(file, "archive:" + Path.DirectorySeparatorChar);
-            }
-            catch (NotSupportedException)
-            {
-                // It wasn't an AssetBundle, try to open the file as a SerializedFile.
-
-                var relativePath = Path.GetRelativePath(rootDirectory, file);
-                writer.WriteSerializedFile(relativePath, file, Path.GetDirectoryName(file));
-
-                ReportProgress(relativePath, fileIndex, cntFiles);
-            }
-
-            if (archive != null)
-            {
-                try
-                {
-                    var assetBundleName = Path.GetRelativePath(rootDirectory, file);
-
-                    writer.BeginAssetBundle(assetBundleName, new FileInfo(file).Length);
-                    ReportProgress(assetBundleName, fileIndex, cntFiles);
-
-                    foreach (var node in archive.Nodes)
-                    {
-                        if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile))
-                        {
-                            try
-                            {
-                                writer.WriteSerializedFile(node.Path, "archive:/" + node.Path, Path.GetDirectoryName(file));
-                            }
-                            catch (Exception e)
-                            {
-                                EraseProgressLine();
-                                Console.Error.WriteLine($"Error processing {node.Path} in archive {file}");
-                                Console.Error.WriteLine(e);
-                                Console.WriteLine();
-                            }
-                        }
-                    }
-                }
-                finally
-                {
-                    writer.EndAssetBundle();
-                    archive.Dispose();
-                }
-            }
-            EraseProgressLine();
-        }
-        catch (NotSupportedException)
-        {
-            EraseProgressLine();
-            Console.Error.WriteLine();
-            //A "failed to load" error will already be logged by the UnityFileSystem library
-        }
-        catch (Exception e)
-        {
-            EraseProgressLine();
-            Console.Error.WriteLine();
-            Console.Error.WriteLine($"Error processing file: {file}");
-            Console.WriteLine($"{e.GetType()}: {e.Message}");
-            if (m_Verbose)
-                Console.WriteLine(e.StackTrace);
-        }
-    }
-
-
-
-    void ProcessAddressablesBuild(string file, SQLiteWriter writer, int fileIndex, int cntFiles)
-    {
-        try
-        {
-            Console.Error.WriteLine(file);
-            using (StreamReader reader = File.OpenText(file))
-            {
-                JsonSerializer serializer = new JsonSerializer();
-                BuildLayout buildLayout = (BuildLayout)serializer.Deserialize(reader, typeof(BuildLayout));
-                writer.WriteAddressablesBuild(file, buildLayout);
-                ReportProgress(file, fileIndex, cntFiles);
-            }
-        }
-        catch (Exception e)
-        {
-            EraseProgressLine();
-            Console.Error.WriteLine();
-            Console.Error.WriteLine($"Error processing file: {file}");
-            Console.WriteLine($"{e.GetType()}: {e.Message}");
-            if (m_Verbose)
-                Console.WriteLine(e.StackTrace);
-        }
-
-    }
-
     int m_LastProgressMessageLength = 0;
 
     void ReportProgress(string relativePath, int fileIndex, int cntFiles)
@@ -235,45 +142,4 @@ void EraseProgressLine()
         else
             Console.WriteLine();
     }
-
-    bool IsAddressablesBuildReport(string filename)
-    {
-        // Read the first line of the JSON file and check if it contains BuildResultHash
-        string firstLine = "";
-        try
-        {
-            using (StreamReader reader = new StreamReader(filename))
-            {
-                firstLine = reader.ReadLine();
-                if (firstLine != null)
-                {
-                    // Remove trailing comma if present and add closing brace to make it valid JSON
-                    if (firstLine.TrimEnd().EndsWith(","))
-                    {
-                        firstLine = firstLine.TrimEnd().TrimEnd(',') + "}";
-                    }
-
-                    using (JsonTextReader jsonReader = new JsonTextReader(new StringReader(firstLine)))
-                    {
-                        JsonSerializer serializer = new JsonSerializer();
-                        var jsonObject = serializer.Deserialize<JObject>(jsonReader);
-
-                        // If the file has BuildResultHash, process it as an Addressables build
-                        if (jsonObject != null && jsonObject["BuildResultHash"] != null)
-                        {
-                            return true;
-                        }
-                    }
-                }
-            }
-        }
-        catch (Exception e)
-        {
-            if (m_Verbose)
-            {
-                Console.Error.WriteLine($"Error reading JSON file {filename}: {e.Message}");
-            }
-        }
-        return false;
-    }
 }