Skip to content

Commit 26d1aa2

Browse files
SkowronskiAndrewSkowronskiAndrew
committed
Analysis Usability Improvements
Try to reduce the noise when files that are not SerializedFiles are found by Analysis. Ignore files by name and extension that are almost certainly NOT SerializedFiles or Archives, which which are found in build output (Player, AssetBundle or ContentArchives) Add a verbose mode and hide callstack and other info unless verbose is set. When verbose is on then any ignored files are printed.
1 parent 0f33e5c commit 26d1aa2

2 files changed

Lines changed: 144 additions & 74 deletions

File tree

‎Analyzer/AnalyzerTool.cs‎

Lines changed: 138 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,14 @@ namespace UnityDataTools.Analyzer;
88

99
public class AnalyzerTool
1010
{
11-
public int Analyze(string path, string databaseName, string searchPattern, bool skipReferences)
11+
bool m_Verbose = false;
12+
13+
public int Analyze(string path, string databaseName, string searchPattern, bool skipReferences, bool verbose)
1214
{
15+
m_Verbose = verbose;
16+
1317
using SQLiteWriter writer = new (databaseName, skipReferences);
14-
18+
1519
try
1620
{
1721
writer.Begin();
@@ -21,102 +25,166 @@ public int Analyze(string path, string databaseName, string searchPattern, bool
2125
Console.Error.WriteLine($"Error creating database: {e.Message}");
2226
return 1;
2327
}
24-
28+
2529
var timer = new Stopwatch();
2630
timer.Start();
2731

2832
var files = Directory.GetFiles(path, searchPattern, SearchOption.AllDirectories);
2933
int i = 1;
30-
int lastLength = 0;
3134
foreach (var file in files)
3235
{
33-
// Automatically ignore these annoying OS X style files meta files.
34-
if (Path.GetFileName(file) == ".DS_Store")
36+
if (ShouldIgnoreFile(file))
37+
{
38+
var relativePath = Path.GetRelativePath(path, file);
39+
40+
if (m_Verbose)
41+
{
42+
Console.WriteLine();
43+
Console.WriteLine($"Ignoring {relativePath}");
44+
}
45+
++i;
3546
continue;
47+
}
48+
49+
ProcessFile(file, path, writer, i, files.Length);
50+
++i;
51+
}
52+
53+
Console.WriteLine();
54+
Console.WriteLine("Finalizing database...");
55+
56+
writer.End();
57+
58+
timer.Stop();
59+
Console.WriteLine();
60+
Console.WriteLine($"Total time: {(timer.Elapsed.TotalMilliseconds / 1000.0):F3} s");
61+
62+
return 0;
63+
}
64+
65+
bool ShouldIgnoreFile(string file)
66+
{
67+
// Unfortunately there is no standard extension for AssetBundles, and SerializedFiles often have no extension at all.
68+
// There is also no distinctive signature at the start of a SerializedFile to immediately recognize it
69+
// (Unity Archives do have this).
70+
// So to reduce noise in UnityDataTool output we filter out files that we have a high confidence are NOT SerializedFiles or Unity Archives.
71+
72+
string fileName = Path.GetFileName(file);
73+
string extension = Path.GetExtension(file);
74+
75+
if ((fileName == ".DS_Store") || // Automatically ignore these annoying OS X style files meta files.
76+
(fileName == "archive_dependencies.bin") ||
77+
(fileName == "scene_info.bin") ||
78+
(fileName == "app.info") ||
79+
(extension == ".txt") ||
80+
(extension == ".resS") ||
81+
(extension == ".resource") ||
82+
(extension == ".json") ||
83+
(extension == ".dll") ||
84+
(extension == ".pdb") ||
85+
(extension == ".manifest") ||
86+
(extension == ".entities") ||
87+
(extension == ".entityheader"))
88+
return true;
89+
return false;
90+
}
91+
92+
void ProcessFile(string file, string rootDirectory, SQLiteWriter writer, int fileIndex, int cntFiles)
93+
{
94+
try
95+
{
96+
UnityArchive archive = null;
3697

3798
try
3899
{
39-
UnityArchive archive = null;
100+
archive = UnityFileSystem.MountArchive(file, "archive:" + Path.DirectorySeparatorChar);
101+
}
102+
catch (NotSupportedException)
103+
{
104+
// It wasn't an AssetBundle, try to open the file as a SerializedFile.
105+
106+
var relativePath = Path.GetRelativePath(rootDirectory, file);
107+
writer.WriteSerializedFile(relativePath, file, Path.GetDirectoryName(file));
108+
109+
ReportProgress(relativePath, fileIndex, cntFiles);
110+
}
40111

112+
if (archive != null)
113+
{
41114
try
42115
{
43-
archive = UnityFileSystem.MountArchive(file, "archive:" + Path.DirectorySeparatorChar);
44-
}
45-
catch (NotSupportedException)
46-
{
47-
// It wasn't an AssetBundle, try to open the file as a SerializedFile.
48-
49-
var relativePath = Path.GetRelativePath(path, file);
116+
var assetBundleName = Path.GetRelativePath(rootDirectory, file);
50117

51-
writer.WriteSerializedFile(relativePath, file, Path.GetDirectoryName(file));
52-
53-
var message = $"Processing {i * 100 / files.Length}% ({i}/{files.Length}) {file}";
54-
Console.Write($"\rProcessing {i * 100 / files.Length}% ({i}/{files.Length}) {file}");
55-
lastLength = message.Length;
56-
}
118+
writer.BeginAssetBundle(assetBundleName, new FileInfo(file).Length);
119+
ReportProgress(assetBundleName, fileIndex, cntFiles);
57120

58-
if (archive != null)
59-
{
60-
try
121+
foreach (var node in archive.Nodes)
61122
{
62-
var assetBundleName = Path.GetRelativePath(path, file);
63-
64-
writer.BeginAssetBundle(assetBundleName, new FileInfo(file).Length);
65-
66-
var message = $"Processing {i * 100 / files.Length}% ({i}/{files.Length}) {assetBundleName}";
67-
Console.Write($"\r{message}{new string(' ', Math.Max(0, lastLength - message.Length))}");
68-
lastLength = message.Length;
69-
70-
foreach (var node in archive.Nodes)
123+
if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile))
71124
{
72-
if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile))
125+
try
126+
{
127+
writer.WriteSerializedFile(node.Path, "archive:/" + node.Path, Path.GetDirectoryName(file));
128+
}
129+
catch (Exception e)
73130
{
74-
try
75-
{
76-
writer.WriteSerializedFile(node.Path, "archive:/" + node.Path, Path.GetDirectoryName(file));
77-
}
78-
catch (Exception e)
79-
{
80-
Console.Error.WriteLine($"\rError processing {node.Path} in archive {file}{new string(' ', Math.Max(0, lastLength - message.Length))}");
81-
Console.Error.WriteLine(e);
82-
Console.WriteLine();
83-
}
131+
EraseProgressLine();
132+
Console.Error.WriteLine($"Error processing {node.Path} in archive {file}");
133+
Console.Error.WriteLine(e);
134+
Console.WriteLine();
84135
}
85136
}
86137
}
87-
finally
88-
{
89-
Console.Write($"\r{new string(' ', lastLength)}");
90-
writer.EndAssetBundle();
91-
archive.Dispose();
92-
}
138+
}
139+
finally
140+
{
141+
writer.EndAssetBundle();
142+
archive.Dispose();
93143
}
94144
}
95-
catch(NotSupportedException) {
96-
Console.Error.WriteLine();
97-
//Console.Error.WriteLine($"File not supported: {file}"); // This is commented out because another codepath will output "failed to load"
98-
}
99-
catch (Exception e)
100-
{
101-
Console.Error.WriteLine();
102-
Console.Error.WriteLine($"Error processing file: {file}");
103-
Console.Write($"{e.GetType()}: ");
104-
Console.WriteLine(e.Message);
145+
EraseProgressLine();
146+
}
147+
catch (NotSupportedException)
148+
{
149+
EraseProgressLine();
150+
Console.Error.WriteLine();
151+
//Console.Error.WriteLine($"File not supported: {file}"); // This is commented out because another codepath will output "failed to load"
152+
}
153+
catch (Exception e)
154+
{
155+
EraseProgressLine();
156+
Console.Error.WriteLine();
157+
Console.Error.WriteLine($"Error processing file: {file}");
158+
Console.WriteLine($"{e.GetType()}: {e.Message}");
159+
if (m_Verbose)
105160
Console.WriteLine(e.StackTrace);
106-
}
107-
108-
++i;
109161
}
162+
}
110163

111-
Console.WriteLine();
112-
Console.WriteLine("Finalizing database...");
164+
int m_LastProgressMessageLength = 0;
113165

114-
writer.End();
166+
void ReportProgress(string relativePath, int fileIndex, int cntFiles)
167+
{
168+
var message = $"Processing {fileIndex * 100 / cntFiles}% ({fileIndex}/{cntFiles}) {relativePath}";
169+
if (!m_Verbose)
170+
{
171+
EraseProgressLine();
172+
Console.Write($"\r{message}");
173+
}
174+
else
175+
{
176+
Console.WriteLine();
177+
Console.WriteLine(message);
178+
}
115179

116-
timer.Stop();
117-
Console.WriteLine();
118-
Console.WriteLine($"Total time: {(timer.Elapsed.TotalMilliseconds / 1000.0):F3} s");
180+
m_LastProgressMessageLength = message.Length;
181+
}
119182

120-
return 0;
183+
void EraseProgressLine()
184+
{
185+
if (!m_Verbose)
186+
Console.Write($"\r{new string(' ', m_LastProgressMessageLength)}");
187+
else
188+
Console.WriteLine();
121189
}
122-
}
190+
}

‎UnityDataTool/Program.cs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public static async Task<int> Main(string[] args)
2323
var sOpt = new Option<bool>(aliases: new[] { "--skip-references", "-s" }, description: "Skip CRC and do not extract references");
2424
var rOpt = new Option<bool>(aliases: new[] { "--extract-references", "-r" }) { IsHidden = true };
2525
var pOpt = new Option<string>(aliases: new[] { "--search-pattern", "-p" }, description: "File search pattern", getDefaultValue: () => "*");
26+
var vOpt = new Option<bool>(aliases: new[] { "--verbose", "-v" }, description: "Verbose output");
2627

2728
var analyzeCommand = new Command("analyze", "Analyze AssetBundles or SerializedFiles.")
2829
{
@@ -31,12 +32,13 @@ public static async Task<int> Main(string[] args)
3132
sOpt,
3233
rOpt,
3334
pOpt,
35+
vOpt
3436
};
3537

3638
analyzeCommand.AddAlias("analyse");
3739
analyzeCommand.SetHandler(
38-
(DirectoryInfo di, string o, bool s, bool r, string p) => Task.FromResult(HandleAnalyze(di, o, s, r, p)),
39-
pathArg, oOpt, sOpt, rOpt, pOpt);
40+
(DirectoryInfo di, string o, bool s, bool r, string p, bool v) => Task.FromResult(HandleAnalyze(di, o, s, r, p, v)),
41+
pathArg, oOpt, sOpt, rOpt, pOpt, vOpt);
4042

4143
rootCommand.AddCommand(analyzeCommand);
4244
}
@@ -130,7 +132,7 @@ enum DumpFormat
130132
Text,
131133
}
132134

133-
static int HandleAnalyze(DirectoryInfo path, string outputFile, bool skipReferences, bool extractReferences, string searchPattern)
135+
static int HandleAnalyze(DirectoryInfo path, string outputFile, bool skipReferences, bool extractReferences, string searchPattern, bool verbose)
134136
{
135137
var analyzer = new AnalyzerTool();
136138

@@ -139,7 +141,7 @@ static int HandleAnalyze(DirectoryInfo path, string outputFile, bool skipReferen
139141
Console.WriteLine("WARNING: --extract-references, -r option is deprecated (references are now extracted by default)");
140142
}
141143

142-
return analyzer.Analyze(path.FullName, outputFile, searchPattern, skipReferences);
144+
return analyzer.Analyze(path.FullName, outputFile, searchPattern, skipReferences, verbose);
143145
}
144146

145147
static int HandleFindReferences(FileInfo databasePath, string outputFile, long? objectId, string objectName, string objectType, bool findAll)

0 commit comments

Comments
 (0)