Skip to content

Commit 71aa39d

Browse files
francis-pagefrancis-page
committed
Fixed wrong SQLite transaction scope that was causing performance issues and some other optimizations
1 parent 1dcc38e commit 71aa39d

7 files changed

Lines changed: 133 additions & 113 deletions

File tree

‎Analyzer/Analyzer.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
</PropertyGroup>
66

77
<ItemGroup>
8-
<PackageReference Include="System.Data.SQLite" Version="1.0.115" />
8+
<PackageReference Include="System.Data.SQLite" Version="1.0.116" />
99
</ItemGroup>
1010

1111
<ItemGroup>

‎Analyzer/SQLite/SQLiteWriter.cs‎

Lines changed: 71 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.Data;
44
using System.Data.SQLite;
5+
using System.Diagnostics.CodeAnalysis;
56
using UnityDataTools.Analyzer.SerializedObjects;
67
using UnityDataTools.Analyzer.SQLite.Handlers;
78
using UnityDataTools.FileSystem;
@@ -22,6 +23,9 @@ public class SQLiteWriter : IWriter
2223
private IdProvider<string> m_SerializedFileIdProvider = new ();
2324
private ObjectIdProvider m_ObjectIdProvider = new ();
2425

26+
// Used to map PPtr fileId to its corresponding serialized file id in the database.
27+
Dictionary<int, int> m_LocalToDbFileId = new ();
28+
2529
private Dictionary<string, ISQLiteHandler> m_Processors = new ()
2630
{
2731
{ "Mesh", new MeshHandler() },
@@ -146,100 +150,102 @@ public void EndAssetBundle()
146150
m_CurrentAssetBundleId = -1;
147151
}
148152

153+
[SuppressMessage("ReSharper.DPA", "DPA0001: Memory allocation issues")]
149154
public void WriteSerializedFile(string filename, string fullPath)
150155
{
151156
using var sf = UnityFileSystem.OpenSerializedFile(fullPath);
152157
using var reader = new UnityFileReader(fullPath, 64 * 1024 * 1024);
158+
var pptrReader = new PPtrReader(sf, reader, AddReference);
153159

154-
// Used to map PPtr fileId to its corresponding serialized file id in the database.
155-
Dictionary<int, int> localToDbFileId = new ();
156160
int serializedFileId = m_SerializedFileIdProvider.GetId(filename.ToLower());
161+
162+
m_LocalToDbFileId.Clear();
157163

158164
using var transaction = m_Database.BeginTransaction();
159165

160166
try
161167
{
162168
m_AddSerializedFileCommand.Parameters["@id"].Value = serializedFileId;
163-
m_AddSerializedFileCommand.Parameters["@asset_bundle"].Value =
164-
m_CurrentAssetBundleId == -1 ? null : m_CurrentAssetBundleId;
169+
m_AddSerializedFileCommand.Parameters["@asset_bundle"].Value = m_CurrentAssetBundleId == -1 ? null : m_CurrentAssetBundleId;
165170
m_AddSerializedFileCommand.Parameters["@name"].Value = filename;
166171
m_AddSerializedFileCommand.ExecuteNonQuery();
167172

168173
int localId = 0;
169-
localToDbFileId.Add(localId++, serializedFileId);
174+
m_LocalToDbFileId.Add(localId++, serializedFileId);
170175
foreach (var extRef in sf.ExternalReferences)
171176
{
172-
localToDbFileId.Add(localId++,
177+
m_LocalToDbFileId.Add(localId++,
173178
m_SerializedFileIdProvider.GetId(extRef.Path.Substring(extRef.Path.LastIndexOf('/') + 1).ToLower()));
174179
}
175-
180+
181+
foreach (var obj in sf.Objects)
182+
{
183+
var currentObjectId = m_ObjectIdProvider.GetId((serializedFileId, obj.Id));
184+
185+
var root = sf.GetTypeTreeRoot(obj.Id);
186+
var offset = obj.Offset;
187+
188+
if (!m_TypeSet.Contains(obj.TypeId))
189+
{
190+
m_AddTypeCommand.Parameters["@id"].Value = obj.TypeId;
191+
m_AddTypeCommand.Parameters["@name"].Value = root.Type;
192+
m_AddTypeCommand.ExecuteNonQuery();
193+
194+
m_TypeSet.Add(obj.TypeId);
195+
}
196+
197+
var randomAccessReader = new RandomAccessReader(sf, root, reader, offset);
198+
199+
string name = null;
200+
long streamDataSize = 0;
201+
202+
if (m_Processors.TryGetValue(root.Type, out var processor))
203+
{
204+
processor.Process(m_ObjectIdProvider, currentObjectId, m_LocalToDbFileId, randomAccessReader,
205+
out name, out streamDataSize);
206+
}
207+
else if (randomAccessReader.HasChild("m_Name"))
208+
{
209+
name = randomAccessReader["m_Name"].GetValue<string>();
210+
}
211+
212+
if (randomAccessReader.HasChild("m_GameObject"))
213+
{
214+
var pptr = randomAccessReader["m_GameObject"];
215+
var fileId = m_LocalToDbFileId[pptr["m_FileID"].GetValue<int>()];
216+
m_AddObjectCommand.Parameters["@game_object"].Value =
217+
m_ObjectIdProvider.GetId((fileId, pptr["m_PathID"].GetValue<long>()));
218+
}
219+
else
220+
{
221+
m_AddObjectCommand.Parameters["@game_object"].Value = null;
222+
}
223+
224+
m_AddObjectCommand.Parameters["@id"].Value = currentObjectId;
225+
m_AddObjectCommand.Parameters["@object_id"].Value = obj.Id;
226+
m_AddObjectCommand.Parameters["@serialized_file"].Value = serializedFileId;
227+
m_AddObjectCommand.Parameters["@type"].Value = obj.TypeId;
228+
m_AddObjectCommand.Parameters["@name"].Value = name;
229+
m_AddObjectCommand.Parameters["@size"].Value = obj.Size + streamDataSize;
230+
m_AddObjectCommand.ExecuteNonQuery();
231+
232+
if (m_ExtractReferences)
233+
{
234+
pptrReader.Process(currentObjectId, offset, root);
235+
}
236+
}
237+
176238
transaction.Commit();
177239
}
178240
catch (Exception)
179241
{
180242
transaction.Rollback();
181243
}
182-
183-
foreach (var obj in sf.Objects)
184-
{
185-
var currentObjectId = m_ObjectIdProvider.GetId((serializedFileId, obj.Id));
186-
187-
var root = sf.GetTypeTreeRoot(obj.Id);
188-
var offset = obj.Offset;
189-
190-
if (!m_TypeSet.Contains(obj.TypeId))
191-
{
192-
m_AddTypeCommand.Parameters["@id"].Value = obj.TypeId;
193-
m_AddTypeCommand.Parameters["@name"].Value = root.Type;
194-
m_AddTypeCommand.ExecuteNonQuery();
195-
196-
m_TypeSet.Add(obj.TypeId);
197-
}
198-
199-
var randomAccessReader = new RandomAccessReader(sf, root, reader, offset);
200-
201-
string name = null;
202-
long streamDataSize = 0;
203-
204-
if (m_Processors.TryGetValue(root.Type, out var processor))
205-
{
206-
processor.Process(m_ObjectIdProvider, currentObjectId, localToDbFileId, randomAccessReader, out name, out streamDataSize);
207-
}
208-
else if (randomAccessReader.HasChild("m_Name"))
209-
{
210-
name = randomAccessReader["m_Name"].GetValue<string>();
211-
}
212-
213-
if (randomAccessReader.HasChild("m_GameObject"))
214-
{
215-
var pptr = randomAccessReader["m_GameObject"];
216-
var fileId = localToDbFileId[pptr["m_FileID"].GetValue<int>()];
217-
m_AddObjectCommand.Parameters["@game_object"].Value = m_ObjectIdProvider.GetId((fileId, pptr["m_PathID"].GetValue<long>()));
218-
}
219-
else
220-
{
221-
m_AddObjectCommand.Parameters["@game_object"].Value = null;
222-
}
223-
224-
m_AddObjectCommand.Parameters["@id"].Value = currentObjectId;
225-
m_AddObjectCommand.Parameters["@object_id"].Value = obj.Id;
226-
m_AddObjectCommand.Parameters["@serialized_file"].Value = serializedFileId;
227-
m_AddObjectCommand.Parameters["@type"].Value = obj.TypeId;
228-
m_AddObjectCommand.Parameters["@name"].Value = name;
229-
m_AddObjectCommand.Parameters["@size"].Value = obj.Size + streamDataSize;
230-
m_AddObjectCommand.ExecuteNonQuery();
231-
232-
if (m_ExtractReferences)
233-
{
234-
var pptrReader = new PPtrReader(sf, root, reader, offset,
235-
(fileId, pathId, propertyPath) =>
236-
AddReference(currentObjectId, m_ObjectIdProvider.GetId((localToDbFileId[fileId], pathId)), propertyPath));
237-
}
238-
}
239244
}
240245

241-
public void AddReference(long objectId, long referencedObjectId, string propertyPath)
246+
public void AddReference(long objectId, int fileId, long pathId, string propertyPath)
242247
{
248+
var referencedObjectId = m_ObjectIdProvider.GetId((m_LocalToDbFileId[fileId], pathId));
243249
m_AddReferenceCommand.Parameters["@object"].Value = objectId;
244250
m_AddReferenceCommand.Parameters["@referenced_object"].Value = referencedObjectId;
245251
m_AddReferenceCommand.Parameters["@property_path"].Value = propertyPath;

‎TextDumper/TextDumperTool.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,10 @@ void RecursiveDump(TypeTreeNode node, ref long offset, int level)
120120
}
121121
}
122122

123-
if ((node.MetaFlags.HasFlag(TypeTreeMetaFlags.AlignBytes) || node.MetaFlags.HasFlag(TypeTreeMetaFlags.AnyChildUsesAlignBytes)))
123+
if (
124+
((int)node.MetaFlags & (int)TypeTreeMetaFlags.AlignBytes) != 0 ||
125+
((int)node.MetaFlags & (int)TypeTreeMetaFlags.AnyChildUsesAlignBytes) != 0
126+
)
124127
{
125128
offset = (offset + 3) & ~(3);
126129
}
@@ -223,7 +226,7 @@ void DumpManagedReferenceRegistry(TypeTreeNode node, ref long offset, int level)
223226

224227
var refIdsArrayNode = refIdsVectorNode.Children[0];
225228

226-
if (refIdsArrayNode.Children.Count != 2 || !refIdsArrayNode.Flags.HasFlag(TypeTreeFlags.IsArray))
229+
if (refIdsArrayNode.Children.Count != 2 || !refIdsArrayNode.IsArray)
227230
throw new Exception("Invalid ManagedReferenceRegistry RefIds array");
228231

229232
// First child is the array size.

‎UnityDataTool/UnityDataTool.csproj‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
<ItemGroup>
99
<PackageReference Include="System.CommandLine" Version="2.0.0-beta3.22114.1" />
10-
<PackageReference Include="System.Data.SQLite" Version="1.0.115.5" />
1110
</ItemGroup>
1211

1312
<ItemGroup>

‎UnityFileSystem/TypeTreeNode.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ public class TypeTreeNode
3737
public bool IsBasicType => IsLeaf && Size > 0;
3838

3939
// True if the field is an array.
40-
public bool IsArray => Flags.HasFlag(TypeTreeFlags.IsArray);
40+
public bool IsArray => ((int)Flags & (int)TypeTreeFlags.IsArray) != 0;
4141

4242
// True if the field is a ManagedReferenceRegistry
43-
public bool IsManagedReferenceRegistry => Flags.HasFlag(TypeTreeFlags.IsManagedReferenceRegistry);
43+
public bool IsManagedReferenceRegistry => ((int)Flags & (int)TypeTreeFlags.IsManagedReferenceRegistry) != 0;
4444

4545
// C# type corresponding to the node type
4646
public Type CSharpType => m_CSharpType.Value;

0 commit comments

Comments
 (0)