From a4d8084954f920698e6484a11803c9add0b17286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E5=BA=86=E9=93=96?= Date: Tue, 28 Jul 2026 14:12:17 +0800 Subject: [PATCH] =?UTF-8?q?fix(validation):=20=E8=AE=A9=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E6=89=A7=E8=A1=8C=E5=AE=8C=E6=95=B4=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 统一表类型描述符中的解析、物化、验证、序列化与代码渲染入口 - 复用不可变数据快照，确保类型转换仅执行一次并记录内存估算指标 - 迁移 tree 与 graph 数据规则并补充模式对等测试和文档 --- docs/designs/graph-table-design.md | 8 +- docs/designs/tree-table-design.md | 23 +- docs/guides/table-types.md | 2 + .../CodeGen/CodeTemplateRendererRegistry.cs | 38 -- .../DataTableGenerator.cs | 20 +- .../DataTableProcessor.cs | 105 ++--- .../Diagnostics/DiagnosticsCollector.cs | 5 + .../Schema/TableSchemaService.cs | 10 +- .../Serialization/DataRowBinarySerializer.cs | 300 ------------- .../Serialization/DataTableBinaryWriter.cs | 11 +- .../Serialization/IDataTableBinaryWriter.cs | 4 +- .../SnapshotDataSerializationPlan.cs | 30 ++ .../Serialization/TableDataMaterializer.cs | 414 ++++++++++++++++++ .../Serialization/TableDataSnapshot.cs | 95 ++++ .../TableSchemaParserRegistry.cs | 25 +- .../TableTypeDescriptorRegistry.cs | 124 ++++++ .../Validation/GraphTableDataValidation.cs | 48 ++ .../Validation/TableDataValidation.cs | 87 ++++ .../Validation/TreeTableDataValidation.cs | 112 +++++ .../GenerationTransactionTests.cs | 119 +++++ tests/DataTables.Tests/ParserTests.cs | 15 +- 21 files changed, 1140 insertions(+), 455 deletions(-) delete mode 100644 src/DataTables.GeneratorCore/CodeGen/CodeTemplateRendererRegistry.cs delete mode 100644 src/DataTables.GeneratorCore/Serialization/DataRowBinarySerializer.cs create mode 100644 src/DataTables.GeneratorCore/Serialization/SnapshotDataSerializationPlan.cs create mode 100644 src/DataTables.GeneratorCore/Serialization/TableDataMaterializer.cs create mode 100644 src/DataTables.GeneratorCore/Serialization/TableDataSnapshot.cs create mode 100644 src/DataTables.GeneratorCore/TableTypeDescriptorRegistry.cs create mode 100644 src/DataTables.GeneratorCore/Validation/GraphTableDataValidation.cs create mode 100644 src/DataTables.GeneratorCore/Validation/TableDataValidation.cs create mode 100644 src/DataTables.GeneratorCore/Validation/TreeTableDataValidation.cs diff --git a/docs/designs/graph-table-design.md b/docs/designs/graph-table-design.md index 8a835aa..35d51f0 100644 --- a/docs/designs/graph-table-design.md +++ b/docs/designs/graph-table-design.md @@ -8,7 +8,7 @@ - 支持有向图和无向图的基础建模。 - 在生成期发现重复节点、非法边引用和不符合项目规则的环。 - 生成强类型图查询 API，支持节点集合、入边、出边、关联边、前驱、后继、两点边、路径和遍历。 -- 通过注册 parser、validator、writer 和模板实现，不在 `DataTableProcessor` 主流程中硬编码 graph 分支。 +- 通过统一表类型描述符注册 parser、materializer、validator、serialization plan 和模板，不在 `DataTableProcessor` 主流程中硬编码 graph 分支。 ## 推荐 Excel 格式 @@ -77,6 +77,8 @@ DTLevelGraph.TryGetEdge(context, "E001", out var edge); - 项目声明为无环图时检测到环。 - 业务字段或 JSON payload 不能按声明类型解析。 +普通导出与 `ValidateOnly` 会基于同一份 materialized snapshot 执行这些生成期规则；校验模式不会写出代码、bytes 或 manifest。 + ### 可以警告 - 节点没有任何入边或出边。 @@ -124,8 +126,8 @@ DTLevelGraph.TryGetEdge(context, "E001", out var edge); ## 实现步骤建议 1. 已新增 `GraphTableParser`，基于普通行表解析边列表并注册内建索引。 -2. 已新增生成期行校验，检查 `EdgeId`、`From`、`To` 和可选 `Weight`。 -3. 已复用普通表二进制写出流程，运行时通过内建索引构建邻接查询。 +2. 已新增独立的生成期行验证阶段，检查 `EdgeId`、`From`、`To` 和可选 `Weight`。 +3. 已复用通用 materialized snapshot 和 serialization plan，运行时通过内建索引构建邻接查询。 4. 已新增 graph 代码生成模板，输出节点集合、边、邻接、前驱/后继、两点边、路径和 BFS 遍历 API。 5. 已为 `DTGen=graph` 注册 parser 和模板。 6. 已增加 parser 与模板测试；后续可继续补充节点 Sheet、环检测和路径 API 测试。 diff --git a/docs/designs/tree-table-design.md b/docs/designs/tree-table-design.md index 98fea07..b124c68 100644 --- a/docs/designs/tree-table-design.md +++ b/docs/designs/tree-table-design.md @@ -5,10 +5,10 @@ ## 目标 - 用表格方式表达稳定的父子层级关系。 -- 在生成期发现缺失必需字段、非法字段类型和普通行表可发现的数据错误；孤儿节点、循环引用和非法根节点校验可作为后续增强。 +- 在生成期发现缺失必需字段、非法字段类型、孤儿节点、循环引用和普通行表可发现的数据错误。 - 生成强类型查询 API，支持按节点、父节点、子节点和根节点查询。 -- 支持客户端或服务端通过预热提前加载树表；完整结构校验可作为后续增强。 -- 通过已注册的 parser 和模板实现，不在 `DataTableProcessor` 主流程中硬编码 tree 分支。 +- 支持客户端或服务端通过预热提前加载树表；完整父子引用与循环校验在写出前完成。 +- 通过统一表类型描述符注册 parser、materializer、validator、serialization plan 和模板，不在 `DataTableProcessor` 主流程中硬编码 tree 分支。 ## 推荐 Excel 格式 @@ -22,7 +22,7 @@ 推荐约定： - `Id` 必填、唯一，并且作为树节点主键。 -- `ParentId` 为空表示根节点；当前解析器会按 `ParentId` 建立分组索引，但不在解析阶段检查非空 `ParentId` 是否引用已存在的 `Id`。 +- `ParentId` 为空表示根节点；生成期关系校验会检查非空 `ParentId` 是否引用已存在的 `Id`。 - `Order` 可选；字段类型留空时会默认为 `int`。当前生成的 `GetChildren` 返回分组索引顺序，不额外按 `Order` 排序。 - `Name`、`Type`、`Payload` 为业务字段，可以按项目需要扩展。 - `Comment` 可选，只用于说明，不进入运行时 payload。 @@ -68,16 +68,19 @@ foreach (var node in DTQuestTree.TraverseDepthFirst(context, "Root")) - 缺少 `Id` 或 `ParentId` 字段。 - `Id` 或 `ParentId` 显式声明为非 `string` 类型。 -- `Id` 重复会通过内建唯一索引写出/加载流程报错。 +- `Id` 为空或重复。 +- 非空 `ParentId` 引用不存在。 +- 检测到父子循环引用，并输出完整循环路径。 - `Order` 存在且值不能按 `int` 或显式声明类型解析。 - 业务字段不能按声明类型解析。 ### 后续可增强 -- 校验非空 `ParentId` 是否引用已存在的 `Id`。 -- 校验循环引用和根节点数量。 +- 校验根节点数量。 - 对同一父节点下重复 `Order`、过深节点等输出项目级警告。 +普通导出与 `ValidateOnly` 消费同一份 materialized snapshot，并执行相同的 tree 验证阶段；`ValidateOnly` 不通过模拟文件写出来触发校验，也不会产生任何代码或数据文件。 + ## 诊断信息 后续结构化诊断可进一步包含： @@ -113,9 +116,9 @@ foreach (var node in DTQuestTree.TraverseDepthFirst(context, "Root")) ## 后续实现建议 -1. 新增更完整的 `TreeTableValidator`，实现引用完整性、循环检测和项目级根节点规则。 -2. 如有需要，新增 tree serialization plan 或 writer，写出节点数据和关系索引。 -3. 增加正常树、多根树、孤儿节点、重复节点和循环引用测试。 +1. 增加项目级根节点数量、同级 `Order` 和最大深度规则，并通过审计模式评估兼容性。 +2. 如有需要，新增 tree 专用 serialization plan，写出节点数据和关系索引。 +3. 继续补充正常树、多根树、孤儿节点、重复节点和循环引用的端到端 fixture。 ## 非目标 diff --git a/docs/guides/table-types.md b/docs/guides/table-types.md index 91e9cf4..a3cde8b 100644 --- a/docs/guides/table-types.md +++ b/docs/guides/table-types.md @@ -6,6 +6,8 @@ 当前代码中的默认解析器注册了 `table`、`matrix`、`column`、`kv`、`tree` 和 `graph`；代码模板也为这些类型提供生成器。 +所有已支持类型都通过同一份表类型描述符组合 parser、数据 materializer、分阶段 validator、serialization plan 和代码 renderer。普通导出与 `ValidateOnly` 会读取并转换同一份不可变数据快照；`ValidateOnly` 会执行数据类型以及 tree/graph 结构校验，但不会写出代码、bytes 或增量 manifest。诊断 metrics 会记录 snapshot 的逻辑行数和近似字节量，供大型 matrix 或全量校验评估内存预算。 + 所有类型的生成静态查询都是 context-first。默认表把 `DataTableContext` 放在第一参数；命名表把名称放在第二参数： ```csharp diff --git a/src/DataTables.GeneratorCore/CodeGen/CodeTemplateRendererRegistry.cs b/src/DataTables.GeneratorCore/CodeGen/CodeTemplateRendererRegistry.cs deleted file mode 100644 index 8c0df5c..0000000 --- a/src/DataTables.GeneratorCore/CodeGen/CodeTemplateRendererRegistry.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace DataTables.GeneratorCore; - -internal sealed class CodeTemplateRendererRegistry -{ - private readonly Dictionary m_Renderers; - - public CodeTemplateRendererRegistry(IEnumerable renderers) - { - ArgumentNullException.ThrowIfNull(renderers); - - m_Renderers = renderers.ToDictionary( - renderer => renderer.DataSetType, - renderer => renderer, - StringComparer.OrdinalIgnoreCase); - } - - public static CodeTemplateRendererRegistry CreateDefault() - { - return new CodeTemplateRendererRegistry( - [ - new DataTableCodeTemplateRenderer("table"), - new DataMatrixCodeTemplateRenderer(), - new DataTableCodeTemplateRenderer("column"), - new KvTableCodeTemplateRenderer(), - new TreeTableCodeTemplateRenderer(), - new GraphTableCodeTemplateRenderer(), - ]); - } - - public bool TryGetRenderer(string dataSetType, out ICodeTemplateRenderer renderer) - { - return m_Renderers.TryGetValue(dataSetType, out renderer!); - } -} diff --git a/src/DataTables.GeneratorCore/DataTableGenerator.cs b/src/DataTables.GeneratorCore/DataTableGenerator.cs index 4c773c1..166ac29 100644 --- a/src/DataTables.GeneratorCore/DataTableGenerator.cs +++ b/src/DataTables.GeneratorCore/DataTableGenerator.cs @@ -14,8 +14,6 @@ namespace DataTables.GeneratorCore; public sealed class DataTableGenerator { - private static readonly CodeTemplateRendererRegistry s_CodeTemplateRendererRegistry = CodeTemplateRendererRegistry.CreateDefault(); - private readonly ConcurrentDictionary m_Locks; public DataTableGenerator() @@ -265,7 +263,7 @@ await Parallel.ForEachAsync(filePaths, (pair, _) => foreach (var m in metricsList) { - logger($" - [{m.File}/{m.Sheet}] Ignored={m.IgnoredFieldCount}, TagFiltered={m.TagFilteredFieldCount}, SkippedCols={m.SkippedColumnCount}, MatrixSkipped={m.MatrixDefaultSkippedCount}, ParseMs={m.ParseElapsedMs}, GenMs={m.GenerateElapsedMs}"); + logger($" - [{m.File}/{m.Sheet}] Ignored={m.IgnoredFieldCount}, TagFiltered={m.TagFilteredFieldCount}, SkippedCols={m.SkippedColumnCount}, MatrixSkipped={m.MatrixDefaultSkippedCount}, SnapshotRows={m.SnapshotRowCount}, SnapshotBytes~={m.SnapshotEstimatedBytes}, ParseMs={m.ParseElapsedMs}, GenMs={m.GenerateElapsedMs}"); } } @@ -688,6 +686,14 @@ private async Task GenerateExcel(string filePath, string usingNamespace, string continue; } + // 数据只读取和类型转换一次。ValidateOnly 与实际导出共享同一份规范快照和验证阶段。 + var dataSnapshot = processor.MaterializeData(sheet); + if (!processor.ValidateData(dataSnapshot)) + { + context.Failed = true; + continue; + } + var codeContent = renderCode ? RenderCodeFile(context) : null; var codeContentHash = codeContent == null ? null : ComputeContentHash(codeContent); if (!outputClaims.TryReserve(context, filePath, codeContentHash, out var writeCode, out var conflict)) @@ -705,10 +711,10 @@ private async Task GenerateExcel(string filePath, string usingNamespace, string GenerateCodeFile(context, codeOutputDir, finalCodeOutputDir, codeContent!, forceOverwrite, logger); } - // 生成数据文件。ValidateOnly 模式只解析、校验并渲染代码模板，不写任何产物。 + // 生成数据文件。ValidateOnly 已完成相同的数据快照验证，但不写任何产物。 if (!validateOnly) { - processor.GenerateDataFile(dataOutputDir, finalDataOutputDir, forceOverwrite, sheet, logger); + processor.GenerateDataFile(dataOutputDir, finalDataOutputDir, forceOverwrite, dataSnapshot, logger); } if (context.Failed) @@ -766,12 +772,12 @@ private static bool ValidSheet(ISheet? sheet) private static string RenderCodeFile(GenerationContext context) { - if (!s_CodeTemplateRendererRegistry.TryGetRenderer(context.DataSetType, out var renderer)) + if (!TableTypeDescriptorRegistry.Default.TryGetDescriptor(context.DataSetType, out var descriptor)) { throw new InvalidOperationException($"未注册 DTGen={context.DataSetType} 的代码生成模板。"); } - return renderer.TransformText(context); + return descriptor.CodeRenderer.TransformText(context); } void GenerateCodeFile(GenerationContext context, string outputDir, string comparisonOutputDir, string content, bool forceOverwrite, ILogger logger) diff --git a/src/DataTables.GeneratorCore/DataTableProcessor.cs b/src/DataTables.GeneratorCore/DataTableProcessor.cs index f0ccd77..96d5a78 100644 --- a/src/DataTables.GeneratorCore/DataTableProcessor.cs +++ b/src/DataTables.GeneratorCore/DataTableProcessor.cs @@ -1,6 +1,5 @@ using System; using System.ComponentModel.DataAnnotations; -using System.IO; using NPOI.SS.UserModel; namespace DataTables.GeneratorCore; @@ -62,43 +61,6 @@ public void CreateGenerationContext(ISheet sheet) m_FirstDataRowIndex = new TableSchemaService(m_Context, m_Options, m_Diagnostics).CreateGenerationContext(sheet); } - // 是否有效行 - private bool IgnoreDataRow(IRow? row) - { - if (row == null || row.FirstCellNum < 0) - { - return false; - } - - // 此行是否被注释 - foreach (var field in m_Context.Fields) - { - if (!field.IsComment) - { - continue; - } - - var cellString = GetCellString(row.GetCell(field.Index)); - if (!string.IsNullOrEmpty(cellString) && cellString == "#") - { - return false; - } - } - - // 是否空行 - foreach (var field in m_Context.Fields) - { - if (field.IsIgnore) { continue; } - - if (!string.IsNullOrEmpty(GetCellString(row.GetCell(field.Index)))) - { - return true; - } - } - - return false; - } - private string GetCellString(ICell? cell) { if (cell == null) @@ -199,16 +161,54 @@ private void ValidateFormulaCellString(ICell cell, string value) } - internal void GenerateDataFile(string outputDir, string comparisonOutputDir, bool forceOverwrite, ISheet sheet, ILogger logger) + internal TableDataSnapshot MaterializeData(ISheet sheet) { - new DataTableBinaryWriter(m_Context, WriteDataRows).GenerateDataFile(outputDir, comparisonOutputDir, forceOverwrite, sheet, logger); + var descriptor = GetTableTypeDescriptor(); + var snapshot = descriptor.DataMaterializer.Materialize(new TableDataMaterializationContext( + m_Context, + sheet, + m_FirstDataRowIndex, + m_Diagnostics, + GetCellString)); + var metrics = m_Diagnostics.GetMetrics(m_Context.FileName, m_Context.SheetName); + metrics.SnapshotRowCount = snapshot.Rows.Count; + metrics.SnapshotEstimatedBytes = snapshot.EstimatedSizeInBytes; + return snapshot; } - internal int WriteDataRows(ISheet sheet, BinaryWriter writer) + internal bool ValidateData(TableDataSnapshot snapshot) { - return new DataRowBinarySerializer(m_Context, m_FirstDataRowIndex, m_Diagnostics, GetCellString, IgnoreDataRow).WriteDataRows(sheet, writer); + var descriptor = GetTableTypeDescriptor(); + var validationContext = new TableDataValidationContext(m_Context, snapshot, m_Diagnostics); + var errorCount = m_Diagnostics.ErrorCount; + foreach (var validationPass in descriptor.ValidationPasses) + { + validationPass.Validate(validationContext); + if (m_Diagnostics.ErrorCount > errorCount) + { + return false; + } + } + + return true; + } + + internal void GenerateDataFile(string outputDir, string comparisonOutputDir, bool forceOverwrite, TableDataSnapshot snapshot, ILogger logger) + { + var descriptor = GetTableTypeDescriptor(); + new DataTableBinaryWriter(m_Context, descriptor.SerializationPlan) + .GenerateDataFile(outputDir, comparisonOutputDir, forceOverwrite, snapshot, logger); } + private TableTypeDescriptor GetTableTypeDescriptor() + { + if (!TableTypeDescriptorRegistry.Default.TryGetDescriptor(m_Context.DataSetType, out var descriptor)) + { + throw new InvalidOperationException($"未注册 DTGen={m_Context.DataSetType} 的表类型描述符。"); + } + + return descriptor; + } internal static DataProcessor GetDataProcessorForSerialization(string type) { @@ -243,27 +243,6 @@ public static string GetLanguageValue(XField field, string text) return DataProcessorUtility.GetDataProcessor(field.TypeName).GenerateTypeValue(text); } - private static string GetRowColString(int row, int col) - { - return string.Format("{0}{1}", ConvertToDigit(col), row + 1); - } - - private static string ConvertToDigit(int num) - { - if (num < 0) - { - throw new ArgumentOutOfRangeException(nameof(num)); - } - else if (num < 26) - { - return Convert.ToString((char)('A' + num)); - } - else - { - return ConvertToDigit(num / 26 - 1) + ConvertToDigit(num % 26); - } - } - /// /// 计算7BitEncodedInt编码后的字节数 /// diff --git a/src/DataTables.GeneratorCore/Diagnostics/DiagnosticsCollector.cs b/src/DataTables.GeneratorCore/Diagnostics/DiagnosticsCollector.cs index 109eeeb..1426767 100644 --- a/src/DataTables.GeneratorCore/Diagnostics/DiagnosticsCollector.cs +++ b/src/DataTables.GeneratorCore/Diagnostics/DiagnosticsCollector.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; namespace DataTables.GeneratorCore; @@ -34,6 +35,8 @@ public sealed class DiagnosticsCollector public IReadOnlyList Items => m_Diagnostics; + public int ErrorCount => m_Diagnostics.Count(item => item.Severity == DiagnosticSeverity.Error); + public DiagnosticsMetrics GetMetrics(string file, string sheet) { var key = file + "|" + sheet; @@ -74,6 +77,8 @@ public DiagnosticsMetrics(string file, string sheet) public int TagFilteredFieldCount { get; set; } public int SkippedColumnCount { get; set; } public int MatrixDefaultSkippedCount { get; set; } + public int SnapshotRowCount { get; set; } + public long SnapshotEstimatedBytes { get; set; } public long ParseElapsedMs { get; set; } public long GenerateElapsedMs { get; set; } } diff --git a/src/DataTables.GeneratorCore/Schema/TableSchemaService.cs b/src/DataTables.GeneratorCore/Schema/TableSchemaService.cs index 34c4f1a..b3477dd 100644 --- a/src/DataTables.GeneratorCore/Schema/TableSchemaService.cs +++ b/src/DataTables.GeneratorCore/Schema/TableSchemaService.cs @@ -5,7 +5,7 @@ namespace DataTables.GeneratorCore; public sealed class TableSchemaService : ITableSchemaService { - private static readonly ITableSchemaParserRegistry s_TableSchemaParserRegistry = TableSchemaParserRegistry.CreateDefault(); + private static readonly TableTypeDescriptorRegistry s_TableTypeDescriptorRegistry = TableTypeDescriptorRegistry.Default; private readonly GenerationContext m_Context; private readonly ParseOptions m_Options; @@ -60,10 +60,10 @@ public int CreateGenerationContext(ISheet sheet) } } - if (!s_TableSchemaParserRegistry.TryGetParser(m_Context.DataSetType, out var parser)) + if (!s_TableTypeDescriptorRegistry.TryGetDescriptor(m_Context.DataSetType, out var descriptor)) { - var supportedTypes = string.Join(", ", s_TableSchemaParserRegistry.SupportedDataSetTypes); - var reservedTypes = string.Join(", ", s_TableSchemaParserRegistry.ReservedDataSetTypes); + var supportedTypes = string.Join(", ", s_TableTypeDescriptorRegistry.SupportedDataSetTypes); + var reservedTypes = string.Join(", ", s_TableTypeDescriptorRegistry.ReservedDataSetTypes); m_Diagnostics.Error( m_Context.FileName, m_Context.SheetName, @@ -72,7 +72,7 @@ public int CreateGenerationContext(ISheet sheet) return -1; } - int nextIndex = parser.Parse(new NpoiSheetReader(sheet), m_Context, m_Options, m_Diagnostics); + int nextIndex = descriptor.SchemaParser.Parse(new NpoiSheetReader(sheet), m_Context, m_Options, m_Diagnostics); if (nextIndex == -1) { m_Diagnostics.Warn(m_Context.FileName, m_Context.SheetName, string.Empty, "表头或字段信息不完整，解析终止"); diff --git a/src/DataTables.GeneratorCore/Serialization/DataRowBinarySerializer.cs b/src/DataTables.GeneratorCore/Serialization/DataRowBinarySerializer.cs deleted file mode 100644 index 0822053..0000000 --- a/src/DataTables.GeneratorCore/Serialization/DataRowBinarySerializer.cs +++ /dev/null @@ -1,300 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using NPOI.SS.UserModel; - -namespace DataTables.GeneratorCore; - -internal sealed class DataRowBinarySerializer -{ - private readonly GenerationContext m_Context; - private readonly int m_FirstDataRowIndex; - private readonly DiagnosticsCollector m_Diagnostics; - private readonly Func m_GetCellString; - private readonly Func m_ShouldWriteDataRow; - - public DataRowBinarySerializer( - GenerationContext context, - int firstDataRowIndex, - DiagnosticsCollector diagnostics, - Func getCellString, - Func shouldWriteDataRow) - { - m_Context = context; - m_FirstDataRowIndex = firstDataRowIndex; - m_Diagnostics = diagnostics; - m_GetCellString = getCellString; - m_ShouldWriteDataRow = shouldWriteDataRow; - } - - public int WriteDataRows(ISheet sheet, BinaryWriter writer) - { - int dataRowCount = 0; - - switch (m_Context.DataSetType) - { - case "tree": - ValidateTreeRows(sheet); - break; - case "graph": - ValidateGraphRows(sheet); - break; - case "kv": - return WriteKvBytes(writer); - } - - if (m_Context.DataSetType == "column") - { - int maxLastCellNum = 0; - for (int r = m_FirstDataRowIndex; r <= sheet.LastRowNum; r++) - { - var row = sheet.GetRow(r); - if (row == null) continue; - if (row.LastCellNum > maxLastCellNum) maxLastCellNum = row.LastCellNum; - } - - for (int c = m_Context.ColumnFirstDataColIndex; c < maxLastCellNum; c++) - { - if (m_Context.ColumnCommentRowIndex >= 0) - { - var markRow = sheet.GetRow(m_Context.ColumnCommentRowIndex); - var mk = m_GetCellString(markRow?.GetCell(c)); - if (!string.IsNullOrEmpty(mk) && mk.TrimStart().StartsWith("#", StringComparison.Ordinal)) - { - m_Diagnostics.GetMetrics(m_Context.FileName, m_Context.SheetName).SkippedColumnCount++; - continue; - } - } - - bool hasAnyValue = false; - foreach (var field in m_Context.Fields) - { - if (field.IsIgnore) continue; - var row = sheet.GetRow(field.Index); - var text = m_GetCellString(row?.GetCell(c)); - if (!string.IsNullOrEmpty(text)) - { - hasAnyValue = true; - break; - } - } - if (!hasAnyValue) continue; - - dataRowCount += WriteColumnBytes(writer, sheet, c); - } - } - else - { - for (int i = m_FirstDataRowIndex; i <= sheet.LastRowNum; i++) - { - var row = sheet.GetRow(i); - if (!m_ShouldWriteDataRow(row)) continue; - dataRowCount += WriteRowBytes(writer, row!); - } - } - - return dataRowCount; - } - - private int WriteKvBytes(BinaryWriter writer) - { - foreach (var field in m_Context.Fields.Where(x => !x.IsIgnore)) - { - var processor = GetDataProcessorWithDiagnostics(field); - try - { - processor.WriteToStream(writer, field.Note); - } - catch (Exception e) - { - throw new FormatException($"解析 kv 配置 {field.Name} 的值时出错: {field.Note}", e); - } - } - - return 1; - } - - private int WriteRowBytes(BinaryWriter writer, IRow row) - { - if (m_Context.DataSetType == "matrix") - { - return WriteMatrixRowBytes(writer, row); - } - - foreach (var field in m_Context.Fields) - { - if (field.IsIgnore) continue; - var processor = GetDataProcessorWithDiagnostics(field); - try - { - processor.WriteToStream(writer, m_GetCellString(row.GetCell(field.Index))); - } - catch (Exception e) - { - throw new FormatException($"解析单元格内容时出错: {GetRowColString(row.RowNum, field.Index)}", e); - } - } - - return 1; - } - - private int WriteMatrixRowBytes(BinaryWriter writer, IRow row) - { - var processor1 = DataTableProcessor.GetDataProcessorForSerialization(m_Context.GetField(DataMatrixTemplate.kKey1)!.TypeName); - var processor2 = DataTableProcessor.GetDataProcessorForSerialization(m_Context.GetField(DataMatrixTemplate.kKey2)!.TypeName); - var processor3 = DataTableProcessor.GetDataProcessorForSerialization(m_Context.GetField(DataMatrixTemplate.kValue)!.TypeName); - int dataRowCount = 0; - - foreach (var pair in m_Context.ColumnIndexToKey2) - { - var cellString = m_GetCellString(row.GetCell(pair.Key)); - if (string.IsNullOrEmpty(cellString) || cellString == m_Context.MatrixDefaultValue) - { - if (cellString == m_Context.MatrixDefaultValue) - { - m_Diagnostics.GetMetrics(m_Context.FileName, m_Context.SheetName).MatrixDefaultSkippedCount++; - } - continue; - } - - dataRowCount++; - WriteCell(writer, processor1, m_GetCellString(row.GetCell(row.FirstCellNum)), row.RowNum, row.FirstCellNum); - WriteCell(writer, processor2, pair.Value, 1, pair.Key); - WriteCell(writer, processor3, cellString, row.RowNum, pair.Key); - } - - return dataRowCount; - } - - private int WriteColumnBytes(BinaryWriter writer, ISheet sheet, int columnIndex) - { - foreach (var field in m_Context.Fields) - { - if (field.IsIgnore) continue; - var processor = GetDataProcessorWithDiagnostics(field); - try - { - var row = sheet.GetRow(field.Index); - processor.WriteToStream(writer, m_GetCellString(row?.GetCell(columnIndex))); - } - catch (Exception e) - { - throw new FormatException($"解析单元格内容时出错: {GetRowColString(field.Index, columnIndex)}", e); - } - } - - return 1; - } - - private static void WriteCell(BinaryWriter writer, DataTableProcessor.DataProcessor processor, string value, int row, int column) - { - try - { - processor.WriteToStream(writer, value); - } - catch (Exception e) - { - throw new FormatException($"解析单元格内容时出错: {GetRowColString(row, column)}", e); - } - } - - private DataTableProcessor.DataProcessor GetDataProcessorWithDiagnostics(XField field) - { - try - { - return DataTableProcessor.GetDataProcessorForSerialization(field.TypeName); - } - catch (DataTableProcessor.DataTypeParseException ex) - { - var cell = string.IsNullOrEmpty(field.TypeCell) ? string.Empty : field.TypeCell; - m_Diagnostics.Error(m_Context.FileName, m_Context.SheetName, cell, $"字段 '{field.Name}' 类型 '{field.TypeName}' 解析失败: {ex.Message}"); - throw new FormatException($"类型解析失败: File='{m_Context.FileName}', Sheet='{m_Context.SheetName}', Field='{field.Name}', TypeCell='{cell}', FieldType='{field.TypeName}'. {ex.Message}", ex); - } - } - - private void ValidateTreeRows(ISheet sheet) - { - var idField = m_Context.GetField("Id") ?? throw new InvalidOperationException("DTGen=tree 缺少 Id 字段"); - var parentField = m_Context.GetField("ParentId") ?? throw new InvalidOperationException("DTGen=tree 缺少 ParentId 字段"); - var orderField = m_Context.GetField("Order"); - var nodes = new Dictionary(StringComparer.Ordinal); - - for (int i = m_FirstDataRowIndex; i <= sheet.LastRowNum; i++) - { - var row = sheet.GetRow(i); - if (!m_ShouldWriteDataRow(row)) continue; - var id = m_GetCellString(row!.GetCell(idField.Index)); - var parentId = m_GetCellString(row.GetCell(parentField.Index)); - if (string.IsNullOrWhiteSpace(id)) throw new FormatException($"DTGen=tree Id 为空: row={i + 1}, cell={GetRowColString(i, idField.Index)}"); - if (!nodes.TryAdd(id, (parentId, i))) throw new FormatException($"DTGen=tree Id 重复: nodeId={id}, row={i + 1}, cell={GetRowColString(i, idField.Index)}"); - if (orderField != null) - { - var orderText = m_GetCellString(row.GetCell(orderField.Index)); - if (!string.IsNullOrWhiteSpace(orderText) && !decimal.TryParse(orderText, out _)) throw new FormatException($"DTGen=tree Order 不是合法数字: nodeId={id}, row={i + 1}, cell={GetRowColString(i, orderField.Index)}, value={orderText}"); - } - } - - foreach (var (id, node) in nodes) - { - if (!string.IsNullOrEmpty(node.ParentId) && !nodes.ContainsKey(node.ParentId)) throw new FormatException($"DTGen=tree ParentId 引用不存在: nodeId={id}, parentId={node.ParentId}, row={node.Row + 1}"); - } - - var visited = new HashSet(StringComparer.Ordinal); - var visiting = new HashSet(StringComparer.Ordinal); - var path = new List(); - foreach (var id in nodes.Keys) Visit(id); - - void Visit(string id) - { - if (visited.Contains(id)) return; - if (!visiting.Add(id)) - { - var start = path.IndexOf(id); - var cycle = start >= 0 ? path.Skip(start).Concat(new[] { id }) : new[] { id, id }; - throw new FormatException($"DTGen=tree 检测到循环引用: {string.Join(" -> ", cycle)}"); - } - path.Add(id); - var parentId = nodes[id].ParentId; - if (!string.IsNullOrEmpty(parentId) && nodes.ContainsKey(parentId)) Visit(parentId); - path.RemoveAt(path.Count - 1); - visiting.Remove(id); - visited.Add(id); - } - } - - private void ValidateGraphRows(ISheet sheet) - { - var edgeIdField = m_Context.GetField("EdgeId") ?? throw new InvalidOperationException("DTGen=graph 缺少 EdgeId 字段"); - var fromField = m_Context.GetField("From") ?? throw new InvalidOperationException("DTGen=graph 缺少 From 字段"); - var toField = m_Context.GetField("To") ?? throw new InvalidOperationException("DTGen=graph 缺少 To 字段"); - var weightField = m_Context.GetField("Weight"); - var edgeIds = new HashSet(StringComparer.Ordinal); - - for (int i = m_FirstDataRowIndex; i <= sheet.LastRowNum; i++) - { - var row = sheet.GetRow(i); - if (!m_ShouldWriteDataRow(row)) continue; - var edgeId = m_GetCellString(row!.GetCell(edgeIdField.Index)); - var from = m_GetCellString(row.GetCell(fromField.Index)); - var to = m_GetCellString(row.GetCell(toField.Index)); - if (string.IsNullOrWhiteSpace(edgeId)) throw new FormatException($"DTGen=graph EdgeId 为空: row={i + 1}, cell={GetRowColString(i, edgeIdField.Index)}"); - if (!edgeIds.Add(edgeId)) throw new FormatException($"DTGen=graph EdgeId 重复: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, edgeIdField.Index)}"); - if (string.IsNullOrWhiteSpace(from)) throw new FormatException($"DTGen=graph From 为空: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, fromField.Index)}"); - if (string.IsNullOrWhiteSpace(to)) throw new FormatException($"DTGen=graph To 为空: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, toField.Index)}"); - if (weightField != null) - { - var weightText = m_GetCellString(row.GetCell(weightField.Index)); - if (!string.IsNullOrWhiteSpace(weightText) && !decimal.TryParse(weightText, out _)) throw new FormatException($"DTGen=graph Weight 不是合法数字: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, weightField.Index)}, value={weightText}"); - } - } - } - - private static string GetRowColString(int row, int col) => string.Format("{0}{1}", ConvertToDigit(col), row + 1); - - private static string ConvertToDigit(int num) - { - if (num < 0) throw new ArgumentOutOfRangeException(nameof(num)); - return num < 26 ? Convert.ToString((char)('A' + num)) : ConvertToDigit(num / 26 - 1) + ConvertToDigit(num % 26); - } -} diff --git a/src/DataTables.GeneratorCore/Serialization/DataTableBinaryWriter.cs b/src/DataTables.GeneratorCore/Serialization/DataTableBinaryWriter.cs index 8634250..4ab4860 100644 --- a/src/DataTables.GeneratorCore/Serialization/DataTableBinaryWriter.cs +++ b/src/DataTables.GeneratorCore/Serialization/DataTableBinaryWriter.cs @@ -2,22 +2,21 @@ using System.IO; using System.Text; using DataTables; -using NPOI.SS.UserModel; namespace DataTables.GeneratorCore; internal sealed class DataTableBinaryWriter : IDataTableBinaryWriter { private readonly GenerationContext m_Context; - private readonly Func m_WriteDataRows; + private readonly ITableDataSerializationPlan m_SerializationPlan; - public DataTableBinaryWriter(GenerationContext context, Func writeDataRows) + public DataTableBinaryWriter(GenerationContext context, ITableDataSerializationPlan serializationPlan) { m_Context = context; - m_WriteDataRows = writeDataRows; + m_SerializationPlan = serializationPlan; } - public void GenerateDataFile(string outputDir, string comparisonOutputDir, bool forceOverwrite, ISheet sheet, ILogger logger) + public void GenerateDataFile(string outputDir, string comparisonOutputDir, bool forceOverwrite, TableDataSnapshot snapshot, ILogger logger) { int startTickCount = Environment.TickCount; string outputFileName = Path.Combine(outputDir, m_Context.GetDataOutputFilePath()); @@ -34,7 +33,7 @@ public void GenerateDataFile(string outputDir, string comparisonOutputDir, bool GetGeneratorVersion(), m_Context.DataTableClassFullName); - int dataRowCount = m_WriteDataRows(sheet, binaryWriter); + int dataRowCount = m_SerializationPlan.WriteDataRows(snapshot, binaryWriter); DataTableBinaryFormat.PatchRowCount(binaryWriter, countPosition, dataRowCount); } diff --git a/src/DataTables.GeneratorCore/Serialization/IDataTableBinaryWriter.cs b/src/DataTables.GeneratorCore/Serialization/IDataTableBinaryWriter.cs index 6d620e2..c72a0c0 100644 --- a/src/DataTables.GeneratorCore/Serialization/IDataTableBinaryWriter.cs +++ b/src/DataTables.GeneratorCore/Serialization/IDataTableBinaryWriter.cs @@ -1,8 +1,6 @@ -using NPOI.SS.UserModel; - namespace DataTables.GeneratorCore; internal interface IDataTableBinaryWriter { - void GenerateDataFile(string outputDir, string comparisonOutputDir, bool forceOverwrite, ISheet sheet, ILogger logger); + void GenerateDataFile(string outputDir, string comparisonOutputDir, bool forceOverwrite, TableDataSnapshot snapshot, ILogger logger); } diff --git a/src/DataTables.GeneratorCore/Serialization/SnapshotDataSerializationPlan.cs b/src/DataTables.GeneratorCore/Serialization/SnapshotDataSerializationPlan.cs new file mode 100644 index 0000000..f7bb45a --- /dev/null +++ b/src/DataTables.GeneratorCore/Serialization/SnapshotDataSerializationPlan.cs @@ -0,0 +1,30 @@ +using System; +using System.IO; + +namespace DataTables.GeneratorCore; + +internal interface ITableDataSerializationPlan +{ + int WriteDataRows(TableDataSnapshot snapshot, BinaryWriter writer); +} + +internal sealed class SnapshotDataSerializationPlan : ITableDataSerializationPlan +{ + public int WriteDataRows(TableDataSnapshot snapshot, BinaryWriter writer) + { + foreach (var row in snapshot.Rows) + { + foreach (var cell in row.Cells) + { + if (cell.State == TableDataCellState.Invalid) + { + throw new InvalidOperationException($"Cannot serialize invalid materialized cell '{cell.CellReference}'."); + } + + writer.Write(cell.EncodedValue.Span); + } + } + + return snapshot.Rows.Count; + } +} diff --git a/src/DataTables.GeneratorCore/Serialization/TableDataMaterializer.cs b/src/DataTables.GeneratorCore/Serialization/TableDataMaterializer.cs new file mode 100644 index 0000000..94e78bd --- /dev/null +++ b/src/DataTables.GeneratorCore/Serialization/TableDataMaterializer.cs @@ -0,0 +1,414 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using NPOI.SS.UserModel; + +namespace DataTables.GeneratorCore; + +internal enum TableDataLayout +{ + Row, + Matrix, + Column, + KeyValue, +} + +internal sealed class TableDataMaterializationContext +{ + public TableDataMaterializationContext( + GenerationContext generationContext, + ISheet sheet, + int firstDataRowIndex, + DiagnosticsCollector diagnostics, + Func getCellString) + { + GenerationContext = generationContext; + Sheet = sheet; + FirstDataRowIndex = firstDataRowIndex; + Diagnostics = diagnostics; + GetCellString = getCellString; + } + + public GenerationContext GenerationContext { get; } + + public ISheet Sheet { get; } + + public int FirstDataRowIndex { get; } + + public DiagnosticsCollector Diagnostics { get; } + + public Func GetCellString { get; } +} + +internal interface ITableDataMaterializer +{ + TableDataSnapshot Materialize(TableDataMaterializationContext context); +} + +internal sealed class TableDataMaterializer : ITableDataMaterializer +{ + private static readonly Regex s_CellRowRegex = new(@"\d+", RegexOptions.Compiled); + + private readonly TableDataLayout m_Layout; + + public TableDataMaterializer(TableDataLayout layout) + { + m_Layout = layout; + } + + public TableDataSnapshot Materialize(TableDataMaterializationContext context) + { + var fieldEncoders = new Dictionary(); + var invalidTypeCells = context.GenerationContext.Fields + .Where(field => !field.IsIgnore) + .Select(field => (Field: field, Encoder: GetFieldEncoder(field, fieldEncoders))) + .Where(item => item.Encoder.Error != null) + .Select(item => new TableDataCell( + item.Field, + TryGetRowIndex(item.Field.TypeCell), + TryGetColumnIndex(item.Field.TypeCell), + item.Field.TypeCell, + string.Empty, + TableDataCellState.Invalid, + [], + item.Encoder.Error)) + .ToArray(); + if (invalidTypeCells.Length > 0) + { + return new TableDataSnapshot([new TableDataRow(sourceRowIndex: -1, invalidTypeCells)]); + } + + return m_Layout switch + { + TableDataLayout.Row => MaterializeRows(context, fieldEncoders), + TableDataLayout.Matrix => MaterializeMatrix(context, fieldEncoders), + TableDataLayout.Column => MaterializeColumns(context, fieldEncoders), + TableDataLayout.KeyValue => MaterializeKeyValue(context, fieldEncoders), + _ => throw new ArgumentOutOfRangeException(nameof(m_Layout), m_Layout, "Unknown table data layout."), + }; + } + + private TableDataSnapshot MaterializeRows(TableDataMaterializationContext context, Dictionary fieldEncoders) + { + var rows = new List(); + for (int rowIndex = context.FirstDataRowIndex; rowIndex <= context.Sheet.LastRowNum; rowIndex++) + { + var row = context.Sheet.GetRow(rowIndex); + if (row == null || row.FirstCellNum < 0) + { + continue; + } + + var reader = new RawRowReader(row, context.GetCellString); + var commentReadErrors = context.GenerationContext.Fields + .Where(field => field.IsComment && reader.Read(field.Index).Error != null) + .ToArray(); + if (commentReadErrors.Length > 0) + { + rows.Add(new TableDataRow( + rowIndex, + commentReadErrors.Select(field => EncodeCell(field, rowIndex, field.Index, reader.Read(field.Index), fieldEncoders)))); + continue; + } + + if (!ShouldIncludeRow(context.GenerationContext, reader)) + { + continue; + } + + rows.Add(new TableDataRow( + rowIndex, + context.GenerationContext.Fields + .Where(field => !field.IsIgnore) + .Select(field => EncodeCell(field, rowIndex, field.Index, reader.Read(field.Index), fieldEncoders)))); + } + + return new TableDataSnapshot(rows); + } + + private TableDataSnapshot MaterializeMatrix(TableDataMaterializationContext context, Dictionary fieldEncoders) + { + var rows = new List(); + var key1Field = context.GenerationContext.GetField(DataMatrixTemplate.kKey1) + ?? throw new InvalidOperationException($"DTGen=matrix 缺少 {DataMatrixTemplate.kKey1} 字段"); + var key2Field = context.GenerationContext.GetField(DataMatrixTemplate.kKey2) + ?? throw new InvalidOperationException($"DTGen=matrix 缺少 {DataMatrixTemplate.kKey2} 字段"); + var valueField = context.GenerationContext.GetField(DataMatrixTemplate.kValue) + ?? throw new InvalidOperationException($"DTGen=matrix 缺少 {DataMatrixTemplate.kValue} 字段"); + + for (int rowIndex = context.FirstDataRowIndex; rowIndex <= context.Sheet.LastRowNum; rowIndex++) + { + var row = context.Sheet.GetRow(rowIndex); + if (row == null || row.FirstCellNum < 0) + { + continue; + } + + var reader = new RawRowReader(row, context.GetCellString); + var rowReadErrors = context.GenerationContext.Fields + .Where(field => !field.IsIgnore && reader.Read(field.Index).Error != null) + .ToArray(); + if (rowReadErrors.Length > 0) + { + rows.Add(new TableDataRow( + rowIndex, + rowReadErrors.Select(field => EncodeCell(field, rowIndex, field.Index, reader.Read(field.Index), fieldEncoders)))); + continue; + } + + if (!ShouldIncludeRow(context.GenerationContext, reader)) + { + continue; + } + + foreach (var pair in context.GenerationContext.ColumnIndexToKey2) + { + var rawValue = reader.Read(pair.Key); + if (rawValue.Error == null + && (string.IsNullOrEmpty(rawValue.Value) + || rawValue.Value == context.GenerationContext.MatrixDefaultValue)) + { + if (rawValue.Value == context.GenerationContext.MatrixDefaultValue) + { + context.Diagnostics.GetMetrics(context.GenerationContext.FileName, context.GenerationContext.SheetName).MatrixDefaultSkippedCount++; + } + continue; + } + + rows.Add(new TableDataRow(rowIndex, + [ + EncodeCell(key1Field, rowIndex, row.FirstCellNum, reader.Read(row.FirstCellNum), fieldEncoders), + EncodeCell(key2Field, context.FirstDataRowIndex - 1, pair.Key, RawCellValue.Virtual(pair.Value), fieldEncoders), + EncodeCell(valueField, rowIndex, pair.Key, rawValue, fieldEncoders), + ])); + } + } + + return new TableDataSnapshot(rows); + } + + private TableDataSnapshot MaterializeColumns(TableDataMaterializationContext context, Dictionary fieldEncoders) + { + var rows = new List(); + int maxLastCellNum = 0; + for (int rowIndex = context.FirstDataRowIndex; rowIndex <= context.Sheet.LastRowNum; rowIndex++) + { + var row = context.Sheet.GetRow(rowIndex); + if (row != null && row.LastCellNum > maxLastCellNum) + { + maxLastCellNum = row.LastCellNum; + } + } + + for (int columnIndex = context.GenerationContext.ColumnFirstDataColIndex; columnIndex < maxLastCellNum; columnIndex++) + { + if (context.GenerationContext.ColumnCommentRowIndex >= 0) + { + var markRow = context.Sheet.GetRow(context.GenerationContext.ColumnCommentRowIndex); + var mark = ReadCell(markRow?.GetCell(columnIndex), context.GetCellString); + if (mark.Error != null) + { + var markerField = new XField(context.GenerationContext.ColumnCommentRowIndex) + { + Name = "#column", + TypeName = "string", + }; + rows.Add(new TableDataRow( + sourceRowIndex: -1, + [EncodeCell(markerField, context.GenerationContext.ColumnCommentRowIndex, columnIndex, mark, fieldEncoders)])); + continue; + } + + if (mark.Error == null + && !string.IsNullOrEmpty(mark.Value) + && mark.Value.TrimStart().StartsWith("#", StringComparison.Ordinal)) + { + context.Diagnostics.GetMetrics(context.GenerationContext.FileName, context.GenerationContext.SheetName).SkippedColumnCount++; + continue; + } + } + + var rawCells = context.GenerationContext.Fields + .Where(field => !field.IsIgnore) + .Select(field => (Field: field, Value: ReadCell(context.Sheet.GetRow(field.Index)?.GetCell(columnIndex), context.GetCellString))) + .ToArray(); + if (!rawCells.Any(item => item.Value.Error != null || !string.IsNullOrEmpty(item.Value.Value))) + { + continue; + } + + rows.Add(new TableDataRow( + sourceRowIndex: -1, + rawCells.Select(item => EncodeCell(item.Field, item.Field.Index, columnIndex, item.Value, fieldEncoders)))); + } + + return new TableDataSnapshot(rows); + } + + private TableDataSnapshot MaterializeKeyValue(TableDataMaterializationContext context, Dictionary fieldEncoders) + { + var cells = context.GenerationContext.Fields + .Where(field => !field.IsIgnore) + .Select(field => + { + var rowIndex = TryGetRowIndex(field.TypeCell); + return EncodeCell(field, rowIndex, field.Index, RawCellValue.Virtual(field.Note), fieldEncoders); + }) + .ToArray(); + + return new TableDataSnapshot([new TableDataRow(sourceRowIndex: -1, cells)]); + } + + private static bool ShouldIncludeRow(GenerationContext context, RawRowReader reader) + { + foreach (var field in context.Fields.Where(field => field.IsComment)) + { + var value = reader.Read(field.Index); + if (value.Error == null && value.Value == "#") + { + return false; + } + } + + return context.Fields + .Where(field => !field.IsIgnore) + .Select(field => reader.Read(field.Index)) + .Any(value => value.Error != null || !string.IsNullOrEmpty(value.Value)); + } + + private TableDataCell EncodeCell( + XField field, + int rowIndex, + int columnIndex, + RawCellValue rawCell, + Dictionary fieldEncoders) + { + var cellReference = rowIndex >= 0 && columnIndex >= 0 + ? ParserUtils.GetCellAddress(rowIndex, columnIndex) + : string.Empty; + if (rawCell.Error != null) + { + return new TableDataCell(field, rowIndex, columnIndex, cellReference, rawCell.Value, TableDataCellState.Invalid, [], rawCell.Error); + } + + var encoder = GetFieldEncoder(field, fieldEncoders); + if (encoder.Error != null) + { + return new TableDataCell(field, rowIndex, columnIndex, cellReference, rawCell.Value, TableDataCellState.Invalid, [], encoder.Error); + } + + try + { + using var stream = new MemoryStream(); + using var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true); + encoder.Processor!.WriteToStream(writer, rawCell.Value); + return new TableDataCell(field, rowIndex, columnIndex, cellReference, rawCell.Value, rawCell.State, stream.ToArray(), null); + } + catch (Exception exception) + { + return new TableDataCell(field, rowIndex, columnIndex, cellReference, rawCell.Value, TableDataCellState.Invalid, [], exception); + } + } + + private static FieldEncoder GetFieldEncoder(XField field, Dictionary fieldEncoders) + { + if (fieldEncoders.TryGetValue(field, out var encoder)) + { + return encoder; + } + + try + { + encoder = new FieldEncoder(DataTableProcessor.GetDataProcessorForSerialization(field.TypeName), null); + } + catch (Exception exception) + { + encoder = new FieldEncoder(null, exception); + } + + fieldEncoders.Add(field, encoder); + return encoder; + } + + private static RawCellValue ReadCell(ICell? cell, Func getCellString) + { + try + { + var value = getCellString(cell); + var state = cell == null + ? TableDataCellState.Missing + : string.IsNullOrEmpty(value) + ? TableDataCellState.Empty + : TableDataCellState.Parsed; + return new RawCellValue(value, state, null); + } + catch (Exception exception) + { + return new RawCellValue(string.Empty, TableDataCellState.Invalid, exception); + } + } + + private static int TryGetRowIndex(string cellReference) + { + var match = s_CellRowRegex.Match(cellReference ?? string.Empty); + return match.Success && int.TryParse(match.Value, out var oneBasedRow) ? oneBasedRow - 1 : -1; + } + + private static int TryGetColumnIndex(string cellReference) + { + int columnIndex = 0; + int letterCount = 0; + foreach (var character in cellReference ?? string.Empty) + { + if (!char.IsLetter(character)) + { + break; + } + + columnIndex = (columnIndex * 26) + (char.ToUpperInvariant(character) - 'A' + 1); + letterCount++; + } + + return letterCount == 0 ? -1 : columnIndex - 1; + } + + private sealed class RawRowReader + { + private readonly IRow m_Row; + private readonly Func m_GetCellString; + private readonly Dictionary m_Values = new(); + + public RawRowReader(IRow row, Func getCellString) + { + m_Row = row; + m_GetCellString = getCellString; + } + + public RawCellValue Read(int columnIndex) + { + if (!m_Values.TryGetValue(columnIndex, out var value)) + { + value = ReadCell(m_Row.GetCell(columnIndex), m_GetCellString); + m_Values.Add(columnIndex, value); + } + + return value; + } + } + + private sealed record FieldEncoder(DataTableProcessor.DataProcessor? Processor, Exception? Error); + + private sealed record RawCellValue(string Value, TableDataCellState State, Exception? Error) + { + public static RawCellValue Virtual(string value) + { + return new RawCellValue( + value, + string.IsNullOrEmpty(value) ? TableDataCellState.Empty : TableDataCellState.Parsed, + null); + } + } +} diff --git a/src/DataTables.GeneratorCore/Serialization/TableDataSnapshot.cs b/src/DataTables.GeneratorCore/Serialization/TableDataSnapshot.cs new file mode 100644 index 0000000..295155f --- /dev/null +++ b/src/DataTables.GeneratorCore/Serialization/TableDataSnapshot.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace DataTables.GeneratorCore; + +internal enum TableDataCellState +{ + Missing, + Empty, + Parsed, + Invalid, +} + +internal sealed class TableDataCell +{ + private readonly byte[] m_EncodedValue; + + public TableDataCell( + XField field, + int rowIndex, + int columnIndex, + string cellReference, + string rawValue, + TableDataCellState state, + byte[] encodedValue, + Exception? error) + { + FieldName = field.Name; + TypeName = field.TypeName; + RowIndex = rowIndex; + ColumnIndex = columnIndex; + CellReference = cellReference; + RawValue = rawValue; + State = state; + m_EncodedValue = encodedValue; + Error = error; + } + + public string FieldName { get; } + + public string TypeName { get; } + + public int RowIndex { get; } + + public int ColumnIndex { get; } + + public string CellReference { get; } + + public string RawValue { get; } + + public TableDataCellState State { get; } + + public ReadOnlyMemory EncodedValue => m_EncodedValue; + + public Exception? Error { get; } +} + +internal sealed class TableDataRow +{ + private readonly IReadOnlyList m_Cells; + + public TableDataRow(int sourceRowIndex, IEnumerable cells) + { + SourceRowIndex = sourceRowIndex; + m_Cells = Array.AsReadOnly(cells.ToArray()); + } + + public int SourceRowIndex { get; } + + public IReadOnlyList Cells => m_Cells; + + public TableDataCell? GetCell(string fieldName) + { + return m_Cells.FirstOrDefault(cell => string.Equals(cell.FieldName, fieldName, StringComparison.Ordinal)); + } +} + +internal sealed class TableDataSnapshot +{ + private readonly IReadOnlyList m_Rows; + + public TableDataSnapshot(IEnumerable rows) + { + m_Rows = Array.AsReadOnly(rows.ToArray()); + EstimatedSizeInBytes = m_Rows.Sum(row => row.Cells.Sum(cell => + (long)cell.EncodedValue.Length + (cell.RawValue.Length * sizeof(char)))); + } + + public IReadOnlyList Rows => m_Rows; + + public IEnumerable Cells => m_Rows.SelectMany(row => row.Cells); + + public long EstimatedSizeInBytes { get; } +} diff --git a/src/DataTables.GeneratorCore/TableSchemaParserRegistry.cs b/src/DataTables.GeneratorCore/TableSchemaParserRegistry.cs index 1438cc5..c66e9c2 100644 --- a/src/DataTables.GeneratorCore/TableSchemaParserRegistry.cs +++ b/src/DataTables.GeneratorCore/TableSchemaParserRegistry.cs @@ -6,16 +6,8 @@ namespace DataTables.GeneratorCore; public sealed class TableSchemaParserRegistry : ITableSchemaParserRegistry { - private static readonly string[] s_ReservedDataSetTypes = - [ - "localized", - "tree", - "partitioned", - "versioned", - "patch" - ]; - private readonly Dictionary m_Parsers; + private readonly IReadOnlyCollection m_ReservedDataSetTypes; public TableSchemaParserRegistry(IEnumerable parsers) { @@ -25,24 +17,19 @@ public TableSchemaParserRegistry(IEnumerable parsers) parser => parser.DataSetType, parser => parser, StringComparer.OrdinalIgnoreCase); + m_ReservedDataSetTypes = TableTypeDescriptorRegistry.Default.ReservedDataSetTypes + .Where(dataSetType => !m_Parsers.ContainsKey(dataSetType)) + .ToArray(); } public static ITableSchemaParserRegistry CreateDefault() { - return new TableSchemaParserRegistry( - [ - new RowTableParser(), - new MatrixTableParser(), - new ColumnTableParser(), - new KvTableParser(), - new TreeTableParser(), - new GraphTableParser() - ]); + return new TableSchemaParserRegistry(TableTypeDescriptorRegistry.Default.Descriptors.Select(descriptor => descriptor.SchemaParser)); } public IReadOnlyCollection SupportedDataSetTypes => m_Parsers.Keys.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToArray(); - public IReadOnlyCollection ReservedDataSetTypes => s_ReservedDataSetTypes; + public IReadOnlyCollection ReservedDataSetTypes => m_ReservedDataSetTypes; public bool TryGetParser(string dataSetType, out ITableSchemaParser parser) { diff --git a/src/DataTables.GeneratorCore/TableTypeDescriptorRegistry.cs b/src/DataTables.GeneratorCore/TableTypeDescriptorRegistry.cs new file mode 100644 index 0000000..7527615 --- /dev/null +++ b/src/DataTables.GeneratorCore/TableTypeDescriptorRegistry.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace DataTables.GeneratorCore; + +internal sealed class TableTypeDescriptor +{ + public TableTypeDescriptor( + string dataSetType, + ITableSchemaParser schemaParser, + ITableDataMaterializer dataMaterializer, + IEnumerable validationPasses, + ITableDataSerializationPlan serializationPlan, + ICodeTemplateRenderer codeRenderer) + { + DataSetType = dataSetType; + SchemaParser = schemaParser; + DataMaterializer = dataMaterializer; + ValidationPasses = validationPasses.OrderBy(pass => pass.Phase).ToArray(); + SerializationPlan = serializationPlan; + CodeRenderer = codeRenderer; + } + + public string DataSetType { get; } + + public ITableSchemaParser SchemaParser { get; } + + public ITableDataMaterializer DataMaterializer { get; } + + public IReadOnlyList ValidationPasses { get; } + + public ITableDataSerializationPlan SerializationPlan { get; } + + public ICodeTemplateRenderer CodeRenderer { get; } +} + +internal sealed class TableTypeDescriptorRegistry +{ + private static readonly string[] s_ReservedDataSetTypes = + [ + "localized", + "partitioned", + "versioned", + "patch", + ]; + + private readonly Dictionary m_Descriptors; + + private TableTypeDescriptorRegistry(IEnumerable descriptors) + { + m_Descriptors = descriptors.ToDictionary( + descriptor => descriptor.DataSetType, + descriptor => descriptor, + StringComparer.OrdinalIgnoreCase); + } + + public static TableTypeDescriptorRegistry Default { get; } = CreateDefault(); + + public IReadOnlyCollection Descriptors => m_Descriptors.Values + .OrderBy(descriptor => descriptor.DataSetType, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + public IReadOnlyCollection SupportedDataSetTypes => m_Descriptors.Keys.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToArray(); + + public IReadOnlyCollection ReservedDataSetTypes => s_ReservedDataSetTypes; + + public bool TryGetDescriptor(string dataSetType, out TableTypeDescriptor descriptor) + { + return m_Descriptors.TryGetValue(dataSetType, out descriptor!); + } + + private static TableTypeDescriptorRegistry CreateDefault() + { + var cellConversion = new CellConversionValidationPass(); + var serialization = new SnapshotDataSerializationPlan(); + + return new TableTypeDescriptorRegistry( + [ + new TableTypeDescriptor( + "table", + new RowTableParser(), + new TableDataMaterializer(TableDataLayout.Row), + [cellConversion], + serialization, + new DataTableCodeTemplateRenderer("table")), + new TableTypeDescriptor( + "matrix", + new MatrixTableParser(), + new TableDataMaterializer(TableDataLayout.Matrix), + [cellConversion], + serialization, + new DataMatrixCodeTemplateRenderer()), + new TableTypeDescriptor( + "column", + new ColumnTableParser(), + new TableDataMaterializer(TableDataLayout.Column), + [cellConversion], + serialization, + new DataTableCodeTemplateRenderer("column")), + new TableTypeDescriptor( + "kv", + new KvTableParser(), + new TableDataMaterializer(TableDataLayout.KeyValue), + [cellConversion], + serialization, + new KvTableCodeTemplateRenderer()), + new TableTypeDescriptor( + "tree", + new TreeTableParser(), + new TableDataMaterializer(TableDataLayout.Row), + [cellConversion, new TreeRowValidationPass(), new TreeRelationshipValidationPass()], + serialization, + new TreeTableCodeTemplateRenderer()), + new TableTypeDescriptor( + "graph", + new GraphTableParser(), + new TableDataMaterializer(TableDataLayout.Row), + [cellConversion, new GraphRowValidationPass()], + serialization, + new GraphTableCodeTemplateRenderer()), + ]); + } +} diff --git a/src/DataTables.GeneratorCore/Validation/GraphTableDataValidation.cs b/src/DataTables.GeneratorCore/Validation/GraphTableDataValidation.cs new file mode 100644 index 0000000..33c7b21 --- /dev/null +++ b/src/DataTables.GeneratorCore/Validation/GraphTableDataValidation.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace DataTables.GeneratorCore; + +internal sealed class GraphRowValidationPass : ITableDataValidationPass +{ + public TableDataValidationPhase Phase => TableDataValidationPhase.RowRules; + + public void Validate(TableDataValidationContext context) + { + var edgeIds = new Dictionary(StringComparer.Ordinal); + foreach (var row in context.Snapshot.Rows) + { + var edgeIdCell = row.GetCell("EdgeId") ?? throw new InvalidOperationException("DTGen=graph 缺少 EdgeId 字段"); + var fromCell = row.GetCell("From") ?? throw new InvalidOperationException("DTGen=graph 缺少 From 字段"); + var toCell = row.GetCell("To") ?? throw new InvalidOperationException("DTGen=graph 缺少 To 字段"); + var edgeId = edgeIdCell.RawValue; + + if (string.IsNullOrWhiteSpace(edgeId)) + { + context.Error(edgeIdCell, $"DTGen=graph EdgeId 为空: row={edgeIdCell.RowIndex + 1}"); + } + else if (!edgeIds.TryAdd(edgeId, edgeIdCell)) + { + context.Error(edgeIdCell, $"DTGen=graph EdgeId 重复: edgeId={edgeId}, firstCell={edgeIds[edgeId].CellReference}"); + } + + if (string.IsNullOrWhiteSpace(fromCell.RawValue)) + { + context.Error(fromCell, $"DTGen=graph From 为空: edgeId={edgeId}"); + } + + if (string.IsNullOrWhiteSpace(toCell.RawValue)) + { + context.Error(toCell, $"DTGen=graph To 为空: edgeId={edgeId}"); + } + + if (row.GetCell("Weight") is { } weightCell + && !string.IsNullOrWhiteSpace(weightCell.RawValue) + && !decimal.TryParse(weightCell.RawValue, NumberStyles.Number, CultureInfo.InvariantCulture, out _)) + { + context.Error(weightCell, $"DTGen=graph Weight 不是合法数字: edgeId={edgeId}, value={weightCell.RawValue}"); + } + } + } +} diff --git a/src/DataTables.GeneratorCore/Validation/TableDataValidation.cs b/src/DataTables.GeneratorCore/Validation/TableDataValidation.cs new file mode 100644 index 0000000..5d08b65 --- /dev/null +++ b/src/DataTables.GeneratorCore/Validation/TableDataValidation.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace DataTables.GeneratorCore; + +internal enum TableDataValidationPhase +{ + CellConversion = 0, + RowRules = 100, + RelationshipRules = 200, + ProjectRules = 300, +} + +internal sealed class TableDataValidationContext +{ + public TableDataValidationContext( + GenerationContext generationContext, + TableDataSnapshot snapshot, + DiagnosticsCollector diagnostics) + { + GenerationContext = generationContext; + Snapshot = snapshot; + Diagnostics = diagnostics; + } + + public GenerationContext GenerationContext { get; } + + public TableDataSnapshot Snapshot { get; } + + public DiagnosticsCollector Diagnostics { get; } + + public void Error(TableDataCell cell, string message) + { + Diagnostics.Error( + GenerationContext.FileName, + GenerationContext.SheetName, + cell.CellReference, + message); + } +} + +internal interface ITableDataValidationPass +{ + TableDataValidationPhase Phase { get; } + + void Validate(TableDataValidationContext context); +} + +internal sealed class CellConversionValidationPass : ITableDataValidationPass +{ + public TableDataValidationPhase Phase => TableDataValidationPhase.CellConversion; + + public void Validate(TableDataValidationContext context) + { + foreach (var cell in context.Snapshot.Cells.Where(cell => cell.State == TableDataCellState.Invalid)) + { + var location = string.IsNullOrEmpty(cell.CellReference) ? string.Empty : $" ({cell.CellReference})"; + if (cell.Error is DataTableProcessor.DataTypeParseException) + { + context.Error( + cell, + $"字段 '{cell.FieldName}' 类型 '{cell.TypeName}' 解析失败{location}: {GetRootMessage(cell.Error)}"); + continue; + } + + context.Error( + cell, + $"字段 '{cell.FieldName}' 的值无法按类型 '{cell.TypeName}' 解析{location}: '{cell.RawValue}'. {GetRootMessage(cell.Error)}"); + } + } + + private static string GetRootMessage(Exception? exception) + { + if (exception == null) + { + return "未知转换错误。"; + } + + while (exception.InnerException != null) + { + exception = exception.InnerException; + } + + return exception.Message; + } +} diff --git a/src/DataTables.GeneratorCore/Validation/TreeTableDataValidation.cs b/src/DataTables.GeneratorCore/Validation/TreeTableDataValidation.cs new file mode 100644 index 0000000..fdbfa96 --- /dev/null +++ b/src/DataTables.GeneratorCore/Validation/TreeTableDataValidation.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace DataTables.GeneratorCore; + +internal sealed class TreeRowValidationPass : ITableDataValidationPass +{ + public TableDataValidationPhase Phase => TableDataValidationPhase.RowRules; + + public void Validate(TableDataValidationContext context) + { + var nodeIds = new Dictionary(StringComparer.Ordinal); + foreach (var row in context.Snapshot.Rows) + { + var idCell = row.GetCell("Id") ?? throw new InvalidOperationException("DTGen=tree 缺少 Id 字段"); + var id = idCell.RawValue; + if (string.IsNullOrWhiteSpace(id)) + { + context.Error(idCell, $"DTGen=tree Id 为空: row={idCell.RowIndex + 1}"); + continue; + } + + if (!nodeIds.TryAdd(id, idCell)) + { + var first = nodeIds[id]; + context.Error(idCell, $"DTGen=tree Id 重复: nodeId={id}, firstCell={first.CellReference}"); + } + + if (row.GetCell("Order") is { } orderCell + && !string.IsNullOrWhiteSpace(orderCell.RawValue) + && !decimal.TryParse(orderCell.RawValue, NumberStyles.Number, CultureInfo.InvariantCulture, out _)) + { + context.Error(orderCell, $"DTGen=tree Order 不是合法数字: nodeId={id}, value={orderCell.RawValue}"); + } + } + } +} + +internal sealed class TreeRelationshipValidationPass : ITableDataValidationPass +{ + public TableDataValidationPhase Phase => TableDataValidationPhase.RelationshipRules; + + public void Validate(TableDataValidationContext context) + { + var nodes = context.Snapshot.Rows.ToDictionary( + row => row.GetCell("Id")!.RawValue, + row => new TreeNode( + row.GetCell("Id")!, + row.GetCell("ParentId") ?? throw new InvalidOperationException("DTGen=tree 缺少 ParentId 字段")), + StringComparer.Ordinal); + + var errorCountBeforeReferences = context.Diagnostics.ErrorCount; + foreach (var node in nodes.Values) + { + if (!string.IsNullOrEmpty(node.ParentCell.RawValue) + && !nodes.ContainsKey(node.ParentCell.RawValue)) + { + context.Error( + node.ParentCell, + $"DTGen=tree ParentId 引用不存在: nodeId={node.IdCell.RawValue}, parentId={node.ParentCell.RawValue}"); + } + } + + if (context.Diagnostics.ErrorCount > errorCountBeforeReferences) + { + return; + } + + var visited = new HashSet(StringComparer.Ordinal); + var visiting = new HashSet(StringComparer.Ordinal); + var path = new List(); + foreach (var id in nodes.Keys) + { + if (Visit(id)) + { + return; + } + } + + bool Visit(string id) + { + if (visited.Contains(id)) + { + return false; + } + + if (!visiting.Add(id)) + { + var start = path.IndexOf(id); + var cycle = start >= 0 ? path.Skip(start).Concat([id]) : [id, id]; + context.Error(nodes[id].IdCell, $"DTGen=tree 检测到循环引用: {string.Join(" -> ", cycle)}"); + return true; + } + + path.Add(id); + var parentId = nodes[id].ParentCell.RawValue; + if (!string.IsNullOrEmpty(parentId) && Visit(parentId)) + { + return true; + } + + path.RemoveAt(path.Count - 1); + visiting.Remove(id); + visited.Add(id); + return false; + } + } + + private sealed record TreeNode(TableDataCell IdCell, TableDataCell ParentCell); +} diff --git a/tests/DataTables.Tests/GenerationTransactionTests.cs b/tests/DataTables.Tests/GenerationTransactionTests.cs index c15fa7d..a2d9c9f 100644 --- a/tests/DataTables.Tests/GenerationTransactionTests.cs +++ b/tests/DataTables.Tests/GenerationTransactionTests.cs @@ -281,6 +281,92 @@ public async Task ValidateOnlyGeneration_ShouldParseAndRenderWithoutWritingOutpu logs.Should().Contain(message => message.Contains("数据表校验完成")); } + [Theory] + [InlineData(GenerationMode.CodeAndData)] + [InlineData(GenerationMode.ValidateOnly)] + public async Task Generation_ShouldRejectInvalidCellValuesBeforeWritingOutputs(GenerationMode generationMode) + { + var root = CreateTempDirectory(); + var input = Directory.CreateDirectory(Path.Combine(root, "input")).FullName; + var code = Directory.CreateDirectory(Path.Combine(root, "code")).FullName; + var data = Directory.CreateDirectory(Path.Combine(root, "data")).FullName; + await CreateWorkbookAsync(Path.Combine(input, "invalid-value.xlsx"), workbook => + AddRowLayoutSheet( + workbook, + "Items", + "table", + "Item", + ["Id"], + ["int"], + ["not-an-int"])); + + var result = await GenerateAsync(input, code, data, generationMode: generationMode); + + result.Succeeded.Should().BeFalse(); + result.Failures.Should().Contain(failure => + failure.Exception.Message.Contains("字段 'Id' 的值无法按类型 'int' 解析", StringComparison.Ordinal)); + Directory.GetFiles(code, "*", SearchOption.AllDirectories).Should().BeEmpty(); + Directory.GetFiles(data, "*", SearchOption.AllDirectories).Should().BeEmpty(); + } + + [Theory] + [InlineData(GenerationMode.CodeAndData)] + [InlineData(GenerationMode.ValidateOnly)] + public async Task Generation_ShouldApplyTreeRelationshipValidationInEveryMode(GenerationMode generationMode) + { + var root = CreateTempDirectory(); + var input = Directory.CreateDirectory(Path.Combine(root, "input")).FullName; + var code = Directory.CreateDirectory(Path.Combine(root, "code")).FullName; + var data = Directory.CreateDirectory(Path.Combine(root, "data")).FullName; + await CreateWorkbookAsync(Path.Combine(input, "tree-cycle.xlsx"), workbook => + AddRowLayoutSheet( + workbook, + "Tree", + "tree", + "Node", + ["Id", "ParentId", "Order"], + ["string", "string", "int"], + ["A", "B", "1"], + ["B", "A", "2"])); + + var result = await GenerateAsync(input, code, data, generationMode: generationMode); + + result.Succeeded.Should().BeFalse(); + result.Failures.Should().Contain(failure => + failure.Exception.Message.Contains("DTGen=tree 检测到循环引用: A -> B -> A", StringComparison.Ordinal)); + Directory.GetFiles(code, "*", SearchOption.AllDirectories).Should().BeEmpty(); + Directory.GetFiles(data, "*", SearchOption.AllDirectories).Should().BeEmpty(); + } + + [Theory] + [InlineData(GenerationMode.CodeAndData)] + [InlineData(GenerationMode.ValidateOnly)] + public async Task Generation_ShouldApplyGraphRowValidationInEveryMode(GenerationMode generationMode) + { + var root = CreateTempDirectory(); + var input = Directory.CreateDirectory(Path.Combine(root, "input")).FullName; + var code = Directory.CreateDirectory(Path.Combine(root, "code")).FullName; + var data = Directory.CreateDirectory(Path.Combine(root, "data")).FullName; + await CreateWorkbookAsync(Path.Combine(input, "graph-duplicate.xlsx"), workbook => + AddRowLayoutSheet( + workbook, + "Graph", + "graph", + "LevelGraph", + ["EdgeId", "From", "To", "Weight"], + ["string", "string", "string", "float"], + ["E1", "A", "B", "1"], + ["E1", "B", "C", "2"])); + + var result = await GenerateAsync(input, code, data, generationMode: generationMode); + + result.Succeeded.Should().BeFalse(); + result.Failures.Should().Contain(failure => + failure.Exception.Message.Contains("DTGen=graph EdgeId 重复: edgeId=E1", StringComparison.Ordinal)); + Directory.GetFiles(code, "*", SearchOption.AllDirectories).Should().BeEmpty(); + Directory.GetFiles(data, "*", SearchOption.AllDirectories).Should().BeEmpty(); + } + [Fact] public async Task ValidateOnlyGeneration_ShouldReportTemplateAndSchemaConflictsWithoutTouchingOutputs() { @@ -602,6 +688,39 @@ private static void AddTableSheet(XSSFWorkbook workbook, string sheetName, strin sheet.GetRow(4).CreateCell(1, CellType.String).SetCellValue("Item"); } + private static void AddRowLayoutSheet( + XSSFWorkbook workbook, + string sheetName, + string dataSetType, + string className, + string[] fieldNames, + string[] fieldTypes, + params string[][] dataRows) + { + fieldTypes.Should().HaveCount(fieldNames.Length); + var sheet = workbook.CreateSheet(sheetName); + sheet.CreateRow(0).CreateCell(0, CellType.String).SetCellValue($"dtgen={dataSetType}, class={className}"); + var commentRow = sheet.CreateRow(1); + var nameRow = sheet.CreateRow(2); + var typeRow = sheet.CreateRow(3); + for (int columnIndex = 0; columnIndex < fieldNames.Length; columnIndex++) + { + commentRow.CreateCell(columnIndex, CellType.String).SetCellValue(fieldNames[columnIndex]); + nameRow.CreateCell(columnIndex, CellType.String).SetCellValue(fieldNames[columnIndex]); + typeRow.CreateCell(columnIndex, CellType.String).SetCellValue(fieldTypes[columnIndex]); + } + + for (int rowIndex = 0; rowIndex < dataRows.Length; rowIndex++) + { + dataRows[rowIndex].Should().HaveCount(fieldNames.Length); + var row = sheet.CreateRow(rowIndex + 4); + for (int columnIndex = 0; columnIndex < fieldNames.Length; columnIndex++) + { + row.CreateCell(columnIndex, CellType.String).SetCellValue(dataRows[rowIndex][columnIndex]); + } + } + } + private static string CreateTempDirectory() { var path = Path.Combine(Path.GetTempPath(), "dt_generation_transaction_" + Guid.NewGuid().ToString("N")); diff --git a/tests/DataTables.Tests/ParserTests.cs b/tests/DataTables.Tests/ParserTests.cs index 5d1cc9f..24cd336 100644 --- a/tests/DataTables.Tests/ParserTests.cs +++ b/tests/DataTables.Tests/ParserTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using FluentAssertions; using NPOI.XSSF.UserModel; using Xunit; @@ -165,9 +166,20 @@ public void Parsers_Should_Expose_DataSetType_Metadata() new MatrixTableParser().DataSetType.Should().Be("matrix"); new ColumnTableParser().DataSetType.Should().Be("column"); new KvTableParser().DataSetType.Should().Be("kv"); + new TreeTableParser().DataSetType.Should().Be("tree"); new GraphTableParser().DataSetType.Should().Be("graph"); } + [Fact] + public void DefaultRegistry_Should_Not_Report_Supported_Types_As_Reserved() + { + var registry = TableSchemaParserRegistry.CreateDefault(); + + registry.SupportedDataSetTypes.Intersect(registry.ReservedDataSetTypes).Should().BeEmpty(); + registry.SupportedDataSetTypes.Should().Contain("tree"); + registry.ReservedDataSetTypes.Should().NotContain("tree"); + } + [Fact] public void CreateGenerationContext_Should_Report_Unknown_DTGen_Type() { @@ -190,7 +202,8 @@ public void CreateGenerationContext_Should_Report_Unknown_DTGen_Type() diagnostic.Cell.Should().Be("A1"); diagnostic.Message.Should().Contain("声明值: unknown"); diagnostic.Message.Should().Contain("支持的类型: column, graph, kv, matrix, table, tree"); - diagnostic.Message.Should().Contain("预留类型: localized, tree, partitioned, versioned, patch"); + diagnostic.Message.Should().Contain("预留类型: localized, partitioned, versioned, patch"); + diagnostic.Message.Should().NotContain("预留类型: localized, tree"); } [Fact]