Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
1. 阅读 [Excel 模板与 Dicts 指南](../templates/README.md)。
2. 阅读 [表类型指南](guides/table-types.md)。
3. 根据使用的表类型阅读对应设计说明。
4. 需要为配置增加合理性或跨表语义检查时,阅读 [表类型数据校验器设计](designs/table-validation-design.md)。

### 程序/工具链维护者

Expand All @@ -51,6 +52,7 @@ docs/
│ ├── graph-table-design.md
│ ├── kv-table-design.md
│ ├── localized-table-design.md
│ ├── table-validation-design.md
│ └── tree-table-design.md
├── reference/ # 稳定协议、格式和 API 参考
│ └── binary-format-v3.md
Expand Down
432 changes: 432 additions & 0 deletions docs/designs/table-validation-design.md

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions docs/guides/table-types.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

本文档说明当前已支持和计划中的表格布局。表类型由 Sheet 元信息行中的 `DTGen` 值选择。

各表类型的生成期数据合理性、结构语义与跨表校验扩展方案见 [表类型数据校验器设计](../designs/table-validation-design.md)。首个内部 validator 切片已经开始实施,但文中规划的声明式规则尚不是已发布语法。

## 已支持类型

当前代码中的默认解析器注册了 `table`、`matrix`、`column`、`kv`、`tree` 和 `graph`;代码模板也为这些类型提供生成器。
Expand Down
4 changes: 4 additions & 0 deletions src/DataTables.GeneratorCore/DataTableGenerator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -688,6 +688,10 @@ private async Task GenerateExcel(string filePath, string usingNamespace, string
continue;
}

// Validate data semantics before rendering or writing either half of the generated pair.
// ValidateOnly intentionally runs the same validators as normal generation.
processor.ValidateData(sheet);

var codeContent = renderCode ? RenderCodeFile(context) : null;
var codeContentHash = codeContent == null ? null : ComputeContentHash(codeContent);
if (!outputClaims.TryReserve(context, filePath, codeContentHash, out var writeCode, out var conflict))
Expand Down
16 changes: 16 additions & 0 deletions src/DataTables.GeneratorCore/DataTableProcessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ namespace DataTables.GeneratorCore;

public sealed partial class DataTableProcessor : IDisposable
{
private static readonly TableDataValidatorRegistry s_TableDataValidatorRegistry = TableDataValidatorRegistry.CreateDefault();
private readonly GenerationContext m_Context;
private readonly IFormulaEvaluator m_FormulaEvaluator;
private readonly string m_Tags;
Expand Down Expand Up @@ -150,6 +151,21 @@ public bool ValidateGenerationContext()
return new TableSchemaValidator(m_Context, m_Options).Validate(m_FirstDataRowIndex);
}

internal void ValidateData(ISheet sheet)
{
if (!s_TableDataValidatorRegistry.TryGetValidator(m_Context.DataSetType, out var validator))
{
throw new InvalidOperationException($"DTGen={m_Context.DataSetType} 缺少数据校验器。");
}

validator.Validate(new TableDataValidationContext(
m_Context,
sheet,
m_FirstDataRowIndex,
GetCellString,
IgnoreDataRow));
}

private void ValidateFormulaCellString(ICell cell, string value)
{
if (!m_Options.ValidateFormulaConsistency || m_Options.FormulaPolicy == FormulaEvaluationPolicy.Off)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using NPOI.SS.UserModel;
Expand Down Expand Up @@ -32,17 +31,7 @@ public int WriteDataRows(ISheet sheet, BinaryWriter writer)
{
int dataRowCount = 0;

switch (m_Context.DataSetType)
{
case "tree":
ValidateTreeRows(sheet);
break;
case "graph":
ValidateGraphRows(sheet);
break;
case "kv":
return WriteKvBytes(writer);
}
if (m_Context.DataSetType == "kv") return WriteKvBytes(writer);

if (m_Context.DataSetType == "column")
{
Expand Down Expand Up @@ -213,83 +202,6 @@ private DataTableProcessor.DataProcessor GetDataProcessorWithDiagnostics(XField
}
}

private void ValidateTreeRows(ISheet sheet)
{
var idField = m_Context.GetField("Id") ?? throw new InvalidOperationException("DTGen=tree 缺少 Id 字段");
var parentField = m_Context.GetField("ParentId") ?? throw new InvalidOperationException("DTGen=tree 缺少 ParentId 字段");
var orderField = m_Context.GetField("Order");
var nodes = new Dictionary<string, (string ParentId, int Row)>(StringComparer.Ordinal);

for (int i = m_FirstDataRowIndex; i <= sheet.LastRowNum; i++)
{
var row = sheet.GetRow(i);
if (!m_ShouldWriteDataRow(row)) continue;
var id = m_GetCellString(row!.GetCell(idField.Index));
var parentId = m_GetCellString(row.GetCell(parentField.Index));
if (string.IsNullOrWhiteSpace(id)) throw new FormatException($"DTGen=tree Id 为空: row={i + 1}, cell={GetRowColString(i, idField.Index)}");
if (!nodes.TryAdd(id, (parentId, i))) throw new FormatException($"DTGen=tree Id 重复: nodeId={id}, row={i + 1}, cell={GetRowColString(i, idField.Index)}");
if (orderField != null)
{
var orderText = m_GetCellString(row.GetCell(orderField.Index));
if (!string.IsNullOrWhiteSpace(orderText) && !decimal.TryParse(orderText, out _)) throw new FormatException($"DTGen=tree Order 不是合法数字: nodeId={id}, row={i + 1}, cell={GetRowColString(i, orderField.Index)}, value={orderText}");
}
}

foreach (var (id, node) in nodes)
{
if (!string.IsNullOrEmpty(node.ParentId) && !nodes.ContainsKey(node.ParentId)) throw new FormatException($"DTGen=tree ParentId 引用不存在: nodeId={id}, parentId={node.ParentId}, row={node.Row + 1}");
}

var visited = new HashSet<string>(StringComparer.Ordinal);
var visiting = new HashSet<string>(StringComparer.Ordinal);
var path = new List<string>();
foreach (var id in nodes.Keys) Visit(id);

void Visit(string id)
{
if (visited.Contains(id)) return;
if (!visiting.Add(id))
{
var start = path.IndexOf(id);
var cycle = start >= 0 ? path.Skip(start).Concat(new[] { id }) : new[] { id, id };
throw new FormatException($"DTGen=tree 检测到循环引用: {string.Join(" -> ", cycle)}");
}
path.Add(id);
var parentId = nodes[id].ParentId;
if (!string.IsNullOrEmpty(parentId) && nodes.ContainsKey(parentId)) Visit(parentId);
path.RemoveAt(path.Count - 1);
visiting.Remove(id);
visited.Add(id);
}
}

private void ValidateGraphRows(ISheet sheet)
{
var edgeIdField = m_Context.GetField("EdgeId") ?? throw new InvalidOperationException("DTGen=graph 缺少 EdgeId 字段");
var fromField = m_Context.GetField("From") ?? throw new InvalidOperationException("DTGen=graph 缺少 From 字段");
var toField = m_Context.GetField("To") ?? throw new InvalidOperationException("DTGen=graph 缺少 To 字段");
var weightField = m_Context.GetField("Weight");
var edgeIds = new HashSet<string>(StringComparer.Ordinal);

for (int i = m_FirstDataRowIndex; i <= sheet.LastRowNum; i++)
{
var row = sheet.GetRow(i);
if (!m_ShouldWriteDataRow(row)) continue;
var edgeId = m_GetCellString(row!.GetCell(edgeIdField.Index));
var from = m_GetCellString(row.GetCell(fromField.Index));
var to = m_GetCellString(row.GetCell(toField.Index));
if (string.IsNullOrWhiteSpace(edgeId)) throw new FormatException($"DTGen=graph EdgeId 为空: row={i + 1}, cell={GetRowColString(i, edgeIdField.Index)}");
if (!edgeIds.Add(edgeId)) throw new FormatException($"DTGen=graph EdgeId 重复: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, edgeIdField.Index)}");
if (string.IsNullOrWhiteSpace(from)) throw new FormatException($"DTGen=graph From 为空: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, fromField.Index)}");
if (string.IsNullOrWhiteSpace(to)) throw new FormatException($"DTGen=graph To 为空: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, toField.Index)}");
if (weightField != null)
{
var weightText = m_GetCellString(row.GetCell(weightField.Index));
if (!string.IsNullOrWhiteSpace(weightText) && !decimal.TryParse(weightText, out _)) throw new FormatException($"DTGen=graph Weight 不是合法数字: edgeId={edgeId}, row={i + 1}, cell={GetRowColString(i, weightField.Index)}, value={weightText}");
}
}
}

private static string GetRowColString(int row, int col) => string.Format("{0}{1}", ConvertToDigit(col), row + 1);

private static string ConvertToDigit(int num)
Expand Down
1 change: 0 additions & 1 deletion src/DataTables.GeneratorCore/TableSchemaParserRegistry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ public sealed class TableSchemaParserRegistry : ITableSchemaParserRegistry
private static readonly string[] s_ReservedDataSetTypes =
[
"localized",
"tree",
"partitioned",
"versioned",
"patch"
Expand Down
39 changes: 39 additions & 0 deletions src/DataTables.GeneratorCore/Validation/GraphTableDataValidator.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;

namespace DataTables.GeneratorCore;

internal sealed class GraphTableDataValidator : ITableDataValidator
{
public string DataSetType => "graph";

public void Validate(TableDataValidationContext context)
{
var edgeIdField = context.Schema.GetField("EdgeId") ?? throw new InvalidOperationException("DTGen=graph 缺少 EdgeId 字段");
var fromField = context.Schema.GetField("From") ?? throw new InvalidOperationException("DTGen=graph 缺少 From 字段");
var toField = context.Schema.GetField("To") ?? throw new InvalidOperationException("DTGen=graph 缺少 To 字段");
var weightField = context.Schema.GetField("Weight");
var edgeIds = new HashSet<string>(StringComparer.Ordinal);

for (int i = context.FirstDataRowIndex; i <= context.Sheet.LastRowNum; i++)
{
var row = context.Sheet.GetRow(i);
if (!context.ShouldValidateRow(row)) continue;
var edgeId = context.GetCellString(row!.GetCell(edgeIdField.Index));
var from = context.GetCellString(row.GetCell(fromField.Index));
var to = context.GetCellString(row.GetCell(toField.Index));
if (string.IsNullOrWhiteSpace(edgeId)) throw new FormatException($"DTGen=graph EdgeId 为空: row={i + 1}, cell={TableDataValidationContext.GetCellReference(i, edgeIdField.Index)}");
if (!edgeIds.Add(edgeId)) throw new FormatException($"DTGen=graph EdgeId 重复: edgeId={edgeId}, row={i + 1}, cell={TableDataValidationContext.GetCellReference(i, edgeIdField.Index)}");
if (string.IsNullOrWhiteSpace(from)) throw new FormatException($"DTGen=graph From 为空: edgeId={edgeId}, row={i + 1}, cell={TableDataValidationContext.GetCellReference(i, fromField.Index)}");
if (string.IsNullOrWhiteSpace(to)) throw new FormatException($"DTGen=graph To 为空: edgeId={edgeId}, row={i + 1}, cell={TableDataValidationContext.GetCellReference(i, toField.Index)}");
if (weightField != null)
{
var weightText = context.GetCellString(row.GetCell(weightField.Index));
if (!string.IsNullOrWhiteSpace(weightText) && !decimal.TryParse(weightText, out _))
{
throw new FormatException($"DTGen=graph Weight 不是合法数字: edgeId={edgeId}, row={i + 1}, cell={TableDataValidationContext.GetCellReference(i, weightField.Index)}, value={weightText}");
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace DataTables.GeneratorCore;

internal interface ITableDataValidator
{
string DataSetType { get; }

void Validate(TableDataValidationContext context);
}
44 changes: 44 additions & 0 deletions src/DataTables.GeneratorCore/Validation/RowTableDataValidator.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
using System;
using System.Collections.Generic;
using System.Linq;

namespace DataTables.GeneratorCore;

internal sealed class RowTableDataValidator : ITableDataValidator
{
public string DataSetType => "table";

public void Validate(TableDataValidationContext context)
{
foreach (var constraint in context.Schema.UniqueConstraints)
{
ValidateUniqueConstraint(context, constraint.Fields);
}
}

private static void ValidateUniqueConstraint(TableDataValidationContext context, IReadOnlyList<string> fieldNames)
{
var fields = fieldNames.Select(name =>
context.Schema.GetField(name) ?? throw new InvalidOperationException($"唯一索引引用了不存在的字段: {name}"))
.ToArray();
var firstRows = new Dictionary<string, int>(StringComparer.Ordinal);

for (var rowIndex = context.FirstDataRowIndex; rowIndex <= context.Sheet.LastRowNum; rowIndex++)
{
var row = context.Sheet.GetRow(rowIndex);
if (!context.ShouldValidateRow(row)) continue;

var values = fields.Select(field => context.GetCellString(row!.GetCell(field.Index))).ToArray();
var key = string.Join("\u001F", values.Select(value => $"{value.Length}:{value}"));
if (firstRows.TryGetValue(key, out var firstRow))
{
throw new FormatException(
$"唯一索引值重复: fields={string.Join("&", fieldNames)}, value=({string.Join(", ", values)}), " +
$"firstRow={firstRow + 1}, duplicateRow={rowIndex + 1}, " +
$"cell={TableDataValidationContext.GetCellReference(rowIndex, fields[0].Index)}");
}

firstRows.Add(key, rowIndex);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
using System;
using NPOI.SS.UserModel;

namespace DataTables.GeneratorCore;

internal sealed class TableDataValidationContext
{
private readonly Func<ICell?, string> m_GetCellString;
private readonly Func<IRow?, bool> m_ShouldWriteDataRow;

public TableDataValidationContext(
GenerationContext schema,
ISheet sheet,
int firstDataRowIndex,
Func<ICell?, string> getCellString,
Func<IRow?, bool> shouldWriteDataRow)
{
Schema = schema;
Sheet = sheet;
FirstDataRowIndex = firstDataRowIndex;
m_GetCellString = getCellString;
m_ShouldWriteDataRow = shouldWriteDataRow;
}

public GenerationContext Schema { get; }

public ISheet Sheet { get; }

public int FirstDataRowIndex { get; }

public string GetCellString(ICell? cell) => m_GetCellString(cell);

public bool ShouldValidateRow(IRow? row) => m_ShouldWriteDataRow(row);

public static string GetCellReference(int row, int column)
{
return $"{ConvertToColumnName(column)}{row + 1}";
}

private static string ConvertToColumnName(int column)
{
if (column < 0) throw new ArgumentOutOfRangeException(nameof(column));
return column < 26
? Convert.ToString((char)('A' + column))
: ConvertToColumnName(column / 26 - 1) + ConvertToColumnName(column % 26);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
using System;
using System.Collections.Generic;
using System.Linq;

namespace DataTables.GeneratorCore;

internal sealed class TableDataValidatorRegistry
{
private readonly Dictionary<string, ITableDataValidator> m_Validators;

public TableDataValidatorRegistry(IEnumerable<ITableDataValidator> validators)
{
ArgumentNullException.ThrowIfNull(validators);
m_Validators = validators.ToDictionary(x => x.DataSetType, StringComparer.OrdinalIgnoreCase);
}

public static TableDataValidatorRegistry CreateDefault()
{
return new TableDataValidatorRegistry(
[
new RowTableDataValidator(),
new NoOpTableDataValidator("matrix"),
new NoOpTableDataValidator("column"),
new NoOpTableDataValidator("kv"),
new TreeTableDataValidator(),
new GraphTableDataValidator()
]);
}

public IReadOnlyCollection<string> SupportedDataSetTypes =>
m_Validators.Keys.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToArray();

public bool TryGetValidator(string dataSetType, out ITableDataValidator validator)
{
return m_Validators.TryGetValue(dataSetType, out validator!);
}

private sealed class NoOpTableDataValidator(string dataSetType) : ITableDataValidator
{
public string DataSetType { get; } = dataSetType;

public void Validate(TableDataValidationContext context)
{
}
}
}
Loading
Loading