From 9852fb38dec1afc2c83519e55471bb6f84ac4d2f Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sat, 5 Sep 2026 19:19:20 +0900 Subject: [PATCH 1/2] Export every annotation candidate, not only the winner MS-DIAL keeps up to NUMBER_OF_ANNOTATION_RESULTS threshold-passing results per annotator, and alignment carries all of them from the representative peak into the spot. Every text export then published only the representative, so a search that could not choose between two references whose spectra are indistinguishable came out looking like a search that had chosen. A product-ion spectrum reports structure indirectly, so that situation is the common case, not the corner case. On the FastLC demo, of 1919 annotated alignment spots only 335 have a single candidate: 115 have two and 1469 have three. Four fifths of the annotations were published as unambiguous when the run had not established that. AlignResult-*.mdcandidate.tsv reports one row per spot and candidate, ranked by the same precedence that picks the representative, so rank 1 is the representative in every row of that file. It carries the annotator, the library entry, the resolved reference fields and the full score block, which is what a downstream record needs in order to state an annotation claim and the evidence behind it. The export is off by default and requested with "Annotation candidates: True" in a Console parameter file. Conventions follow the .mdprovenance.tsv sidecar rather than the .mdalign columns, because the consumer is the same audit pipeline: an empty cell wherever the run established nothing, and round-trip precision rather than display rounding. A spectral score is written only when a comparison actually happened, and that condition now lives in one place, AnnotationScoreFormat.IsComputed, which the .mdalign and .mdpeak columns already used. The five non-spectral similarity terms have no equivalent of the -1 sentinel, so an unused term is stored as 0 and cannot be told apart from a term scored 0; the file says so rather than implying a measurement. The annotator identifier is an absolute library path in a Console run, which is what .mdalign already records in its Comment column and what the parameter file records. It is kept verbatim because it is the join key back to those artifacts; database_id beside it carries the short database name for anything that should not repeat a local path. Six tests, asserting whole lines rather than substrings: the ranked order, that a reference match outranks a higher-priority suggestion, that an uncompared spectrum publishes no score, that an unannotated spot and a "set unknown" assignment contribute no row, and that an unresolvable reference leaves the reference columns empty rather than invented. Co-Authored-By: Claude Opus 5 --- .../Export/AlignmentCandidateExporter.cs | 211 ++++++++++++++++++ .../MsdialCore/Export/AnnotationCandidates.cs | 74 ++++++ .../Export/AnnotationScoreFormat.cs | 20 +- .../MsdialCoreTestApp/Parser/ConfigParser.cs | 21 ++ .../MsdialCoreTestApp/Process/LcmsProcess.cs | 15 +- .../Parser/ConfigParserTests.cs | 17 ++ .../Export/AlignmentCandidateExporterTests.cs | 204 +++++++++++++++++ 7 files changed, 556 insertions(+), 6 deletions(-) create mode 100644 src/MSDIAL5/MsdialCore/Export/AlignmentCandidateExporter.cs create mode 100644 src/MSDIAL5/MsdialCore/Export/AnnotationCandidates.cs create mode 100644 tests/MSDIAL5/MsdialCoreTests/Export/AlignmentCandidateExporterTests.cs diff --git a/src/MSDIAL5/MsdialCore/Export/AlignmentCandidateExporter.cs b/src/MSDIAL5/MsdialCore/Export/AlignmentCandidateExporter.cs new file mode 100644 index 000000000..61831ce41 --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Export/AlignmentCandidateExporter.cs @@ -0,0 +1,211 @@ +using CompMs.Common.Components; +using CompMs.Common.DataObj.Result; +using CompMs.Common.Enum; +using CompMs.MsdialCore.Algorithm.Annotation; +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Utility; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace CompMs.MsdialCore.Export; + +/// +/// Exports every annotation candidate MS-DIAL kept for an alignment spot, one row per candidate, rather +/// than only the representative one that the .mdalign columns carry. +/// +/// +/// A product-ion spectrum reports structure only indirectly, so a search often cannot choose between +/// references whose spectra are indistinguishable. MS-DIAL already keeps the alternatives, up to +/// NUMBER_OF_ANNOTATION_RESULTS per annotator, and alignment carries them from the representative peak +/// into the spot. Publishing only the winner turns "the search could not decide" into "the search +/// decided", which is the kind of unearned certainty this sidecar exists to remove. +/// +/// Conventions follow the sibling rather than the .mdalign +/// text export, because the consumer is the same audit pipeline: tab separated, UTF-8 without a BOM, an +/// empty cell wherever the run established nothing, and full round-trip precision instead of the display +/// rounding of a spreadsheet column. A score that reads 0.977 in .mdalign therefore reads 0.977049112 +/// here; they are the same measurement. +/// +public sealed class AlignmentCandidateExporter +{ + /// The value written wherever the run established nothing. + private const string NotApplicable = ""; + + private static readonly string[] Headers = [ + "alignment_master_id", + "alignment_local_id", + "parent_alignment_id", + "candidate_rank", + "candidate_count", + "is_representative", + "annotator_id", + "database_id", + "source", + "priority", + "library_id", + "name", + "formula", + "ontology", + "inchikey", + "smiles", + "reference_mz", + "reference_rt_min", + "reference_adduct", + "annotation_tag_vs1", + "is_reference_matched", + "is_annotation_suggested", + "is_precursor_mz_match", + "is_spectrum_match", + "is_spectrum_comparison_performed", + "total_score", + "mz_similarity", + "rt_similarity", + "ri_similarity", + "ccs_similarity", + "isotope_similarity", + "simple_dot_product", + "weighted_dot_product", + "reverse_dot_product", + "matched_peaks_count", + "matched_peaks_percentage", + ]; + + private readonly IMatchResultRefer? _refer; + private readonly IReadOnlyDictionary _databaseIdByAnnotator; + private readonly MachineCategory _machineCategory; + + public AlignmentCandidateExporter( + IMatchResultRefer? refer, + DataBaseStorage? databases, + MachineCategory machineCategory) + { + _refer = refer; + _databaseIdByAnnotator = databases is null + ? new Dictionary() + : AnnotationCandidates.DatabaseIdByAnnotator(databases); + _machineCategory = machineCategory; + } + + public void Export(Stream stream, IEnumerable spots) + { + using var writer = new StreamWriter(stream, new UTF8Encoding(false), 1024, leaveOpen: true); + WriteRow(writer, Headers); + foreach (var spot in Flatten(spots)) { + var candidates = AnnotationCandidates.Of(spot.MatchResults); + var representative = spot.MatchResults?.Representative; + for (int rank = 0; rank < candidates.Count; rank++) { + var candidate = candidates[rank]; + WriteCandidate(writer, spot, candidate, rank + 1, candidates.Count, ReferenceEquals(candidate, representative)); + } + } + } + + private static IEnumerable Flatten(IEnumerable spots) + { + foreach (var spot in spots) { + yield return spot; + foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? []) { + yield return driftSpot; + } + } + } + + private void WriteCandidate( + StreamWriter writer, + AlignmentSpotProperty spot, + MsScanMatchResult candidate, + int rank, + int candidateCount, + bool isRepresentative) + { + var reference = _refer?.Refer(candidate); + var hadProductIonSpectrum = spot.IsMsmsAssigned; + // One decision, shared with the .mdalign and .mdpeak score columns: a spectral score exists only + // when a product-ion spectrum was actually compared against a reference spectrum. The rendering + // differs, an empty cell here against the text "null" there, but the condition must not. + var spectrumScored = AnnotationScoreFormat.IsComputed(candidate, hadProductIonSpectrum); + WriteRow(writer, [ + spot.MasterAlignmentID.ToString(CultureInfo.InvariantCulture), + spot.AlignmentID.ToString(CultureInfo.InvariantCulture), + spot.ParentAlignmentID.ToString(CultureInfo.InvariantCulture), + rank.ToString(CultureInfo.InvariantCulture), + candidateCount.ToString(CultureInfo.InvariantCulture), + BooleanText(isRepresentative), + Sanitize(candidate.AnnotatorID), + DatabaseId(candidate), + SourceText(candidate.Source), + candidate.Priority.ToString(CultureInfo.InvariantCulture), + candidate.LibraryID.ToString(CultureInfo.InvariantCulture), + Sanitize(string.IsNullOrEmpty(candidate.Name) ? reference?.Name : candidate.Name), + Sanitize(reference?.Formula?.FormulaString), + Sanitize(string.IsNullOrEmpty(reference?.CompoundClass) ? reference?.Ontology : reference?.CompoundClass), + Sanitize(string.IsNullOrEmpty(candidate.InChIKey) ? reference?.InChIKey : candidate.InChIKey), + Sanitize(reference?.SMILES), + reference is null ? NotApplicable : Format(reference.PrecursorMz), + reference?.ChromXs is null ? NotApplicable : Format(reference.ChromXs.RT.Value), + Sanitize(reference?.AdductType?.AdductIonName), + DataAccess.GetAnnotationCode(candidate, _machineCategory).ToString(CultureInfo.InvariantCulture), + BooleanText(candidate.IsReferenceMatched), + BooleanText(candidate.IsAnnotationSuggested), + BooleanText(candidate.IsPrecursorMzMatch), + BooleanText(candidate.IsSpectrumMatch), + BooleanText(spectrumScored), + Format(candidate.TotalScore), + // The five similarity terms below have no equivalent of the -1 sentinel that marks an + // unattempted spectral comparison, so an unused term is stored as 0 and cannot be told apart + // from a term that was evaluated and scored 0. They are written as stored; a consumer that + // needs the distinction has to read which terms the run's parameter file enabled. + Format(candidate.AcurateMassSimilarity), + Format(candidate.RtSimilarity), + Format(candidate.RiSimilarity), + Format(candidate.CcsSimilarity), + Format(candidate.IsotopeSimilarity), + Score(spectrumScored, candidate.SimpleDotProduct), + Score(spectrumScored, candidate.WeightedDotProduct), + Score(spectrumScored, candidate.ReverseDotProduct), + Score(spectrumScored, candidate.MatchedPeaksCount), + Score(spectrumScored, candidate.MatchedPeaksPercentage), + ]); + } + + private string DatabaseId(MsScanMatchResult candidate) + => candidate.AnnotatorID is string id && _databaseIdByAnnotator.TryGetValue(id, out var databaseId) + ? Sanitize(databaseId) + : NotApplicable; + + /// Writes one row, refusing a field count that does not match the header. + private static void WriteRow(StreamWriter writer, string[] values) + { + if (values.Length != Headers.Length) { + throw new InvalidOperationException( + $"Alignment candidate row has {values.Length} fields but the header declares {Headers.Length}."); + } + writer.WriteLine(string.Join("\t", values)); + } + + private static string Score(bool computed, float value) + => computed ? Format(value) : NotApplicable; + + /// + /// The stored score fields are Single, and G9 round-trips a Single exactly. The widening to double is + /// deliberate and matches : .NET Framework formats a Single through + /// a 7-significant-digit intermediate, which shifts values that sit just below a rounding boundary. + /// + private static string Format(float value) + => ((double)value).ToString("G9", CultureInfo.InvariantCulture); + + private static string Format(double value) + => value.ToString("G17", CultureInfo.InvariantCulture); + + private static string SourceText(SourceType source) + => source.ToString().Replace(" ", string.Empty); + + private static string BooleanText(bool value) => value ? "true" : "false"; + + private static string Sanitize(string? value) + => (value ?? string.Empty).Replace('\t', ' ').Replace('\r', ' ').Replace('\n', ' '); +} diff --git a/src/MSDIAL5/MsdialCore/Export/AnnotationCandidates.cs b/src/MSDIAL5/MsdialCore/Export/AnnotationCandidates.cs new file mode 100644 index 000000000..ef200841a --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Export/AnnotationCandidates.cs @@ -0,0 +1,74 @@ +using CompMs.Common.DataObj.Result; +using CompMs.MsdialCore.DataObj; +using System.Collections.Generic; +using System.Linq; + +namespace CompMs.MsdialCore.Export; + +/// +/// The one definition of which annotation candidates an export publishes for a peak spot, and of the +/// database each candidate's annotator drew from. +/// +/// +/// MS-DIAL keeps up to NUMBER_OF_ANNOTATION_RESULTS threshold-passing results per annotator, but +/// every text export so far published only . The +/// alternatives were computed, carried through alignment, and then dropped at the file boundary, so a +/// reader could not tell an unambiguous match from one where the search did not choose between two +/// references it could not tell apart. +/// +/// The order is , which is the same order that picks +/// the representative: manual assignment, then reference match, then suggestion, then annotator +/// priority, then total score. A reference match with a lower annotator priority therefore still +/// outranks a higher-priority precursor-only suggestion, which is the intended scientific precedence. +/// +public static class AnnotationCandidates +{ + /// + /// The candidates of one spot or peak, best first, or an empty list when nothing was annotated. + /// + public static IReadOnlyList Of(MsScanMatchResultContainer? results) + { + if (results is null) { + return []; + } + return results.TopResults.Where(IsPublishable).ToArray(); + } + + /// + /// True for a candidate that names a reference. Decoys are already excluded upstream by + /// . + /// + /// + /// Two things are deliberately not candidates. An empty container reports a single synthetic + /// unknown result, and "set unknown" in the GUI stores a real one, both carrying + /// ; neither names a molecule. A result that matched no database at + /// all carries no evidence to publish either. A manual assignment keeps the database bit it was + /// promoted from, so it stays a candidate and simply sorts first. + /// + private static bool IsPublishable(MsScanMatchResult? result) + => result is not null && !result.IsUnknown && result.AnyMatched; + + /// + /// Maps each annotator identifier to the identifier of the database it searched. + /// + public static IReadOnlyDictionary DatabaseIdByAnnotator(DataBaseStorage storage) + { + var map = new Dictionary(); + foreach (var db in storage.MetabolomicsDataBases) { + foreach (var pair in db.Pairs) { + map.Add(pair.AnnotatorID, db.DataBaseID); + } + } + foreach (var db in storage.ProteomicsDataBases) { + foreach (var pair in db.Pairs) { + map.Add(pair.AnnotatorID, db.DataBaseID); + } + } + foreach (var db in storage.EadLipidomicsDatabases) { + foreach (var pair in db.Pairs) { + map.Add(pair.AnnotatorID, db.DataBaseID); + } + } + return map; + } +} diff --git a/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs b/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs index bfe4d1642..01e08e756 100644 --- a/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs +++ b/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs @@ -52,15 +52,27 @@ public static class AnnotationScoreFormat /// /// A numeric format string, for example "F3". public static string Score(MsScanMatchResult? result, bool hadProductIonSpectrum, Func value, string format) { - if (result is null || !result.IsSpectrumComparisonPerformed) { - return NotComputed; - } - if (!hadProductIonSpectrum && IsScoreBlockUnset(result)) { + if (!IsComputed(result, hadProductIonSpectrum)) { return NotComputed; } return value(result).ToString(format); } + /// + /// Whether the spectral score fields of hold measurements. + /// + /// + /// Exposed separately because the audit sidecars render an absent value as an empty cell rather + /// than as the text "null" that the .mdpeak and .mdalign columns use. The decision of whether a + /// score exists must not fork with the rendering, so both go through this. + /// + public static bool IsComputed(MsScanMatchResult? result, bool hadProductIonSpectrum) { + if (result is null || !result.IsSpectrumComparisonPerformed) { + return false; + } + return hadProductIonSpectrum || !IsScoreBlockUnset(result); + } + /// /// True when every spectral score field still holds the default 0, so the block carries no /// evidence that a comparison happened. A real comparison can also produce all zeros, which is diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs index d3982035c..49d16ec46 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs @@ -152,6 +152,27 @@ public static bool ReadDetailedAlignmentProvenance(string filepath) { return false; } + public static bool ReadAnnotationCandidateExport(string filepath) { + using (var sr = new StreamReader(filepath, Encoding.ASCII)) { + while (sr.Peek() > -1) { + readFieldValues(sr.ReadLine(), out string method, out string value, out bool isReadable); + if (!isReadable) { + continue; + } + switch (method.ToLower()) { + case "annotation candidates": + case "export annotation candidates": + var valueLower = value.ToLower(); + if (valueLower == "true" || valueLower == "false") { + return bool.Parse(valueLower); + } + break; + } + } + } + return false; + } + private static string ReadMspAnnotatorSettingsFilePath(string filepath) { using (var sr = new StreamReader(filepath, Encoding.ASCII)) { while (sr.Peek() > -1) { diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs index 93d0dc9d4..2c745e7a2 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs @@ -35,6 +35,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool var param = ConfigParser.ReadForLcmsParameter(methodFile); var isAlignmentLightMode = ConfigParser.ReadAlignmentLightMode(methodFile); var exportDetailedAlignmentProvenance = ConfigParser.ReadDetailedAlignmentProvenance(methodFile); + var exportAnnotationCandidates = ConfigParser.ReadAnnotationCandidateExport(methodFile); var isCorrectlyImported = CommonProcess.SetProjectProperty(param, inputFolder, out List analysisFiles, out AlignmentFileBean alignmentFile); if (!isCorrectlyImported) { return -1; @@ -97,7 +98,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool container.DataBases.SetDataBaseMapper(container.DataBaseMapper); Console.WriteLine("Start processing.."); - return ExecuteAsync(container, outputFolder, isProjectSaved, isAlignmentLightMode, exportDetailedAlignmentProvenance).Result; + return ExecuteAsync(container, outputFolder, isProjectSaved, isAlignmentLightMode, exportDetailedAlignmentProvenance, exportAnnotationCandidates).Result; } private async Task ExecuteAsync( @@ -105,7 +106,8 @@ private async Task ExecuteAsync( string outputFolder, bool isProjectSaved, bool isAlignmentLightMode, - bool exportDetailedAlignmentProvenance) { + bool exportDetailedAlignmentProvenance, + bool exportAnnotationCandidates) { var projectDataStorage = new ProjectDataStorage(new ProjectParameter(DateTime.Now, outputFolder, Path.ChangeExtension(storage.Parameter.ProjectParam.ProjectFileName, ".mdproject"))); projectDataStorage.AddStorage(storage); @@ -203,6 +205,15 @@ IQuantValueAccessor CreateQuantAccessor(string exportType) => alignmentLightPeak } Console.WriteLine($"Alignment peak ID matrix: {peakIdOutputFile}"); + if (exportAnnotationCandidates) { + var candidateOutputFile = Path.Combine(outputFolder, alignmentFile.FileName + ".mdcandidate.tsv"); + using (var candidateStream = File.Open(candidateOutputFile, FileMode.Create, FileAccess.Write)) { + new AlignmentCandidateExporter(storage.DataBaseMapper, storage.DataBases, storage.Parameter.MachineCategory) + .Export(candidateStream, result.AlignmentSpotProperties); + } + Console.WriteLine($"Annotation candidates: {candidateOutputFile}"); + } + if (exportDetailedAlignmentProvenance) { var provenanceOutputFile = Path.Combine(outputFolder, alignmentFile.FileName + ".mdprovenance.tsv"); using (var provenanceStream = File.Open(provenanceOutputFile, FileMode.Create, FileAccess.Write)) { diff --git a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs index 57c5d2143..f291cfcb7 100644 --- a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs +++ b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs @@ -136,6 +136,23 @@ public void ReadDetailedAlignmentProvenance_DefaultsToFalseAndAcceptsBothAliases Assert.IsFalse(ConfigParser.ReadDetailedAlignmentProvenance(nonBoolean)); } + [TestMethod] + public void ReadAnnotationCandidateExport_DefaultsToFalseAndAcceptsBothAliases() + { + using var directory = new TemporaryDirectory(); + var defaultMethod = directory.CreateFile("default.txt", "Ion mode: Negative\n"); + var shortAlias = directory.CreateFile("short.txt", "Annotation candidates: True\n"); + var longAlias = directory.CreateFile("long.txt", "Export annotation candidates: true\n"); + var explicitlyOff = directory.CreateFile("off.txt", "Annotation candidates: FALSE\n"); + var nonBoolean = directory.CreateFile("bad.txt", "Annotation candidates: all of them\n"); + + Assert.IsFalse(ConfigParser.ReadAnnotationCandidateExport(defaultMethod)); + Assert.IsTrue(ConfigParser.ReadAnnotationCandidateExport(shortAlias)); + Assert.IsTrue(ConfigParser.ReadAnnotationCandidateExport(longAlias)); + Assert.IsFalse(ConfigParser.ReadAnnotationCandidateExport(explicitlyOff)); + Assert.IsFalse(ConfigParser.ReadAnnotationCandidateExport(nonBoolean)); + } + private sealed class TemporaryDirectory : IDisposable { public TemporaryDirectory() diff --git a/tests/MSDIAL5/MsdialCoreTests/Export/AlignmentCandidateExporterTests.cs b/tests/MSDIAL5/MsdialCoreTests/Export/AlignmentCandidateExporterTests.cs new file mode 100644 index 000000000..53c0ed035 --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/Export/AlignmentCandidateExporterTests.cs @@ -0,0 +1,204 @@ +using CompMs.Common.Components; +using CompMs.Common.DataObj.Property; +using CompMs.Common.DataObj.Result; +using CompMs.Common.Enum; +using CompMs.MsdialCore.Algorithm.Annotation; +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Export; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace CompMs.MsdialCoreTests.Export; + +[TestClass] +public class AlignmentCandidateExporterTests +{ + // Whole-line assertions, for the same reason the provenance exporter tests use them: a substring + // assertion cannot see a ragged field count, and this file has three hand-maintained column lists. + private const string ExpectedHeader = + "alignment_master_id\talignment_local_id\tparent_alignment_id\tcandidate_rank\tcandidate_count\t" + + "is_representative\tannotator_id\tdatabase_id\tsource\tpriority\tlibrary_id\tname\tformula\t" + + "ontology\tinchikey\tsmiles\treference_mz\treference_rt_min\treference_adduct\t" + + "annotation_tag_vs1\tis_reference_matched\tis_annotation_suggested\tis_precursor_mz_match\t" + + "is_spectrum_match\tis_spectrum_comparison_performed\ttotal_score\tmz_similarity\t" + + "rt_similarity\tri_similarity\tccs_similarity\tisotope_similarity\tsimple_dot_product\t" + + "weighted_dot_product\treverse_dot_product\tmatched_peaks_count\tmatched_peaks_percentage"; + + [TestMethod] + public void EveryCandidateIsExportedOnceInRankOrder() + { + var best = ReferenceMatch("Quercetin", libraryId: 7, totalScore: 0.75f); + var alternative = ReferenceMatch("Morin", libraryId: 8, totalScore: 0.5f); + var spot = SpotWith(best, alternative); + + var lines = ExportLines(spot, Refer((7, ReferenceFor("Quercetin")), (8, ReferenceFor("Morin")))); + + Assert.AreEqual(ExpectedHeader, lines[0]); + Assert.AreEqual(3, lines.Length); + Assert.AreEqual( + "12\t10\t-1\t1\t2\ttrue\tmsp\t\tMspDB\t1\t7\tQuercetin\tC15H10O7\tflavonoid\tQUERCETIN-KEY\tc1cc(O)ccc1\t" + + "300.125\t2.5\t[M+H]+\t430\ttrue\tfalse\ttrue\ttrue\ttrue\t0.75\t0.5\t0\t0\t0\t0\t0.5\t0.25\t0.75\t12\t0.5", + lines[1]); + Assert.AreEqual( + "12\t10\t-1\t2\t2\tfalse\tmsp\t\tMspDB\t1\t8\tMorin\tC15H10O7\tflavonoid\tMORIN-KEY\tc1cc(O)ccc1\t" + + "300.125\t2.5\t[M+H]+\t430\ttrue\tfalse\ttrue\ttrue\ttrue\t0.5\t0.5\t0\t0\t0\t0\t0.5\t0.25\t0.75\t12\t0.5", + lines[2]); + } + + /// + /// The precedence that picks the representative is the precedence the file publishes: a reference + /// match outranks a precursor-only suggestion even when the suggestion's annotator has the higher + /// priority. + /// + [TestMethod] + public void AReferenceMatchOutranksAHigherPrioritySuggestion() + { + var suggestion = ReferenceMatch("Suggested", libraryId: 8, totalScore: 0.5f); + suggestion.IsReferenceMatched = false; + suggestion.IsAnnotationSuggested = true; + suggestion.IsSpectrumMatch = false; + suggestion.Priority = 9; + var match = ReferenceMatch("Matched", libraryId: 7, totalScore: 0.5f); + + var lines = ExportLines(SpotWith(suggestion, match), Refer()); + + CollectionAssert.AreEqual( + new[] { "Matched", "Suggested" }, + lines.Skip(1).Select(line => line.Split('\t')[11]).ToArray()); + CollectionAssert.AreEqual( + new[] { "true", "false" }, + lines.Skip(1).Select(line => line.Split('\t')[5]).ToArray()); + } + + /// + /// A spectral score that was never computed is an empty cell, never a 0. The distinction is the same + /// one makes for the .mdalign columns. + /// + [TestMethod] + public void AnUncomparedSpectrumPublishesNoSpectralScore() + { + var precursorOnly = ReferenceMatch("Suggested", libraryId: 7, totalScore: 0.5f); + precursorOnly.IsSpectrumMatch = false; + // The scoring functions return -1 when there was no product-ion spectrum to compare. + precursorOnly.SquaredSimpleDotProduct = -1f; + precursorOnly.SquaredWeightedDotProduct = -1f; + precursorOnly.SquaredReverseDotProduct = -1f; + precursorOnly.MatchedPeaksCount = -1f; + precursorOnly.MatchedPeaksPercentage = -1f; + + var fields = ExportLines(SpotWith(precursorOnly), Refer())[1].Split('\t'); + + Assert.AreEqual("false", fields[24], "is_spectrum_comparison_performed"); + CollectionAssert.AreEqual( + new[] { "", "", "", "", "" }, + new[] { fields[31], fields[32], fields[33], fields[34], fields[35] }); + Assert.AreEqual("0.5", fields[25], "the aggregate total score is still a measurement"); + } + + [TestMethod] + public void AnUnannotatedSpotContributesNoRow() + { + var lines = ExportLines(SpotWith(), Refer()); + + Assert.AreEqual(ExpectedHeader, lines[0]); + Assert.AreEqual(1, lines.Length); + } + + /// + /// "Set unknown" stores a real match result carrying the unknown flag. It names no molecule, so it is + /// not a candidate. + /// + [TestMethod] + public void ASetUnknownAssignmentIsNotACandidate() + { + var unknown = new MsScanMatchResult { Source = SourceType.Manual | SourceType.Unknown, }; + + var lines = ExportLines(SpotWith(unknown), Refer()); + + Assert.AreEqual(1, lines.Length); + } + + /// + /// A candidate whose reference cannot be resolved still publishes its own evidence; the columns that + /// come from the reference are empty rather than invented. + /// + [TestMethod] + public void AnUnresolvableReferenceLeavesTheReferenceColumnsEmpty() + { + var fields = ExportLines(SpotWith(ReferenceMatch("Quercetin", libraryId: 7, totalScore: 0.5f)), Refer())[1].Split('\t'); + + Assert.AreEqual("Quercetin", fields[11], "the name is on the match result itself"); + CollectionAssert.AreEqual( + new[] { "", "", "", "", "" }, + new[] { fields[12], fields[13], fields[15], fields[16], fields[17] }); + } + + private static MsScanMatchResult ReferenceMatch(string name, int libraryId, float totalScore) + => new() { + Name = name, + Source = SourceType.MspDB, + AnnotatorID = "msp", + Priority = 1, + LibraryID = libraryId, + TotalScore = totalScore, + IsReferenceMatched = true, + IsPrecursorMzMatch = true, + IsSpectrumMatch = true, + AcurateMassSimilarity = 0.5f, + SquaredSimpleDotProduct = 0.25f, + SquaredWeightedDotProduct = 0.0625f, + SquaredReverseDotProduct = 0.5625f, + MatchedPeaksCount = 12f, + MatchedPeaksPercentage = 0.5f, + }; + + private static MoleculeMsReference ReferenceFor(string name) + => new() { + Name = name, + Formula = new Formula { FormulaString = "C15H10O7", }, + Ontology = "flavonoid", + InChIKey = name.ToUpperInvariant() + "-KEY", + SMILES = "c1cc(O)ccc1", + PrecursorMz = 300.125, + ChromXs = new ChromXs(2.5), + AdductType = AdductIon.GetAdductIon("[M+H]+"), + }; + + private static AlignmentSpotProperty SpotWith(params MsScanMatchResult[] results) + { + var spot = new AlignmentSpotProperty { + MasterAlignmentID = 12, + AlignmentID = 10, + ParentAlignmentID = -1, + AlignedPeakProperties = [], + }; + spot.MatchResults.AddResults(results); + return spot; + } + + private static string[] ExportLines(AlignmentSpotProperty spot, IMatchResultRefer refer) + { + using var stream = new MemoryStream(); + new AlignmentCandidateExporter(refer, null, MachineCategory.LCMS).Export(stream, [spot]); + return Encoding.UTF8.GetString(stream.ToArray()) + .Split('\n') + .Select(line => line.TrimEnd('\r')) + .Where(line => line.Length > 0) + .ToArray(); + } + + private static IMatchResultRefer Refer(params (int LibraryId, MoleculeMsReference Reference)[] references) + => new StubRefer(references.ToDictionary(pair => pair.LibraryId, pair => pair.Reference)); + + private sealed class StubRefer(Dictionary byLibraryId) + : IMatchResultRefer + { + public string Key => "stub"; + + public MoleculeMsReference? Refer(MsScanMatchResult? result) + => result is not null && byLibraryId.TryGetValue(result.LibraryID, out var reference) ? reference : null; + } +} From 6ec3694a0ed9d07f2c4b9f2a200dec8175c22e99 Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sat, 5 Sep 2026 19:19:46 +0900 Subject: [PATCH 2/2] Say "A or B" in mzTab-M instead of asserting one answer mzTab-M already has a way to report that a search did not choose: evidence rows that share an evidence_input_id came from the same input spectrum, rank orders them, and the feature row that references them carries ambiguity code 1. The exporter wrote one evidence row per feature, hardcoded rank to 1 and the ambiguity code to null, so a file could not say what MS-DIAL had actually found. On the FastLC demo the evidence section goes from 848 rows to 2228, and 714 of the 848 features that carry an identification turn out to have alternatives the search kept and the file used to drop. One of them, alignment feature 108, matches three separate RIKEN N-VS1 records at the same nominal mass; the third is the one a different run file independently chose, which the old file could not have shown. Evidence identifiers are now assigned in a layout pass before the feature section is written. The reference set on a feature row therefore comes from the same table as the rows themselves and cannot outlive a row that was never written; the two conditions used to be spelled out separately and did not agree, the feature side omitting the MS/MS-assigned, blank-filtered and internal-standard tests. The identifiers are file-unique and consecutive within one input, which the alignment identifier they used to reuse was not: an ion-mobility drift spot is written with its parent's metadata dictionary and so reported the parent's number. For the same reason evidence_input_id now comes from the spot the row describes. A drift row and its parent are different input spectra and must not be grouped as alternatives for one another. Each row's database identifier, name, formula, SMILES, theoretical m/z and score block are resolved from its own candidate. theoretical_mass_to_charge was read from the representative's metadata dictionary under a key that only the LC and IM accessors provide, so a GC-MS file reported the number 0 for it; it is now resolved through the same reference lookup for every row, and an unresolvable reference is the null token rather than a mass of zero. spectra_ref keeps its existing rule, the member files whose own top annotation is this library entry, applied to the row's own candidate: 79 of the 1380 lower-ranked rows have one, and the rest report null because no file independently chose them. Each evidence row is now written as a line. The previous form wrote the row without a terminator and let the caller close it, which appended a trailing separator to every one of them: the built-in mzTab-M structural check reports "SME column count 27 differs from its header count 26" on all 848 rows of a current master run and reports nothing on this one. With more than one candidate per feature that form would also have run them together on a single physical line. Measured against master at 6b8e3e9d3, same demo and method file: the 14 per-file .mdpeak and .mdmsp, the .mdalign, the alignment .mdmsp, the .mdpeakid.tsv and the .qa.tsv are byte-identical. Inside the mzTab the metadata section differs only in the run-identifier line, which is a timestamp, and the summary section is byte-identical. Only two feature-row fields change, the evidence reference list and the ambiguity code, and nothing else on those rows moves. Co-Authored-By: Claude Opus 5 --- .../MsdialCore/Export/IMetadataAccessor.cs | 10 + .../MsdialCore/Export/MztabFormatExport.cs | 283 +++++++++++------- .../Export/MztabFormatExportTests.cs | 116 +++++++ .../Resources/Export/test_mztab.mzTab.txt | 6 +- 4 files changed, 307 insertions(+), 108 deletions(-) diff --git a/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs b/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs index 6eefdf1a0..bfa7a47f2 100644 --- a/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs +++ b/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs @@ -30,6 +30,16 @@ public BaseMetadataAccessor(IMatchResultRefer _parameter; + + /// + /// The reference lookup this accessor resolves the representative match result through. + /// + /// + /// Exposed for exports that describe candidates other than the representative one, which need the + /// same lookup applied to a different match result. is already read the + /// same way by the mzTab-M exporter. + /// + public IMatchResultRefer? Refer => _refer; public string[] GetHeaders() => GetHeadersCore(); IReadOnlyDictionary IMetadataAccessor.GetContent(AlignmentSpotProperty spot, IMSScanProperty msdec) { diff --git a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs index 80aa1b246..e0125080f 100644 --- a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs +++ b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs @@ -1,5 +1,7 @@ -using CompMs.Common.DataObj.Result; +using CompMs.Common.Components; +using CompMs.Common.DataObj.Result; using CompMs.Common.Enum; +using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; using CompMs.MsdialCore.MSDec; using CompMs.MsdialCore.Parameter; @@ -74,7 +76,12 @@ string outfile var exportFileName = Path.GetFileNameWithoutExtension(outfile); var mztabId = exportFileName; // as filename - var meta = (metaAccessor as BaseMetadataAccessor).Parameter; + var baseAccessor = metaAccessor as BaseMetadataAccessor; + var meta = baseAccessor.Parameter; + // The accessor resolves a reference only for the representative match result. A row for + // any other candidate needs the same lookup applied to that candidate, so the refer + // itself is borrowed rather than the already-resolved metadata. + var refer = baseAccessor.Refer; using var sw = new StreamWriter(stream, Encoding.ASCII, bufferSize: 1024, leaveOpen: true); //set common parameter @@ -123,6 +130,13 @@ string outfile sw.WriteLine(); + // The SMF section is written before the SME section but has to reference it, so the evidence + // rows are laid out first. Sourcing the references from this table rather than re-deriving + // them in the SMF writer also means a reference cannot survive a row that was never written: + // the two conditions used to be written out separately and did not agree, the SMF side + // omitting the MS/MS-assigned, blank-filtered and internal-standard tests. + var smeGroups = AssignSmeGroups(spots, meta); + //SMF section var SmfDataHeader = WriteSmfHeader(sw, meta, RawFileMetadataDic); //SMF data @@ -131,13 +145,13 @@ string outfile var metadata = metaAccessor.GetContent(spot, msdecResults[spot.MasterAlignmentID]); WriteSmfDataLine( sw, spot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, spot) ); foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { WriteSmfDataLine( sw, driftSpot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, driftSpot) ); } } @@ -151,40 +165,11 @@ string outfile ////SME data foreach (var spot in spots) { - if (spot.IsMsmsAssigned != true) { continue; } - if (spot.IsManuallyModifiedForAnnotation == true) { continue; } - if (spot.MatchResults.IsTextDbBasedRepresentative == true) { continue; } - - if (spot.Name == "") { continue; } - if (spot.IsBlankFilteredByPostCurator) { continue; } - if (meta.IsNormalizeSplash && spot.InternalStandardAlignmentID == -1) - { - continue; - } - if (meta.IsNormalizeIS && spot.InternalStandardAlignmentID == -1) - { - continue; - } - //if (analysisParamForLC.IsNormalizeSplash && splashQuant == 0 && alignedSpots[i].InternalStandardAlignmentID != -1) { return; } - //else if (analysisParamForLC.IsNormalizeSplash && splashQuant == 1 && alignedSpots[i].InternalStandardAlignmentID == -1) { return; } - - var metadata = metaAccessor.GetContent(spot, msdecResults[spot.MasterAlignmentID]); - if(metadata["Metabolite name"].Contains("no MS2")){ continue; } - - WriteSmeDataLine( - sw, spot, meta, msdecResults[spot.MasterAlignmentID], - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata - ); + WriteSmeDataLines(sw, spot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, spot)); foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { - if (!driftSpot.IsMsmsAssigned) { continue; } - - WriteSmeDataLine( - sw, driftSpot, meta, msdecResults[spot.MasterAlignmentID], - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata - ); + WriteSmeDataLines(sw, driftSpot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, driftSpot)); } - sw.WriteLine(""); } sw.WriteLine(""); } @@ -202,7 +187,12 @@ string outfile { var exportFileName = Path.GetFileNameWithoutExtension(outfile); var mztabId = exportFileName; // as filename - var meta = (metaAccessor as BaseMetadataAccessor).Parameter; + var baseAccessor = metaAccessor as BaseMetadataAccessor; + var meta = baseAccessor.Parameter; + // The accessor resolves a reference only for the representative match result. A row for + // any other candidate needs the same lookup applied to that candidate, so the refer + // itself is borrowed rather than the already-resolved metadata. + var refer = baseAccessor.Refer; using var sw = new StreamWriter(stream, Encoding.ASCII, bufferSize: 1024, leaveOpen: true); var idConfidenceMeasure = SetIdConfidenceMeasure(meta.MachineCategory, idConfidenceDefault); @@ -220,6 +210,7 @@ string outfile .ToDictionary(x => x.Key, x => x.Value); var internalStandardDic = SetStandardDic(spots); + var smeGroups = AssignSmeGroups(spots, meta); WriteMtdSection(sw, mztabId, meta, spots, RawFileMetadataDic, AnalysisFileClassDic, idConfidenceMeasure, database); sw.WriteLine(); @@ -247,8 +238,9 @@ string outfile ); WriteSmfDataLine( smfWriter, spot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, spot) ); + WriteSmeDataLines(smeWriter, spot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, spot)); foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { WriteSmlDataLine( @@ -257,28 +249,10 @@ string outfile ); WriteSmfDataLine( smfWriter, driftSpot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata - ); - } - - if (!ShouldWriteSmeLine(spot, meta, metadata)) { - continue; - } - - WriteSmeDataLine( - smeWriter, spot, meta, msdec, - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata - ); - foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) - { - if (!driftSpot.IsMsmsAssigned) { continue; } - - WriteSmeDataLine( - smeWriter, driftSpot, meta, msdec, - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, driftSpot) ); + WriteSmeDataLines(smeWriter, driftSpot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, driftSpot)); } - smeWriter.WriteLine(""); } } @@ -295,20 +269,92 @@ string outfile } } - private static bool ShouldWriteSmeLine( - AlignmentSpotProperty spot, - ParameterBase meta, - IReadOnlyDictionary metadata) { + /// + /// Whether a spot contributes evidence rows at all. + /// + /// + /// The "no MS2" test used to read the metadata dictionary, whose "Metabolite name" entry is the + /// spot name with an empty one replaced by "Unknown". Reading the spot directly is the same test + /// once the empty name is excluded above, and it lets the assignment pass run without building a + /// metadata dictionary per spot. + /// + private static bool ShouldWriteSmeLine(AlignmentSpotProperty spot, ParameterBase meta) { if (spot.IsMsmsAssigned != true) { return false; } if (spot.IsManuallyModifiedForAnnotation == true) { return false; } if (spot.MatchResults.IsTextDbBasedRepresentative == true) { return false; } - if (spot.Name == "") { return false; } + if (string.IsNullOrEmpty(spot.Name)) { return false; } if (spot.IsBlankFilteredByPostCurator) { return false; } if (meta.IsNormalizeSplash && spot.InternalStandardAlignmentID == -1) { return false; } if (meta.IsNormalizeIS && spot.InternalStandardAlignmentID == -1) { return false; } - return !metadata["Metabolite name"].Contains("no MS2"); + return !spot.Name.Contains("no MS2"); + } + + /// + /// The evidence rows one spot contributes: its annotation candidates, best first, paired with the + /// file-unique SME identifiers they will be written under. + /// + private sealed class SmeGroup + { + public SmeGroup(IReadOnlyList candidates, int firstSmeId) { + Candidates = candidates; + FirstSmeId = firstSmeId; + } + + public IReadOnlyList Candidates { get; } + + /// The identifier of the rank 1 row; the rest follow it consecutively. + public int FirstSmeId { get; } + + public int SmeId(int index) => FirstSmeId + index; + + /// + /// The mzTab-M ambiguity code for the SMF row that references this group: 1 when the group + /// holds alternative identifications of the same feature, and null when there is nothing to + /// be ambiguous about. Code 2, several evidence streams for one molecule, does not apply + /// because every row here comes from the same input spectrum. + /// + public string AmbiguityCode => Candidates.Count > 1 ? "1" : "null"; + + public string SmeIdRefs => string.Join("|", Enumerable.Range(0, Candidates.Count).Select(i => SmeId(i).ToString())); + } + + /// + /// Lays out the SME section ahead of writing, so the SMF rows can reference identifiers that are + /// guaranteed to exist and the ranks within one input spectrum are consecutive. + /// + /// + /// Keyed by spot instance rather than by alignment identifier: a drift spot of an ion-mobility run + /// is written with its parent's metadata dictionary, so the identifier alone does not distinguish + /// the two. + /// + private static Dictionary AssignSmeGroups( + IReadOnlyList spots, + ParameterBase meta) { + var groups = new Dictionary(); + var nextSmeId = 1; + foreach (var spot in spots) { + if (!ShouldWriteSmeLine(spot, meta)) { continue; } + nextSmeId = Assign(groups, spot, nextSmeId); + foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { + if (!driftSpot.IsMsmsAssigned) { continue; } + nextSmeId = Assign(groups, driftSpot, nextSmeId); + } + } + return groups; } + private static int Assign(Dictionary groups, AlignmentSpotProperty spot, int nextSmeId) { + var candidates = AnnotationCandidates.Of(spot.MatchResults); + if (candidates.Count == 0) { + return nextSmeId; + } + groups[spot] = new SmeGroup(candidates, nextSmeId); + return nextSmeId + candidates.Count; + } + + private static SmeGroup? GroupOf(Dictionary groups, AlignmentSpotProperty spot) + => groups.TryGetValue(spot, out var group) ? group : null; + private static void ReplayTemporarySection(StreamWriter sw, string path) { using var reader = new StreamReader(File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read), Encoding.ASCII); string line; @@ -452,7 +498,8 @@ private void WriteSmfDataLine( IReadOnlyDictionary AnalysisFileClassDic, IReadOnlyList SmfDataHeader, IReadOnlyDictionary internalStandardDic, - IReadOnlyDictionary metadata + IReadOnlyDictionary metadata, + SmeGroup? smeGroup ) { var smfPrefix = "SMF"; @@ -460,9 +507,10 @@ IReadOnlyDictionary metadata var matchResult = spot.MatchResults.Representative; var smfID = metadata["Alignment ID"]; - var smeIDrefs = "null"; - - var smeIDrefAmbiguity_code = "null"; + // Taken from the evidence layout rather than re-derived here, so the reference set and the + // rows that actually get written cannot disagree. + var smeIDrefs = smeGroup?.SmeIdRefs ?? "null"; + var smeIDrefAmbiguity_code = smeGroup?.AmbiguityCode ?? "null"; var isotopomer = "null"; var expMassToCharge = spot.MassCenter.ToString(); @@ -488,17 +536,6 @@ IReadOnlyDictionary metadata charge = "-" + charge; } - if (spot.Name is not null - && spot.Name != "Unknown" - && spot.Name != "null" - && spot.Name != "" - && !metadata["Metabolite name"].Contains("no MS2") - && spot.MatchResults.IsTextDbBasedRepresentative != true - && spot.IsManuallyModifiedForAnnotation != true) - { - smeIDrefs = smfID.ToString(); - } - var LineMetaData = new List() { smfPrefix,smfID.ToString(), smeIDrefs.ToString(), smeIDrefAmbiguity_code, adductIons, isotopomer, expMassToCharge, charge , retentionTime.ToString(),retentionTimeStart.ToString(),retentionTimeEnd.ToString() @@ -520,30 +557,63 @@ IReadOnlyDictionary metadata sw.WriteLine(string.Join(Separator, LineMetaData) + Separator + string.Join(Separator, LineData)); } - public void WriteSmeDataLine( + /// + /// Writes one evidence row per annotation candidate of a spot. + /// + /// + /// mzTab-M already has a way to say "A or B": evidence rows that share an evidence_input_id came + /// from the same input spectrum, rank orders them, and the feature row that references them + /// carries ambiguity code 1. MS-DIAL keeps up to NUMBER_OF_ANNOTATION_RESULTS threshold-passing + /// candidates per annotator and alignment carries them into the spot, so the alternatives existed + /// all along and were discarded at the file boundary, leaving every identification looking + /// unambiguous. Rank 1 is the representative and its columns are unchanged. + /// + private void WriteSmeDataLines( StreamWriter sw, AlignmentSpotProperty spot, ParameterBase param, - MSDecResult msdec, - IReadOnlyList database, + IMatchResultRefer? refer, + IReadOnlyDictionary RawFileMetadataDic, + IReadOnlyDictionary idConfidenceMeasure, + SmeGroup? smeGroup + ) + { + if (smeGroup is null) { + return; + } + for (int index = 0; index < smeGroup.Candidates.Count; index++) { + WriteSmeDataLine( + sw, spot, param, refer, RawFileMetadataDic, idConfidenceMeasure, + smeGroup.Candidates[index], smeGroup.SmeId(index), index + 1); + } + } + + private void WriteSmeDataLine( + StreamWriter sw, + AlignmentSpotProperty spot, + ParameterBase param, + IMatchResultRefer? refer, IReadOnlyDictionary RawFileMetadataDic, IReadOnlyDictionary idConfidenceMeasure, - IReadOnlyList analysisFiles, - IReadOnlyDictionary metadata + MsScanMatchResult candidate, + int smeID, + int rank ) { var smePrefix = "SME"; - var smeID = metadata["Alignment ID"]; - var evidenceInputID = metadata["Alignment ID"]; ; // need to consider + // The spot this row describes, not the parent whose metadata dictionary the caller reused, so + // an ion-mobility drift row is not grouped with its parent as an alternative for one input. + var evidenceInputID = spot.MasterAlignmentID; + var reference = refer?.Refer(candidate); var inchi = "null"; var uri = "null"; - var adductIons = SetAdductTypeString(metadata["Adduct type"]?.ToString() ?? "null"); + var adductIons = SetAdductTypeString(spot.AdductType?.AdductIonName ?? "null"); var expMassToCharge = spot.MassCenter.ToString(); // var derivatizedForm = "null"; var identificationMethod = idConfidenceDefault; var manualCurationScore = "null"; - if (spot.IsManuallyModifiedForAnnotation == true) + if (candidate.IsManuallyModified) { manualCurationScore = "100"; identificationMethod = idConfidenceManual; @@ -556,11 +626,17 @@ IReadOnlyDictionary metadata charge = "-" + spot.AdductType.ChargeNumber.ToString(); } - var repName = spot.MatchResults.Representative.Name.Split('|').Last(); - var repLibraryID = spot.MatchResults.Representative.LibraryID; - var chemicalFormula = metadata["Formula"]; - var smiles = metadata["SMILES"]; - var theoreticalMassToCharge = metadata.TryGetValue("Reference m/z", out var refmz) ? refmz : "0"; + var repName = (candidate.Name ?? string.Empty).Split('|').Last(); + var repLibraryID = candidate.LibraryID; + var chemicalFormula = ValueOrNull(reference?.Formula?.FormulaString); + var smiles = ValueOrNull(reference?.SMILES); + // Resolved from this candidate rather than read from the representative's metadata. A + // reference that does not resolve is the mzTab null token; it used to be the number 0, which + // is not a mass. + var theoreticalMassToCharge = reference is null ? "null" : reference.PrecursorMz.ToString("F5"); + // The member spectra whose own top annotation is this candidate. A lower-ranked alternative + // usually has none, and reports null: no file independently chose it. That the alternatives + // were scored against the same query spectrum is already stated by evidence_input_id. var spectraRefList = new List(); // multiple files if (_lightPeakStore is null) { var properties = spot.AlignedPeakProperties; @@ -591,17 +667,12 @@ IReadOnlyDictionary metadata msLevel = "[MS, MS:1000511, ms level, 2]"; } - var rank = "1"; // need to consider - - var databaseIdentifier = "null"; - var rep = spot?.MatchResults?.Representative; - if (rep != null && - rep.AnnotatorID != null && - _annotatorID2DataBaseID.TryGetValue(rep.AnnotatorID, out var databaseID) && - !string.IsNullOrEmpty(rep.Name)) + if (candidate.AnnotatorID != null && + _annotatorID2DataBaseID.TryGetValue(candidate.AnnotatorID, out var databaseID) && + !string.IsNullOrEmpty(candidate.Name)) { - databaseIdentifier = _annotatorID2DataBaseID[rep.AnnotatorID!] + ":" + rep.Name.Split('|').Last(); + databaseIdentifier = databaseID + ":" + candidate.Name.Split('|').Last(); } var SmeLine = new List() { @@ -610,11 +681,13 @@ IReadOnlyDictionary metadata spectraRef, identificationMethod, msLevel }; - SmeLine.AddRange(SetExportScoreList(idConfidenceMeasure, spot.MatchResults.Representative, manualCurationScore)); - - SmeLine.Add(rank); - sw.Write(String.Join("\t", SmeLine.Select(item => string.IsNullOrEmpty(item) ? "null" : item).ToList()) + "\t"); + SmeLine.AddRange(SetExportScoreList(idConfidenceMeasure, candidate, manualCurationScore)); + SmeLine.Add(rank.ToString()); + // One line per evidence row. The previous form wrote the row without a terminator and let the + // caller close it, which appended a trailing separator to every line and would have run the + // candidates of one spot together on a single physical line. + sw.WriteLine(String.Join(" ", SmeLine.Select(item => string.IsNullOrEmpty(item) ? "null" : item).ToList())); } private static void AddSpectraRef(List spectraRefList, int msRunID, int ms1ScanID, int ms2ScanID) { diff --git a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs index 1a7ec9522..645726187 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs @@ -54,6 +54,122 @@ public async Task MztabFormatExporterTest() { CollectionAssert.AreEqual(buffer, stream.GetBuffer()); } + /// + /// A spot whose search kept more than one candidate publishes them all as ranked evidence rows of + /// one input spectrum, which is how mzTab-M says "A or B". + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task AnIndistinguishableAlternativeIsPublishedAsASecondRankedEvidenceRow() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + + var spot = container.AlignmentSpotProperties.First(s => s.Name == "Quercetin"); + var alternative = spot.MatchResults.Representative.Clone(); + alternative.Name = "Morin"; + alternative.LibraryID = spot.MatchResults.Representative.LibraryID + 1000; + // Strictly lower on the last term of the ordering only, so the representative does not change + // and the alternative is unambiguously rank 2. + alternative.TotalScore = spot.MatchResults.Representative.TotalScore - 0.1f; + spot.MatchResults.AddResult(alternative); + + var lines = ExportToLines(storage, container, msdecs); + + var evidence = lines.Where(line => line.StartsWith("SME\t")).Select(line => line.Split('\t')).ToArray(); + var forThisSpot = evidence.Where(f => f[2] == spot.MasterAlignmentID.ToString()).ToArray(); + Assert.AreEqual(2, forThisSpot.Length, "both candidates should be published"); + CollectionAssert.AreEqual(new[] { "Quercetin", "Morin" }, forThisSpot.Select(f => f[7]).ToArray()); + CollectionAssert.AreEqual(new[] { "1", "2" }, forThisSpot.Select(f => f[f.Length - 1]).ToArray()); + Assert.AreEqual("small_test_msp:Morin", forThisSpot[1][3]); + + // The rows carry consecutive, file-unique identifiers and the feature row references both. + var smeIds = forThisSpot.Select(f => f[1]).ToArray(); + Assert.AreEqual(int.Parse(smeIds[0]) + 1, int.Parse(smeIds[1])); + var feature = lines.Select(line => line.Split('\t')) + .Single(f => f[0] == "SMF" && f[1] == spot.MasterAlignmentID.ToString()); + Assert.AreEqual(string.Join("|", smeIds), feature[2]); + Assert.AreEqual("1", feature[3], "two alternatives for one feature is ambiguity code 1"); + } + + /// + /// One candidate is not an ambiguity, so the feature row leaves the code null. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ASingleCandidateLeavesTheAmbiguityCodeNull() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var spot = container.AlignmentSpotProperties.First(s => s.Name == "Quercetin"); + + var lines = ExportToLines(storage, container, msdecs); + + var feature = lines.Select(line => line.Split('\t')) + .Single(f => f[0] == "SMF" && f[1] == spot.MasterAlignmentID.ToString()); + Assert.AreEqual("null", feature[3]); + Assert.IsFalse(feature[2].Contains("|"), "a single reference needs no separator"); + } + + /// + /// A feature row must not reference an evidence row that was never written. The reference set and + /// the rows come from the same layout pass, so this holds for every row of the file. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task EveryFeatureReferenceResolvesToAnEvidenceRow() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + + var lines = ExportToLines(storage, container, msdecs); + var rows = lines.Select(line => line.Split('\t')).ToArray(); + + var written = rows.Where(f => f[0] == "SME").Select(f => f[1]).ToArray(); + var referenced = rows.Where(f => f[0] == "SMF" && f[2] != "null").SelectMany(f => f[2].Split('|')).ToArray(); + CollectionAssert.AreEquivalent(written, referenced); + CollectionAssert.AllItemsAreUnique(written); + } + + private static async Task<(IMsdialDataStorage, AlignmentResultContainer, List)> LoadExportFixtureAsync() { + IMsdialDataStorage storage; + using (var streamManager = new DirectoryTreeStreamManager("./Resources/Export")) { + storage = await MsdialDataStorage.Serializer.LoadAsync(streamManager, "Dataset_2025_07_31_12_31_11.mddata", "", ""); + storage.FixDatasetFolder("./Resources/Export"); + } + var alignmentFile = storage.AlignmentFiles.Last(); + var container = AlignmentResultContainer.Load(alignmentFile); + var loader = new MSDec.MSDecLoader(alignmentFile.SpectraFilePath, []); + return (storage, container, loader.LoadMSDecResults()); + } + + private static string[] ExportToLines( + IMsdialDataStorage storage, + AlignmentResultContainer container, + List msdecs) { + var exporter = new MztabFormatExporter(storage.DataBases); + using var stream = new MemoryStream(); + exporter.MztabFormatExporterCore( + stream, + [.. container.AlignmentSpotProperties], + msdecs, + storage.AnalysisFiles, + new StubMetadataAccessor(storage.DataBaseMapper, storage.Parameter), + new LegacyQuantValueAccessor("Height", storage.Parameter), + [StatsValue.Average, StatsValue.Stdev,], + "mztab_test"); + return System.Text.Encoding.ASCII.GetString(stream.ToArray()).Split('\n').Select(line => line.TrimEnd('\r')).ToArray(); + } + [TestMethod()] //[DeploymentItem(@"Resources\Export\small_test_project.mdproject", @"Resources\Export")] diff --git a/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt b/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt index 044a96251..4d0a977e6 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt +++ b/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt @@ -63,7 +63,7 @@ SMF 0 null null [M+H]1+ null 300.1852722167969 1 115.687 115.687 115.687 1017 17 SMF 1 1 null [M+H]1+ null 301.0346374511719 1 119.6405 118.444 120.83699999999999 12642 20826 SMF 2 null null [M+H]1+ null 302.0978088378906 1 118.857 118.857 118.857 5881 15691 SMF 3 null null [M+H]1+ null 302.1385192871094 1 118.804 118.804 118.804 1285 2241 -SMF 4 4 null [M+H]1+ null 303.0500183105469 1 118.8305 118.62400000000001 119.037 536230 1837469 +SMF 4 2 null [M+H]1+ null 303.0500183105469 1 118.8305 118.62400000000001 119.037 536230 1837469 SMF 5 null null [M+H]1+ null 303.12347412109375 1 119.397 119.397 119.397 314 5489 SMF 6 null null [M+H]1+ null 303.3472595214844 1 118.677 118.677 118.677 479 1459 SMF 7 null null [M+H]1+ null 303.7848205566406 1 118.4705 118.084 118.857 1451 10300 @@ -73,6 +73,6 @@ SMF 10 null null [M+H]1+ null 304.62640380859375 1 118.857 118.857 118.857 268 1 SMF 11 null null [M+H]1+ null 304.8061828613281 1 119.037 119.037 119.037 384 1553 SEH SME_ID evidence_input_id database_identifier chemical_formula smiles inchi chemical_name uri derivatized_form adduct_ion exp_mass_to_charge charge theoretical_mass_to_charge spectra_ref identification_method ms_level id_confidence_measure[1] id_confidence_measure[2] id_confidence_measure[3] id_confidence_measure[4] id_confidence_measure[5] id_confidence_measure[6] id_confidence_measure[7] id_confidence_measure[8] rank -SME 1 1 small_test_msp:Isodemethylwedelolactone C15H8O7 O=C1OC=2C(O)=CC(O)=CC2C=3OC=4C=C(O)C(O)=CC4C13 null Isodemethylwedelolactone null null [M+H]1+ 301.0346374511719 1 0 ms_run[2]:ms1scanID=3490 ms2scanID=3488 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.5242083 0 0.8979989 0.646961 0.69420624 0.88054246 4 0.6666667 1 -SME 4 4 small_test_msp:Quercetin C15H10O7 C1=CC(=C(C=C1C2=C(C(=O)C3=C(C=C(C=C3O2)O)O)O)O)O null Quercetin null null [M+H]1+ 303.0500183105469 1 0 ms_run[1]:ms1scanID=3510 ms2scanID=3468| ms_run[2]:ms1scanID=3440 ms2scanID=3418 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.6437206 0 0.9999581 0.6832075 0.7282375 0.8728807 15 0.88235295 1 +SME 1 1 small_test_msp:Isodemethylwedelolactone C15H8O7 O=C1OC=2C(O)=CC(O)=CC2C=3OC=4C=C(O)C(O)=CC4C13 null Isodemethylwedelolactone null null [M+H]1+ 301.0346374511719 1 301.03000 ms_run[2]:ms1scanID=3490 ms2scanID=3488 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.5242083 0 0.8979989 0.646961 0.69420624 0.88054246 4 0.6666667 1 +SME 2 4 small_test_msp:Quercetin C15H10O7 C1=CC(=C(C=C1C2=C(C(=O)C3=C(C=C(C=C3O2)O)O)O)O)O null Quercetin null null [M+H]1+ 303.0500183105469 1 303.04993 ms_run[1]:ms1scanID=3510 ms2scanID=3468| ms_run[2]:ms1scanID=3440 ms2scanID=3418 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.6437206 0 0.9999581 0.6832075 0.7282375 0.8728807 15 0.88235295 1