diff --git a/src/MSDIAL5/MsdialCore/Export/AlignmentCandidateExporter.cs b/src/MSDIAL5/MsdialCore/Export/AlignmentCandidateExporter.cs new file mode 100644 index 000000000..61831ce41 --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Export/AlignmentCandidateExporter.cs @@ -0,0 +1,211 @@ +using CompMs.Common.Components; +using CompMs.Common.DataObj.Result; +using CompMs.Common.Enum; +using CompMs.MsdialCore.Algorithm.Annotation; +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Utility; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace CompMs.MsdialCore.Export; + +/// +/// Exports every annotation candidate MS-DIAL kept for an alignment spot, one row per candidate, rather +/// than only the representative one that the .mdalign columns carry. +/// +/// +/// A product-ion spectrum reports structure only indirectly, so a search often cannot choose between +/// references whose spectra are indistinguishable. MS-DIAL already keeps the alternatives, up to +/// NUMBER_OF_ANNOTATION_RESULTS per annotator, and alignment carries them from the representative peak +/// into the spot. Publishing only the winner turns "the search could not decide" into "the search +/// decided", which is the kind of unearned certainty this sidecar exists to remove. +/// +/// Conventions follow the sibling rather than the .mdalign +/// text export, because the consumer is the same audit pipeline: tab separated, UTF-8 without a BOM, an +/// empty cell wherever the run established nothing, and full round-trip precision instead of the display +/// rounding of a spreadsheet column. A score that reads 0.977 in .mdalign therefore reads 0.977049112 +/// here; they are the same measurement. +/// +public sealed class AlignmentCandidateExporter +{ + /// The value written wherever the run established nothing. + private const string NotApplicable = ""; + + private static readonly string[] Headers = [ + "alignment_master_id", + "alignment_local_id", + "parent_alignment_id", + "candidate_rank", + "candidate_count", + "is_representative", + "annotator_id", + "database_id", + "source", + "priority", + "library_id", + "name", + "formula", + "ontology", + "inchikey", + "smiles", + "reference_mz", + "reference_rt_min", + "reference_adduct", + "annotation_tag_vs1", + "is_reference_matched", + "is_annotation_suggested", + "is_precursor_mz_match", + "is_spectrum_match", + "is_spectrum_comparison_performed", + "total_score", + "mz_similarity", + "rt_similarity", + "ri_similarity", + "ccs_similarity", + "isotope_similarity", + "simple_dot_product", + "weighted_dot_product", + "reverse_dot_product", + "matched_peaks_count", + "matched_peaks_percentage", + ]; + + private readonly IMatchResultRefer? _refer; + private readonly IReadOnlyDictionary _databaseIdByAnnotator; + private readonly MachineCategory _machineCategory; + + public AlignmentCandidateExporter( + IMatchResultRefer? refer, + DataBaseStorage? databases, + MachineCategory machineCategory) + { + _refer = refer; + _databaseIdByAnnotator = databases is null + ? new Dictionary() + : AnnotationCandidates.DatabaseIdByAnnotator(databases); + _machineCategory = machineCategory; + } + + public void Export(Stream stream, IEnumerable spots) + { + using var writer = new StreamWriter(stream, new UTF8Encoding(false), 1024, leaveOpen: true); + WriteRow(writer, Headers); + foreach (var spot in Flatten(spots)) { + var candidates = AnnotationCandidates.Of(spot.MatchResults); + var representative = spot.MatchResults?.Representative; + for (int rank = 0; rank < candidates.Count; rank++) { + var candidate = candidates[rank]; + WriteCandidate(writer, spot, candidate, rank + 1, candidates.Count, ReferenceEquals(candidate, representative)); + } + } + } + + private static IEnumerable Flatten(IEnumerable spots) + { + foreach (var spot in spots) { + yield return spot; + foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? []) { + yield return driftSpot; + } + } + } + + private void WriteCandidate( + StreamWriter writer, + AlignmentSpotProperty spot, + MsScanMatchResult candidate, + int rank, + int candidateCount, + bool isRepresentative) + { + var reference = _refer?.Refer(candidate); + var hadProductIonSpectrum = spot.IsMsmsAssigned; + // One decision, shared with the .mdalign and .mdpeak score columns: a spectral score exists only + // when a product-ion spectrum was actually compared against a reference spectrum. The rendering + // differs, an empty cell here against the text "null" there, but the condition must not. + var spectrumScored = AnnotationScoreFormat.IsComputed(candidate, hadProductIonSpectrum); + WriteRow(writer, [ + spot.MasterAlignmentID.ToString(CultureInfo.InvariantCulture), + spot.AlignmentID.ToString(CultureInfo.InvariantCulture), + spot.ParentAlignmentID.ToString(CultureInfo.InvariantCulture), + rank.ToString(CultureInfo.InvariantCulture), + candidateCount.ToString(CultureInfo.InvariantCulture), + BooleanText(isRepresentative), + Sanitize(candidate.AnnotatorID), + DatabaseId(candidate), + SourceText(candidate.Source), + candidate.Priority.ToString(CultureInfo.InvariantCulture), + candidate.LibraryID.ToString(CultureInfo.InvariantCulture), + Sanitize(string.IsNullOrEmpty(candidate.Name) ? reference?.Name : candidate.Name), + Sanitize(reference?.Formula?.FormulaString), + Sanitize(string.IsNullOrEmpty(reference?.CompoundClass) ? reference?.Ontology : reference?.CompoundClass), + Sanitize(string.IsNullOrEmpty(candidate.InChIKey) ? reference?.InChIKey : candidate.InChIKey), + Sanitize(reference?.SMILES), + reference is null ? NotApplicable : Format(reference.PrecursorMz), + reference?.ChromXs is null ? NotApplicable : Format(reference.ChromXs.RT.Value), + Sanitize(reference?.AdductType?.AdductIonName), + DataAccess.GetAnnotationCode(candidate, _machineCategory).ToString(CultureInfo.InvariantCulture), + BooleanText(candidate.IsReferenceMatched), + BooleanText(candidate.IsAnnotationSuggested), + BooleanText(candidate.IsPrecursorMzMatch), + BooleanText(candidate.IsSpectrumMatch), + BooleanText(spectrumScored), + Format(candidate.TotalScore), + // The five similarity terms below have no equivalent of the -1 sentinel that marks an + // unattempted spectral comparison, so an unused term is stored as 0 and cannot be told apart + // from a term that was evaluated and scored 0. They are written as stored; a consumer that + // needs the distinction has to read which terms the run's parameter file enabled. + Format(candidate.AcurateMassSimilarity), + Format(candidate.RtSimilarity), + Format(candidate.RiSimilarity), + Format(candidate.CcsSimilarity), + Format(candidate.IsotopeSimilarity), + Score(spectrumScored, candidate.SimpleDotProduct), + Score(spectrumScored, candidate.WeightedDotProduct), + Score(spectrumScored, candidate.ReverseDotProduct), + Score(spectrumScored, candidate.MatchedPeaksCount), + Score(spectrumScored, candidate.MatchedPeaksPercentage), + ]); + } + + private string DatabaseId(MsScanMatchResult candidate) + => candidate.AnnotatorID is string id && _databaseIdByAnnotator.TryGetValue(id, out var databaseId) + ? Sanitize(databaseId) + : NotApplicable; + + /// Writes one row, refusing a field count that does not match the header. + private static void WriteRow(StreamWriter writer, string[] values) + { + if (values.Length != Headers.Length) { + throw new InvalidOperationException( + $"Alignment candidate row has {values.Length} fields but the header declares {Headers.Length}."); + } + writer.WriteLine(string.Join("\t", values)); + } + + private static string Score(bool computed, float value) + => computed ? Format(value) : NotApplicable; + + /// + /// The stored score fields are Single, and G9 round-trips a Single exactly. The widening to double is + /// deliberate and matches : .NET Framework formats a Single through + /// a 7-significant-digit intermediate, which shifts values that sit just below a rounding boundary. + /// + private static string Format(float value) + => ((double)value).ToString("G9", CultureInfo.InvariantCulture); + + private static string Format(double value) + => value.ToString("G17", CultureInfo.InvariantCulture); + + private static string SourceText(SourceType source) + => source.ToString().Replace(" ", string.Empty); + + private static string BooleanText(bool value) => value ? "true" : "false"; + + private static string Sanitize(string? value) + => (value ?? string.Empty).Replace('\t', ' ').Replace('\r', ' ').Replace('\n', ' '); +} diff --git a/src/MSDIAL5/MsdialCore/Export/AnnotationCandidates.cs b/src/MSDIAL5/MsdialCore/Export/AnnotationCandidates.cs new file mode 100644 index 000000000..ef200841a --- /dev/null +++ b/src/MSDIAL5/MsdialCore/Export/AnnotationCandidates.cs @@ -0,0 +1,74 @@ +using CompMs.Common.DataObj.Result; +using CompMs.MsdialCore.DataObj; +using System.Collections.Generic; +using System.Linq; + +namespace CompMs.MsdialCore.Export; + +/// +/// The one definition of which annotation candidates an export publishes for a peak spot, and of the +/// database each candidate's annotator drew from. +/// +/// +/// MS-DIAL keeps up to NUMBER_OF_ANNOTATION_RESULTS threshold-passing results per annotator, but +/// every text export so far published only . The +/// alternatives were computed, carried through alignment, and then dropped at the file boundary, so a +/// reader could not tell an unambiguous match from one where the search did not choose between two +/// references it could not tell apart. +/// +/// The order is , which is the same order that picks +/// the representative: manual assignment, then reference match, then suggestion, then annotator +/// priority, then total score. A reference match with a lower annotator priority therefore still +/// outranks a higher-priority precursor-only suggestion, which is the intended scientific precedence. +/// +public static class AnnotationCandidates +{ + /// + /// The candidates of one spot or peak, best first, or an empty list when nothing was annotated. + /// + public static IReadOnlyList Of(MsScanMatchResultContainer? results) + { + if (results is null) { + return []; + } + return results.TopResults.Where(IsPublishable).ToArray(); + } + + /// + /// True for a candidate that names a reference. Decoys are already excluded upstream by + /// . + /// + /// + /// Two things are deliberately not candidates. An empty container reports a single synthetic + /// unknown result, and "set unknown" in the GUI stores a real one, both carrying + /// ; neither names a molecule. A result that matched no database at + /// all carries no evidence to publish either. A manual assignment keeps the database bit it was + /// promoted from, so it stays a candidate and simply sorts first. + /// + private static bool IsPublishable(MsScanMatchResult? result) + => result is not null && !result.IsUnknown && result.AnyMatched; + + /// + /// Maps each annotator identifier to the identifier of the database it searched. + /// + public static IReadOnlyDictionary DatabaseIdByAnnotator(DataBaseStorage storage) + { + var map = new Dictionary(); + foreach (var db in storage.MetabolomicsDataBases) { + foreach (var pair in db.Pairs) { + map.Add(pair.AnnotatorID, db.DataBaseID); + } + } + foreach (var db in storage.ProteomicsDataBases) { + foreach (var pair in db.Pairs) { + map.Add(pair.AnnotatorID, db.DataBaseID); + } + } + foreach (var db in storage.EadLipidomicsDatabases) { + foreach (var pair in db.Pairs) { + map.Add(pair.AnnotatorID, db.DataBaseID); + } + } + return map; + } +} diff --git a/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs b/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs index bfe4d1642..01e08e756 100644 --- a/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs +++ b/src/MSDIAL5/MsdialCore/Export/AnnotationScoreFormat.cs @@ -52,15 +52,27 @@ public static class AnnotationScoreFormat /// /// A numeric format string, for example "F3". public static string Score(MsScanMatchResult? result, bool hadProductIonSpectrum, Func value, string format) { - if (result is null || !result.IsSpectrumComparisonPerformed) { - return NotComputed; - } - if (!hadProductIonSpectrum && IsScoreBlockUnset(result)) { + if (!IsComputed(result, hadProductIonSpectrum)) { return NotComputed; } return value(result).ToString(format); } + /// + /// Whether the spectral score fields of hold measurements. + /// + /// + /// Exposed separately because the audit sidecars render an absent value as an empty cell rather + /// than as the text "null" that the .mdpeak and .mdalign columns use. The decision of whether a + /// score exists must not fork with the rendering, so both go through this. + /// + public static bool IsComputed(MsScanMatchResult? result, bool hadProductIonSpectrum) { + if (result is null || !result.IsSpectrumComparisonPerformed) { + return false; + } + return hadProductIonSpectrum || !IsScoreBlockUnset(result); + } + /// /// True when every spectral score field still holds the default 0, so the block carries no /// evidence that a comparison happened. A real comparison can also produce all zeros, which is diff --git a/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs b/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs index 6eefdf1a0..bfa7a47f2 100644 --- a/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs +++ b/src/MSDIAL5/MsdialCore/Export/IMetadataAccessor.cs @@ -30,6 +30,16 @@ public BaseMetadataAccessor(IMatchResultRefer _parameter; + + /// + /// The reference lookup this accessor resolves the representative match result through. + /// + /// + /// Exposed for exports that describe candidates other than the representative one, which need the + /// same lookup applied to a different match result. is already read the + /// same way by the mzTab-M exporter. + /// + public IMatchResultRefer? Refer => _refer; public string[] GetHeaders() => GetHeadersCore(); IReadOnlyDictionary IMetadataAccessor.GetContent(AlignmentSpotProperty spot, IMSScanProperty msdec) { diff --git a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs index 80aa1b246..e0125080f 100644 --- a/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs +++ b/src/MSDIAL5/MsdialCore/Export/MztabFormatExport.cs @@ -1,5 +1,7 @@ -using CompMs.Common.DataObj.Result; +using CompMs.Common.Components; +using CompMs.Common.DataObj.Result; using CompMs.Common.Enum; +using CompMs.MsdialCore.Algorithm.Annotation; using CompMs.MsdialCore.DataObj; using CompMs.MsdialCore.MSDec; using CompMs.MsdialCore.Parameter; @@ -74,7 +76,12 @@ string outfile var exportFileName = Path.GetFileNameWithoutExtension(outfile); var mztabId = exportFileName; // as filename - var meta = (metaAccessor as BaseMetadataAccessor).Parameter; + var baseAccessor = metaAccessor as BaseMetadataAccessor; + var meta = baseAccessor.Parameter; + // The accessor resolves a reference only for the representative match result. A row for + // any other candidate needs the same lookup applied to that candidate, so the refer + // itself is borrowed rather than the already-resolved metadata. + var refer = baseAccessor.Refer; using var sw = new StreamWriter(stream, Encoding.ASCII, bufferSize: 1024, leaveOpen: true); //set common parameter @@ -123,6 +130,13 @@ string outfile sw.WriteLine(); + // The SMF section is written before the SME section but has to reference it, so the evidence + // rows are laid out first. Sourcing the references from this table rather than re-deriving + // them in the SMF writer also means a reference cannot survive a row that was never written: + // the two conditions used to be written out separately and did not agree, the SMF side + // omitting the MS/MS-assigned, blank-filtered and internal-standard tests. + var smeGroups = AssignSmeGroups(spots, meta); + //SMF section var SmfDataHeader = WriteSmfHeader(sw, meta, RawFileMetadataDic); //SMF data @@ -131,13 +145,13 @@ string outfile var metadata = metaAccessor.GetContent(spot, msdecResults[spot.MasterAlignmentID]); WriteSmfDataLine( sw, spot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, spot) ); foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { WriteSmfDataLine( sw, driftSpot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, driftSpot) ); } } @@ -151,40 +165,11 @@ string outfile ////SME data foreach (var spot in spots) { - if (spot.IsMsmsAssigned != true) { continue; } - if (spot.IsManuallyModifiedForAnnotation == true) { continue; } - if (spot.MatchResults.IsTextDbBasedRepresentative == true) { continue; } - - if (spot.Name == "") { continue; } - if (spot.IsBlankFilteredByPostCurator) { continue; } - if (meta.IsNormalizeSplash && spot.InternalStandardAlignmentID == -1) - { - continue; - } - if (meta.IsNormalizeIS && spot.InternalStandardAlignmentID == -1) - { - continue; - } - //if (analysisParamForLC.IsNormalizeSplash && splashQuant == 0 && alignedSpots[i].InternalStandardAlignmentID != -1) { return; } - //else if (analysisParamForLC.IsNormalizeSplash && splashQuant == 1 && alignedSpots[i].InternalStandardAlignmentID == -1) { return; } - - var metadata = metaAccessor.GetContent(spot, msdecResults[spot.MasterAlignmentID]); - if(metadata["Metabolite name"].Contains("no MS2")){ continue; } - - WriteSmeDataLine( - sw, spot, meta, msdecResults[spot.MasterAlignmentID], - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata - ); + WriteSmeDataLines(sw, spot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, spot)); foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { - if (!driftSpot.IsMsmsAssigned) { continue; } - - WriteSmeDataLine( - sw, driftSpot, meta, msdecResults[spot.MasterAlignmentID], - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata - ); + WriteSmeDataLines(sw, driftSpot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, driftSpot)); } - sw.WriteLine(""); } sw.WriteLine(""); } @@ -202,7 +187,12 @@ string outfile { var exportFileName = Path.GetFileNameWithoutExtension(outfile); var mztabId = exportFileName; // as filename - var meta = (metaAccessor as BaseMetadataAccessor).Parameter; + var baseAccessor = metaAccessor as BaseMetadataAccessor; + var meta = baseAccessor.Parameter; + // The accessor resolves a reference only for the representative match result. A row for + // any other candidate needs the same lookup applied to that candidate, so the refer + // itself is borrowed rather than the already-resolved metadata. + var refer = baseAccessor.Refer; using var sw = new StreamWriter(stream, Encoding.ASCII, bufferSize: 1024, leaveOpen: true); var idConfidenceMeasure = SetIdConfidenceMeasure(meta.MachineCategory, idConfidenceDefault); @@ -220,6 +210,7 @@ string outfile .ToDictionary(x => x.Key, x => x.Value); var internalStandardDic = SetStandardDic(spots); + var smeGroups = AssignSmeGroups(spots, meta); WriteMtdSection(sw, mztabId, meta, spots, RawFileMetadataDic, AnalysisFileClassDic, idConfidenceMeasure, database); sw.WriteLine(); @@ -247,8 +238,9 @@ string outfile ); WriteSmfDataLine( smfWriter, spot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, spot) ); + WriteSmeDataLines(smeWriter, spot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, spot)); foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { WriteSmlDataLine( @@ -257,28 +249,10 @@ string outfile ); WriteSmfDataLine( smfWriter, driftSpot, meta, quantAccessor, stats, RawFileMetadataDic, AnalysisFileClassDic, - SmfDataHeader, internalStandardDic, metadata - ); - } - - if (!ShouldWriteSmeLine(spot, meta, metadata)) { - continue; - } - - WriteSmeDataLine( - smeWriter, spot, meta, msdec, - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata - ); - foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) - { - if (!driftSpot.IsMsmsAssigned) { continue; } - - WriteSmeDataLine( - smeWriter, driftSpot, meta, msdec, - database, RawFileMetadataDic, idConfidenceMeasure, files, metadata + SmfDataHeader, internalStandardDic, metadata, GroupOf(smeGroups, driftSpot) ); + WriteSmeDataLines(smeWriter, driftSpot, meta, refer, RawFileMetadataDic, idConfidenceMeasure, GroupOf(smeGroups, driftSpot)); } - smeWriter.WriteLine(""); } } @@ -295,20 +269,92 @@ string outfile } } - private static bool ShouldWriteSmeLine( - AlignmentSpotProperty spot, - ParameterBase meta, - IReadOnlyDictionary metadata) { + /// + /// Whether a spot contributes evidence rows at all. + /// + /// + /// The "no MS2" test used to read the metadata dictionary, whose "Metabolite name" entry is the + /// spot name with an empty one replaced by "Unknown". Reading the spot directly is the same test + /// once the empty name is excluded above, and it lets the assignment pass run without building a + /// metadata dictionary per spot. + /// + private static bool ShouldWriteSmeLine(AlignmentSpotProperty spot, ParameterBase meta) { if (spot.IsMsmsAssigned != true) { return false; } if (spot.IsManuallyModifiedForAnnotation == true) { return false; } if (spot.MatchResults.IsTextDbBasedRepresentative == true) { return false; } - if (spot.Name == "") { return false; } + if (string.IsNullOrEmpty(spot.Name)) { return false; } if (spot.IsBlankFilteredByPostCurator) { return false; } if (meta.IsNormalizeSplash && spot.InternalStandardAlignmentID == -1) { return false; } if (meta.IsNormalizeIS && spot.InternalStandardAlignmentID == -1) { return false; } - return !metadata["Metabolite name"].Contains("no MS2"); + return !spot.Name.Contains("no MS2"); + } + + /// + /// The evidence rows one spot contributes: its annotation candidates, best first, paired with the + /// file-unique SME identifiers they will be written under. + /// + private sealed class SmeGroup + { + public SmeGroup(IReadOnlyList candidates, int firstSmeId) { + Candidates = candidates; + FirstSmeId = firstSmeId; + } + + public IReadOnlyList Candidates { get; } + + /// The identifier of the rank 1 row; the rest follow it consecutively. + public int FirstSmeId { get; } + + public int SmeId(int index) => FirstSmeId + index; + + /// + /// The mzTab-M ambiguity code for the SMF row that references this group: 1 when the group + /// holds alternative identifications of the same feature, and null when there is nothing to + /// be ambiguous about. Code 2, several evidence streams for one molecule, does not apply + /// because every row here comes from the same input spectrum. + /// + public string AmbiguityCode => Candidates.Count > 1 ? "1" : "null"; + + public string SmeIdRefs => string.Join("|", Enumerable.Range(0, Candidates.Count).Select(i => SmeId(i).ToString())); + } + + /// + /// Lays out the SME section ahead of writing, so the SMF rows can reference identifiers that are + /// guaranteed to exist and the ranks within one input spectrum are consecutive. + /// + /// + /// Keyed by spot instance rather than by alignment identifier: a drift spot of an ion-mobility run + /// is written with its parent's metadata dictionary, so the identifier alone does not distinguish + /// the two. + /// + private static Dictionary AssignSmeGroups( + IReadOnlyList spots, + ParameterBase meta) { + var groups = new Dictionary(); + var nextSmeId = 1; + foreach (var spot in spots) { + if (!ShouldWriteSmeLine(spot, meta)) { continue; } + nextSmeId = Assign(groups, spot, nextSmeId); + foreach (var driftSpot in spot.AlignmentDriftSpotFeatures ?? Enumerable.Empty()) { + if (!driftSpot.IsMsmsAssigned) { continue; } + nextSmeId = Assign(groups, driftSpot, nextSmeId); + } + } + return groups; } + private static int Assign(Dictionary groups, AlignmentSpotProperty spot, int nextSmeId) { + var candidates = AnnotationCandidates.Of(spot.MatchResults); + if (candidates.Count == 0) { + return nextSmeId; + } + groups[spot] = new SmeGroup(candidates, nextSmeId); + return nextSmeId + candidates.Count; + } + + private static SmeGroup? GroupOf(Dictionary groups, AlignmentSpotProperty spot) + => groups.TryGetValue(spot, out var group) ? group : null; + private static void ReplayTemporarySection(StreamWriter sw, string path) { using var reader = new StreamReader(File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read), Encoding.ASCII); string line; @@ -452,7 +498,8 @@ private void WriteSmfDataLine( IReadOnlyDictionary AnalysisFileClassDic, IReadOnlyList SmfDataHeader, IReadOnlyDictionary internalStandardDic, - IReadOnlyDictionary metadata + IReadOnlyDictionary metadata, + SmeGroup? smeGroup ) { var smfPrefix = "SMF"; @@ -460,9 +507,10 @@ IReadOnlyDictionary metadata var matchResult = spot.MatchResults.Representative; var smfID = metadata["Alignment ID"]; - var smeIDrefs = "null"; - - var smeIDrefAmbiguity_code = "null"; + // Taken from the evidence layout rather than re-derived here, so the reference set and the + // rows that actually get written cannot disagree. + var smeIDrefs = smeGroup?.SmeIdRefs ?? "null"; + var smeIDrefAmbiguity_code = smeGroup?.AmbiguityCode ?? "null"; var isotopomer = "null"; var expMassToCharge = spot.MassCenter.ToString(); @@ -488,17 +536,6 @@ IReadOnlyDictionary metadata charge = "-" + charge; } - if (spot.Name is not null - && spot.Name != "Unknown" - && spot.Name != "null" - && spot.Name != "" - && !metadata["Metabolite name"].Contains("no MS2") - && spot.MatchResults.IsTextDbBasedRepresentative != true - && spot.IsManuallyModifiedForAnnotation != true) - { - smeIDrefs = smfID.ToString(); - } - var LineMetaData = new List() { smfPrefix,smfID.ToString(), smeIDrefs.ToString(), smeIDrefAmbiguity_code, adductIons, isotopomer, expMassToCharge, charge , retentionTime.ToString(),retentionTimeStart.ToString(),retentionTimeEnd.ToString() @@ -520,30 +557,63 @@ IReadOnlyDictionary metadata sw.WriteLine(string.Join(Separator, LineMetaData) + Separator + string.Join(Separator, LineData)); } - public void WriteSmeDataLine( + /// + /// Writes one evidence row per annotation candidate of a spot. + /// + /// + /// mzTab-M already has a way to say "A or B": evidence rows that share an evidence_input_id came + /// from the same input spectrum, rank orders them, and the feature row that references them + /// carries ambiguity code 1. MS-DIAL keeps up to NUMBER_OF_ANNOTATION_RESULTS threshold-passing + /// candidates per annotator and alignment carries them into the spot, so the alternatives existed + /// all along and were discarded at the file boundary, leaving every identification looking + /// unambiguous. Rank 1 is the representative and its columns are unchanged. + /// + private void WriteSmeDataLines( StreamWriter sw, AlignmentSpotProperty spot, ParameterBase param, - MSDecResult msdec, - IReadOnlyList database, + IMatchResultRefer? refer, + IReadOnlyDictionary RawFileMetadataDic, + IReadOnlyDictionary idConfidenceMeasure, + SmeGroup? smeGroup + ) + { + if (smeGroup is null) { + return; + } + for (int index = 0; index < smeGroup.Candidates.Count; index++) { + WriteSmeDataLine( + sw, spot, param, refer, RawFileMetadataDic, idConfidenceMeasure, + smeGroup.Candidates[index], smeGroup.SmeId(index), index + 1); + } + } + + private void WriteSmeDataLine( + StreamWriter sw, + AlignmentSpotProperty spot, + ParameterBase param, + IMatchResultRefer? refer, IReadOnlyDictionary RawFileMetadataDic, IReadOnlyDictionary idConfidenceMeasure, - IReadOnlyList analysisFiles, - IReadOnlyDictionary metadata + MsScanMatchResult candidate, + int smeID, + int rank ) { var smePrefix = "SME"; - var smeID = metadata["Alignment ID"]; - var evidenceInputID = metadata["Alignment ID"]; ; // need to consider + // The spot this row describes, not the parent whose metadata dictionary the caller reused, so + // an ion-mobility drift row is not grouped with its parent as an alternative for one input. + var evidenceInputID = spot.MasterAlignmentID; + var reference = refer?.Refer(candidate); var inchi = "null"; var uri = "null"; - var adductIons = SetAdductTypeString(metadata["Adduct type"]?.ToString() ?? "null"); + var adductIons = SetAdductTypeString(spot.AdductType?.AdductIonName ?? "null"); var expMassToCharge = spot.MassCenter.ToString(); // var derivatizedForm = "null"; var identificationMethod = idConfidenceDefault; var manualCurationScore = "null"; - if (spot.IsManuallyModifiedForAnnotation == true) + if (candidate.IsManuallyModified) { manualCurationScore = "100"; identificationMethod = idConfidenceManual; @@ -556,11 +626,17 @@ IReadOnlyDictionary metadata charge = "-" + spot.AdductType.ChargeNumber.ToString(); } - var repName = spot.MatchResults.Representative.Name.Split('|').Last(); - var repLibraryID = spot.MatchResults.Representative.LibraryID; - var chemicalFormula = metadata["Formula"]; - var smiles = metadata["SMILES"]; - var theoreticalMassToCharge = metadata.TryGetValue("Reference m/z", out var refmz) ? refmz : "0"; + var repName = (candidate.Name ?? string.Empty).Split('|').Last(); + var repLibraryID = candidate.LibraryID; + var chemicalFormula = ValueOrNull(reference?.Formula?.FormulaString); + var smiles = ValueOrNull(reference?.SMILES); + // Resolved from this candidate rather than read from the representative's metadata. A + // reference that does not resolve is the mzTab null token; it used to be the number 0, which + // is not a mass. + var theoreticalMassToCharge = reference is null ? "null" : reference.PrecursorMz.ToString("F5"); + // The member spectra whose own top annotation is this candidate. A lower-ranked alternative + // usually has none, and reports null: no file independently chose it. That the alternatives + // were scored against the same query spectrum is already stated by evidence_input_id. var spectraRefList = new List(); // multiple files if (_lightPeakStore is null) { var properties = spot.AlignedPeakProperties; @@ -591,17 +667,12 @@ IReadOnlyDictionary metadata msLevel = "[MS, MS:1000511, ms level, 2]"; } - var rank = "1"; // need to consider - - var databaseIdentifier = "null"; - var rep = spot?.MatchResults?.Representative; - if (rep != null && - rep.AnnotatorID != null && - _annotatorID2DataBaseID.TryGetValue(rep.AnnotatorID, out var databaseID) && - !string.IsNullOrEmpty(rep.Name)) + if (candidate.AnnotatorID != null && + _annotatorID2DataBaseID.TryGetValue(candidate.AnnotatorID, out var databaseID) && + !string.IsNullOrEmpty(candidate.Name)) { - databaseIdentifier = _annotatorID2DataBaseID[rep.AnnotatorID!] + ":" + rep.Name.Split('|').Last(); + databaseIdentifier = databaseID + ":" + candidate.Name.Split('|').Last(); } var SmeLine = new List() { @@ -610,11 +681,13 @@ IReadOnlyDictionary metadata spectraRef, identificationMethod, msLevel }; - SmeLine.AddRange(SetExportScoreList(idConfidenceMeasure, spot.MatchResults.Representative, manualCurationScore)); - - SmeLine.Add(rank); - sw.Write(String.Join("\t", SmeLine.Select(item => string.IsNullOrEmpty(item) ? "null" : item).ToList()) + "\t"); + SmeLine.AddRange(SetExportScoreList(idConfidenceMeasure, candidate, manualCurationScore)); + SmeLine.Add(rank.ToString()); + // One line per evidence row. The previous form wrote the row without a terminator and let the + // caller close it, which appended a trailing separator to every line and would have run the + // candidates of one spot together on a single physical line. + sw.WriteLine(String.Join(" ", SmeLine.Select(item => string.IsNullOrEmpty(item) ? "null" : item).ToList())); } private static void AddSpectraRef(List spectraRefList, int msRunID, int ms1ScanID, int ms2ScanID) { diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs index d3982035c..49d16ec46 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs @@ -152,6 +152,27 @@ public static bool ReadDetailedAlignmentProvenance(string filepath) { return false; } + public static bool ReadAnnotationCandidateExport(string filepath) { + using (var sr = new StreamReader(filepath, Encoding.ASCII)) { + while (sr.Peek() > -1) { + readFieldValues(sr.ReadLine(), out string method, out string value, out bool isReadable); + if (!isReadable) { + continue; + } + switch (method.ToLower()) { + case "annotation candidates": + case "export annotation candidates": + var valueLower = value.ToLower(); + if (valueLower == "true" || valueLower == "false") { + return bool.Parse(valueLower); + } + break; + } + } + } + return false; + } + private static string ReadMspAnnotatorSettingsFilePath(string filepath) { using (var sr = new StreamReader(filepath, Encoding.ASCII)) { while (sr.Peek() > -1) { diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs index 93d0dc9d4..2c745e7a2 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs @@ -35,6 +35,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool var param = ConfigParser.ReadForLcmsParameter(methodFile); var isAlignmentLightMode = ConfigParser.ReadAlignmentLightMode(methodFile); var exportDetailedAlignmentProvenance = ConfigParser.ReadDetailedAlignmentProvenance(methodFile); + var exportAnnotationCandidates = ConfigParser.ReadAnnotationCandidateExport(methodFile); var isCorrectlyImported = CommonProcess.SetProjectProperty(param, inputFolder, out List analysisFiles, out AlignmentFileBean alignmentFile); if (!isCorrectlyImported) { return -1; @@ -97,7 +98,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool container.DataBases.SetDataBaseMapper(container.DataBaseMapper); Console.WriteLine("Start processing.."); - return ExecuteAsync(container, outputFolder, isProjectSaved, isAlignmentLightMode, exportDetailedAlignmentProvenance).Result; + return ExecuteAsync(container, outputFolder, isProjectSaved, isAlignmentLightMode, exportDetailedAlignmentProvenance, exportAnnotationCandidates).Result; } private async Task ExecuteAsync( @@ -105,7 +106,8 @@ private async Task ExecuteAsync( string outputFolder, bool isProjectSaved, bool isAlignmentLightMode, - bool exportDetailedAlignmentProvenance) { + bool exportDetailedAlignmentProvenance, + bool exportAnnotationCandidates) { var projectDataStorage = new ProjectDataStorage(new ProjectParameter(DateTime.Now, outputFolder, Path.ChangeExtension(storage.Parameter.ProjectParam.ProjectFileName, ".mdproject"))); projectDataStorage.AddStorage(storage); @@ -203,6 +205,15 @@ IQuantValueAccessor CreateQuantAccessor(string exportType) => alignmentLightPeak } Console.WriteLine($"Alignment peak ID matrix: {peakIdOutputFile}"); + if (exportAnnotationCandidates) { + var candidateOutputFile = Path.Combine(outputFolder, alignmentFile.FileName + ".mdcandidate.tsv"); + using (var candidateStream = File.Open(candidateOutputFile, FileMode.Create, FileAccess.Write)) { + new AlignmentCandidateExporter(storage.DataBaseMapper, storage.DataBases, storage.Parameter.MachineCategory) + .Export(candidateStream, result.AlignmentSpotProperties); + } + Console.WriteLine($"Annotation candidates: {candidateOutputFile}"); + } + if (exportDetailedAlignmentProvenance) { var provenanceOutputFile = Path.Combine(outputFolder, alignmentFile.FileName + ".mdprovenance.tsv"); using (var provenanceStream = File.Open(provenanceOutputFile, FileMode.Create, FileAccess.Write)) { diff --git a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs index 57c5d2143..f291cfcb7 100644 --- a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs +++ b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs @@ -136,6 +136,23 @@ public void ReadDetailedAlignmentProvenance_DefaultsToFalseAndAcceptsBothAliases Assert.IsFalse(ConfigParser.ReadDetailedAlignmentProvenance(nonBoolean)); } + [TestMethod] + public void ReadAnnotationCandidateExport_DefaultsToFalseAndAcceptsBothAliases() + { + using var directory = new TemporaryDirectory(); + var defaultMethod = directory.CreateFile("default.txt", "Ion mode: Negative\n"); + var shortAlias = directory.CreateFile("short.txt", "Annotation candidates: True\n"); + var longAlias = directory.CreateFile("long.txt", "Export annotation candidates: true\n"); + var explicitlyOff = directory.CreateFile("off.txt", "Annotation candidates: FALSE\n"); + var nonBoolean = directory.CreateFile("bad.txt", "Annotation candidates: all of them\n"); + + Assert.IsFalse(ConfigParser.ReadAnnotationCandidateExport(defaultMethod)); + Assert.IsTrue(ConfigParser.ReadAnnotationCandidateExport(shortAlias)); + Assert.IsTrue(ConfigParser.ReadAnnotationCandidateExport(longAlias)); + Assert.IsFalse(ConfigParser.ReadAnnotationCandidateExport(explicitlyOff)); + Assert.IsFalse(ConfigParser.ReadAnnotationCandidateExport(nonBoolean)); + } + private sealed class TemporaryDirectory : IDisposable { public TemporaryDirectory() diff --git a/tests/MSDIAL5/MsdialCoreTests/Export/AlignmentCandidateExporterTests.cs b/tests/MSDIAL5/MsdialCoreTests/Export/AlignmentCandidateExporterTests.cs new file mode 100644 index 000000000..53c0ed035 --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTests/Export/AlignmentCandidateExporterTests.cs @@ -0,0 +1,204 @@ +using CompMs.Common.Components; +using CompMs.Common.DataObj.Property; +using CompMs.Common.DataObj.Result; +using CompMs.Common.Enum; +using CompMs.MsdialCore.Algorithm.Annotation; +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Export; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace CompMs.MsdialCoreTests.Export; + +[TestClass] +public class AlignmentCandidateExporterTests +{ + // Whole-line assertions, for the same reason the provenance exporter tests use them: a substring + // assertion cannot see a ragged field count, and this file has three hand-maintained column lists. + private const string ExpectedHeader = + "alignment_master_id\talignment_local_id\tparent_alignment_id\tcandidate_rank\tcandidate_count\t" + + "is_representative\tannotator_id\tdatabase_id\tsource\tpriority\tlibrary_id\tname\tformula\t" + + "ontology\tinchikey\tsmiles\treference_mz\treference_rt_min\treference_adduct\t" + + "annotation_tag_vs1\tis_reference_matched\tis_annotation_suggested\tis_precursor_mz_match\t" + + "is_spectrum_match\tis_spectrum_comparison_performed\ttotal_score\tmz_similarity\t" + + "rt_similarity\tri_similarity\tccs_similarity\tisotope_similarity\tsimple_dot_product\t" + + "weighted_dot_product\treverse_dot_product\tmatched_peaks_count\tmatched_peaks_percentage"; + + [TestMethod] + public void EveryCandidateIsExportedOnceInRankOrder() + { + var best = ReferenceMatch("Quercetin", libraryId: 7, totalScore: 0.75f); + var alternative = ReferenceMatch("Morin", libraryId: 8, totalScore: 0.5f); + var spot = SpotWith(best, alternative); + + var lines = ExportLines(spot, Refer((7, ReferenceFor("Quercetin")), (8, ReferenceFor("Morin")))); + + Assert.AreEqual(ExpectedHeader, lines[0]); + Assert.AreEqual(3, lines.Length); + Assert.AreEqual( + "12\t10\t-1\t1\t2\ttrue\tmsp\t\tMspDB\t1\t7\tQuercetin\tC15H10O7\tflavonoid\tQUERCETIN-KEY\tc1cc(O)ccc1\t" + + "300.125\t2.5\t[M+H]+\t430\ttrue\tfalse\ttrue\ttrue\ttrue\t0.75\t0.5\t0\t0\t0\t0\t0.5\t0.25\t0.75\t12\t0.5", + lines[1]); + Assert.AreEqual( + "12\t10\t-1\t2\t2\tfalse\tmsp\t\tMspDB\t1\t8\tMorin\tC15H10O7\tflavonoid\tMORIN-KEY\tc1cc(O)ccc1\t" + + "300.125\t2.5\t[M+H]+\t430\ttrue\tfalse\ttrue\ttrue\ttrue\t0.5\t0.5\t0\t0\t0\t0\t0.5\t0.25\t0.75\t12\t0.5", + lines[2]); + } + + /// + /// The precedence that picks the representative is the precedence the file publishes: a reference + /// match outranks a precursor-only suggestion even when the suggestion's annotator has the higher + /// priority. + /// + [TestMethod] + public void AReferenceMatchOutranksAHigherPrioritySuggestion() + { + var suggestion = ReferenceMatch("Suggested", libraryId: 8, totalScore: 0.5f); + suggestion.IsReferenceMatched = false; + suggestion.IsAnnotationSuggested = true; + suggestion.IsSpectrumMatch = false; + suggestion.Priority = 9; + var match = ReferenceMatch("Matched", libraryId: 7, totalScore: 0.5f); + + var lines = ExportLines(SpotWith(suggestion, match), Refer()); + + CollectionAssert.AreEqual( + new[] { "Matched", "Suggested" }, + lines.Skip(1).Select(line => line.Split('\t')[11]).ToArray()); + CollectionAssert.AreEqual( + new[] { "true", "false" }, + lines.Skip(1).Select(line => line.Split('\t')[5]).ToArray()); + } + + /// + /// A spectral score that was never computed is an empty cell, never a 0. The distinction is the same + /// one makes for the .mdalign columns. + /// + [TestMethod] + public void AnUncomparedSpectrumPublishesNoSpectralScore() + { + var precursorOnly = ReferenceMatch("Suggested", libraryId: 7, totalScore: 0.5f); + precursorOnly.IsSpectrumMatch = false; + // The scoring functions return -1 when there was no product-ion spectrum to compare. + precursorOnly.SquaredSimpleDotProduct = -1f; + precursorOnly.SquaredWeightedDotProduct = -1f; + precursorOnly.SquaredReverseDotProduct = -1f; + precursorOnly.MatchedPeaksCount = -1f; + precursorOnly.MatchedPeaksPercentage = -1f; + + var fields = ExportLines(SpotWith(precursorOnly), Refer())[1].Split('\t'); + + Assert.AreEqual("false", fields[24], "is_spectrum_comparison_performed"); + CollectionAssert.AreEqual( + new[] { "", "", "", "", "" }, + new[] { fields[31], fields[32], fields[33], fields[34], fields[35] }); + Assert.AreEqual("0.5", fields[25], "the aggregate total score is still a measurement"); + } + + [TestMethod] + public void AnUnannotatedSpotContributesNoRow() + { + var lines = ExportLines(SpotWith(), Refer()); + + Assert.AreEqual(ExpectedHeader, lines[0]); + Assert.AreEqual(1, lines.Length); + } + + /// + /// "Set unknown" stores a real match result carrying the unknown flag. It names no molecule, so it is + /// not a candidate. + /// + [TestMethod] + public void ASetUnknownAssignmentIsNotACandidate() + { + var unknown = new MsScanMatchResult { Source = SourceType.Manual | SourceType.Unknown, }; + + var lines = ExportLines(SpotWith(unknown), Refer()); + + Assert.AreEqual(1, lines.Length); + } + + /// + /// A candidate whose reference cannot be resolved still publishes its own evidence; the columns that + /// come from the reference are empty rather than invented. + /// + [TestMethod] + public void AnUnresolvableReferenceLeavesTheReferenceColumnsEmpty() + { + var fields = ExportLines(SpotWith(ReferenceMatch("Quercetin", libraryId: 7, totalScore: 0.5f)), Refer())[1].Split('\t'); + + Assert.AreEqual("Quercetin", fields[11], "the name is on the match result itself"); + CollectionAssert.AreEqual( + new[] { "", "", "", "", "" }, + new[] { fields[12], fields[13], fields[15], fields[16], fields[17] }); + } + + private static MsScanMatchResult ReferenceMatch(string name, int libraryId, float totalScore) + => new() { + Name = name, + Source = SourceType.MspDB, + AnnotatorID = "msp", + Priority = 1, + LibraryID = libraryId, + TotalScore = totalScore, + IsReferenceMatched = true, + IsPrecursorMzMatch = true, + IsSpectrumMatch = true, + AcurateMassSimilarity = 0.5f, + SquaredSimpleDotProduct = 0.25f, + SquaredWeightedDotProduct = 0.0625f, + SquaredReverseDotProduct = 0.5625f, + MatchedPeaksCount = 12f, + MatchedPeaksPercentage = 0.5f, + }; + + private static MoleculeMsReference ReferenceFor(string name) + => new() { + Name = name, + Formula = new Formula { FormulaString = "C15H10O7", }, + Ontology = "flavonoid", + InChIKey = name.ToUpperInvariant() + "-KEY", + SMILES = "c1cc(O)ccc1", + PrecursorMz = 300.125, + ChromXs = new ChromXs(2.5), + AdductType = AdductIon.GetAdductIon("[M+H]+"), + }; + + private static AlignmentSpotProperty SpotWith(params MsScanMatchResult[] results) + { + var spot = new AlignmentSpotProperty { + MasterAlignmentID = 12, + AlignmentID = 10, + ParentAlignmentID = -1, + AlignedPeakProperties = [], + }; + spot.MatchResults.AddResults(results); + return spot; + } + + private static string[] ExportLines(AlignmentSpotProperty spot, IMatchResultRefer refer) + { + using var stream = new MemoryStream(); + new AlignmentCandidateExporter(refer, null, MachineCategory.LCMS).Export(stream, [spot]); + return Encoding.UTF8.GetString(stream.ToArray()) + .Split('\n') + .Select(line => line.TrimEnd('\r')) + .Where(line => line.Length > 0) + .ToArray(); + } + + private static IMatchResultRefer Refer(params (int LibraryId, MoleculeMsReference Reference)[] references) + => new StubRefer(references.ToDictionary(pair => pair.LibraryId, pair => pair.Reference)); + + private sealed class StubRefer(Dictionary byLibraryId) + : IMatchResultRefer + { + public string Key => "stub"; + + public MoleculeMsReference? Refer(MsScanMatchResult? result) + => result is not null && byLibraryId.TryGetValue(result.LibraryID, out var reference) ? reference : null; + } +} diff --git a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs index 1a7ec9522..645726187 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Export/MztabFormatExportTests.cs @@ -54,6 +54,122 @@ public async Task MztabFormatExporterTest() { CollectionAssert.AreEqual(buffer, stream.GetBuffer()); } + /// + /// A spot whose search kept more than one candidate publishes them all as ranked evidence rows of + /// one input spectrum, which is how mzTab-M says "A or B". + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task AnIndistinguishableAlternativeIsPublishedAsASecondRankedEvidenceRow() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + + var spot = container.AlignmentSpotProperties.First(s => s.Name == "Quercetin"); + var alternative = spot.MatchResults.Representative.Clone(); + alternative.Name = "Morin"; + alternative.LibraryID = spot.MatchResults.Representative.LibraryID + 1000; + // Strictly lower on the last term of the ordering only, so the representative does not change + // and the alternative is unambiguously rank 2. + alternative.TotalScore = spot.MatchResults.Representative.TotalScore - 0.1f; + spot.MatchResults.AddResult(alternative); + + var lines = ExportToLines(storage, container, msdecs); + + var evidence = lines.Where(line => line.StartsWith("SME\t")).Select(line => line.Split('\t')).ToArray(); + var forThisSpot = evidence.Where(f => f[2] == spot.MasterAlignmentID.ToString()).ToArray(); + Assert.AreEqual(2, forThisSpot.Length, "both candidates should be published"); + CollectionAssert.AreEqual(new[] { "Quercetin", "Morin" }, forThisSpot.Select(f => f[7]).ToArray()); + CollectionAssert.AreEqual(new[] { "1", "2" }, forThisSpot.Select(f => f[f.Length - 1]).ToArray()); + Assert.AreEqual("small_test_msp:Morin", forThisSpot[1][3]); + + // The rows carry consecutive, file-unique identifiers and the feature row references both. + var smeIds = forThisSpot.Select(f => f[1]).ToArray(); + Assert.AreEqual(int.Parse(smeIds[0]) + 1, int.Parse(smeIds[1])); + var feature = lines.Select(line => line.Split('\t')) + .Single(f => f[0] == "SMF" && f[1] == spot.MasterAlignmentID.ToString()); + Assert.AreEqual(string.Join("|", smeIds), feature[2]); + Assert.AreEqual("1", feature[3], "two alternatives for one feature is ambiguity code 1"); + } + + /// + /// One candidate is not an ambiguity, so the feature row leaves the code null. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task ASingleCandidateLeavesTheAmbiguityCodeNull() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + var spot = container.AlignmentSpotProperties.First(s => s.Name == "Quercetin"); + + var lines = ExportToLines(storage, container, msdecs); + + var feature = lines.Select(line => line.Split('\t')) + .Single(f => f[0] == "SMF" && f[1] == spot.MasterAlignmentID.ToString()); + Assert.AreEqual("null", feature[3]); + Assert.IsFalse(feature[2].Contains("|"), "a single reference needs no separator"); + } + + /// + /// A feature row must not reference an evidence row that was never written. The reference set and + /// the rows come from the same layout pass, so this holds for every row of the file. + /// + [TestMethod()] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11.mddata", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\Dataset_2025_07_31_12_31_11_Loaded.msp2.dbs", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.arf2", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06_PeakProperties.arf", @"Resources\Export")] + [DeploymentItem(@"Resources\Export\AlignmentResult_2025_07_31_12_33_06.dcl", @"Resources\Export")] + public async Task EveryFeatureReferenceResolvesToAnEvidenceRow() { + var (storage, container, msdecs) = await LoadExportFixtureAsync(); + + var lines = ExportToLines(storage, container, msdecs); + var rows = lines.Select(line => line.Split('\t')).ToArray(); + + var written = rows.Where(f => f[0] == "SME").Select(f => f[1]).ToArray(); + var referenced = rows.Where(f => f[0] == "SMF" && f[2] != "null").SelectMany(f => f[2].Split('|')).ToArray(); + CollectionAssert.AreEquivalent(written, referenced); + CollectionAssert.AllItemsAreUnique(written); + } + + private static async Task<(IMsdialDataStorage, AlignmentResultContainer, List)> LoadExportFixtureAsync() { + IMsdialDataStorage storage; + using (var streamManager = new DirectoryTreeStreamManager("./Resources/Export")) { + storage = await MsdialDataStorage.Serializer.LoadAsync(streamManager, "Dataset_2025_07_31_12_31_11.mddata", "", ""); + storage.FixDatasetFolder("./Resources/Export"); + } + var alignmentFile = storage.AlignmentFiles.Last(); + var container = AlignmentResultContainer.Load(alignmentFile); + var loader = new MSDec.MSDecLoader(alignmentFile.SpectraFilePath, []); + return (storage, container, loader.LoadMSDecResults()); + } + + private static string[] ExportToLines( + IMsdialDataStorage storage, + AlignmentResultContainer container, + List msdecs) { + var exporter = new MztabFormatExporter(storage.DataBases); + using var stream = new MemoryStream(); + exporter.MztabFormatExporterCore( + stream, + [.. container.AlignmentSpotProperties], + msdecs, + storage.AnalysisFiles, + new StubMetadataAccessor(storage.DataBaseMapper, storage.Parameter), + new LegacyQuantValueAccessor("Height", storage.Parameter), + [StatsValue.Average, StatsValue.Stdev,], + "mztab_test"); + return System.Text.Encoding.ASCII.GetString(stream.ToArray()).Split('\n').Select(line => line.TrimEnd('\r')).ToArray(); + } + [TestMethod()] //[DeploymentItem(@"Resources\Export\small_test_project.mdproject", @"Resources\Export")] diff --git a/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt b/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt index 044a96251..4d0a977e6 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt +++ b/tests/MSDIAL5/MsdialCoreTests/Resources/Export/test_mztab.mzTab.txt @@ -63,7 +63,7 @@ SMF 0 null null [M+H]1+ null 300.1852722167969 1 115.687 115.687 115.687 1017 17 SMF 1 1 null [M+H]1+ null 301.0346374511719 1 119.6405 118.444 120.83699999999999 12642 20826 SMF 2 null null [M+H]1+ null 302.0978088378906 1 118.857 118.857 118.857 5881 15691 SMF 3 null null [M+H]1+ null 302.1385192871094 1 118.804 118.804 118.804 1285 2241 -SMF 4 4 null [M+H]1+ null 303.0500183105469 1 118.8305 118.62400000000001 119.037 536230 1837469 +SMF 4 2 null [M+H]1+ null 303.0500183105469 1 118.8305 118.62400000000001 119.037 536230 1837469 SMF 5 null null [M+H]1+ null 303.12347412109375 1 119.397 119.397 119.397 314 5489 SMF 6 null null [M+H]1+ null 303.3472595214844 1 118.677 118.677 118.677 479 1459 SMF 7 null null [M+H]1+ null 303.7848205566406 1 118.4705 118.084 118.857 1451 10300 @@ -73,6 +73,6 @@ SMF 10 null null [M+H]1+ null 304.62640380859375 1 118.857 118.857 118.857 268 1 SMF 11 null null [M+H]1+ null 304.8061828613281 1 119.037 119.037 119.037 384 1553 SEH SME_ID evidence_input_id database_identifier chemical_formula smiles inchi chemical_name uri derivatized_form adduct_ion exp_mass_to_charge charge theoretical_mass_to_charge spectra_ref identification_method ms_level id_confidence_measure[1] id_confidence_measure[2] id_confidence_measure[3] id_confidence_measure[4] id_confidence_measure[5] id_confidence_measure[6] id_confidence_measure[7] id_confidence_measure[8] rank -SME 1 1 small_test_msp:Isodemethylwedelolactone C15H8O7 O=C1OC=2C(O)=CC(O)=CC2C=3OC=4C=C(O)C(O)=CC4C13 null Isodemethylwedelolactone null null [M+H]1+ 301.0346374511719 1 0 ms_run[2]:ms1scanID=3490 ms2scanID=3488 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.5242083 0 0.8979989 0.646961 0.69420624 0.88054246 4 0.6666667 1 -SME 4 4 small_test_msp:Quercetin C15H10O7 C1=CC(=C(C=C1C2=C(C(=O)C3=C(C=C(C=C3O2)O)O)O)O)O null Quercetin null null [M+H]1+ 303.0500183105469 1 0 ms_run[1]:ms1scanID=3510 ms2scanID=3468| ms_run[2]:ms1scanID=3440 ms2scanID=3418 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.6437206 0 0.9999581 0.6832075 0.7282375 0.8728807 15 0.88235295 1 +SME 1 1 small_test_msp:Isodemethylwedelolactone C15H8O7 O=C1OC=2C(O)=CC(O)=CC2C=3OC=4C=C(O)C(O)=CC4C13 null Isodemethylwedelolactone null null [M+H]1+ 301.0346374511719 1 301.03000 ms_run[2]:ms1scanID=3490 ms2scanID=3488 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.5242083 0 0.8979989 0.646961 0.69420624 0.88054246 4 0.6666667 1 +SME 2 4 small_test_msp:Quercetin C15H10O7 C1=CC(=C(C=C1C2=C(C(=O)C3=C(C=C(C=C3O2)O)O)O)O)O null Quercetin null null [M+H]1+ 303.0500183105469 1 303.04993 ms_run[1]:ms1scanID=3510 ms2scanID=3468| ms_run[2]:ms1scanID=3440 ms2scanID=3418 [,, MS-DIAL algorithm matching score, ] [MS, MS:1000511, ms level, 2] 1.6437206 0 0.9999581 0.6832075 0.7282375 0.8728807 15 0.88235295 1