options and metadata

Author: KSemenenko

AGENTS.md

@@ -11,6 +11,7 @@ If I tell you to remember something, you do the same, update
 ## Rules to follow
 - MIME handling: always use `ManagedCode.MimeTypes` for MIME constants, lookups, and validation logic.
 - Treat this repository as a high-fidelity port of `microsoft-markitdown`: every test fixture copied from the upstream `tests/test_files/` directory must be referenced by .NET tests (either as positive conversions or explicit unsupported cases). No orphaned fixtures.
+- CSV parsing must use the `Sep` library; avoid Sylvan or other CSV parsers for new or updated code.
 
 # Repository Guidelines
 

README.md

@@ -377,6 +377,7 @@ Console.WriteLine(urlResult.Title);
 ### Customise the pipeline with options
 
 ```csharp
+using System;
 using Azure;
 using MarkItDown;
 
@@ -387,6 +388,14 @@ var options = new MarkItDownOptions
         await myCaptionService.DescribeAsync(bytes, info, token),
     AudioTranscriber = async (bytes, info, token) =>
         await speechClient.TranscribeAsync(bytes, info, token),
+    Segments = new SegmentOptions
+    {
+        IncludeSegmentMetadataInMarkdown = true,
+        Audio = new AudioSegmentOptions
+        {
+            SegmentDuration = TimeSpan.FromMinutes(2)
+        }
+    },
     DocumentIntelligence = new DocumentIntelligenceOptions
     {
         Endpoint = "https://<your-resource>.cognitiveservices.azure.com/",
@@ -395,6 +404,8 @@ var options = new MarkItDownOptions
 };
 
 var markItDown = new MarkItDown(options);
+
+// Segments are still available programmatically even when annotations are disabled.
 ```
 
 ### Custom converters
@@ -682,7 +693,7 @@ public class DocumentConversionFunction
 
 - **`MarkItDown`** - Main entry point for conversions
 - **`IDocumentConverter`** - Interface for format-specific converters
-- **`DocumentConverterResult`** - Contains the converted Markdown and optional metadata
+- **`DocumentConverterResult`** - Contains the aggregate Markdown plus structured `DocumentSegment` entries
 - **`StreamInfo`** - Metadata about the input stream (MIME type, extension, charset, etc.)
 - **`ConverterRegistration`** - Associates converters with priority for selection
 
@@ -710,6 +721,26 @@ MarkItDown includes these converters in priority order:
 - **`ImageConverter`** - Image metadata via ExifTool and optional captions
 - **`PlainTextConverter`** - Plain text, Markdown, and other text formats (fallback)
 
+### Structured Segments & Metadata
+
+Every conversion populates `DocumentConverterResult.Segments` with strongly typed `DocumentSegment` instances. Segments preserve natural breakpoints (pages, slides, sheets, archive entries, audio ranges) alongside rich metadata:
+
+- `Type` and `Number` expose what the segment represents (for example page/slide numbers)
+- `Label` carries human-readable descriptors when available
+- `StartTime`/`EndTime` capture media timelines for audio/video content
+- `AdditionalMetadata` holds contextual properties such as archive entry paths or sheet names
+
+```csharp
+var result = await markItDown.ConvertAsync("report.pdf");
+
+foreach (var segment in result.Segments)
+{
+    Console.WriteLine($"[{segment.Type}] #{segment.Number}: {segment.Label}");
+}
+```
+
+Runtime behaviour is controlled through `SegmentOptions` on `MarkItDownOptions`. Enabling `IncludeSegmentMetadataInMarkdown` emits inline annotations like `[page:1]`, `[sheet:Sales]`, or `[timecode:00:01:00-00:02:00]` directly in the Markdown stream. Audio transcripts honour `Segments.Audio.SegmentDuration`, while still collapsing short transcripts into a single, time-aware slice.
+
 ### Converter Priority & Detection
 
 - Priority-based dispatch (lower values processed first)

src/MarkItDown/Converters/AudioConverter.cs

@@ -1,4 +1,7 @@
+using System;
 using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -44,17 +47,24 @@ public sealed class AudioConverter : IDocumentConverter
 
     private readonly IAudioMetadataExtractor metadataExtractor;
     private readonly IAudioTranscriber transcriber;
-
-    public AudioConverter(string? exifToolPath = null, Func<byte[], StreamInfo, CancellationToken, Task<string?>>? transcribeAsync = null)
-        : this(new ExifToolAudioMetadataExtractor(exifToolPath),
-            transcribeAsync is null ? NoOpAudioTranscriber.Instance : new DelegateAudioTranscriber(transcribeAsync))
+    private readonly SegmentOptions segmentOptions;
+
+    public AudioConverter(
+        string? exifToolPath = null,
+        Func<byte[], StreamInfo, CancellationToken, Task<string?>>? transcribeAsync = null,
+        SegmentOptions? segmentOptions = null)
+        : this(
+            new ExifToolAudioMetadataExtractor(exifToolPath),
+            transcribeAsync is null ? NoOpAudioTranscriber.Instance : new DelegateAudioTranscriber(transcribeAsync),
+            segmentOptions)
     {
     }
 
-    internal AudioConverter(IAudioMetadataExtractor metadataExtractor, IAudioTranscriber transcriber)
+    internal AudioConverter(IAudioMetadataExtractor metadataExtractor, IAudioTranscriber transcriber, SegmentOptions? segmentOptions = null)
     {
         this.metadataExtractor = metadataExtractor ?? throw new ArgumentNullException(nameof(metadataExtractor));
         this.transcriber = transcriber ?? throw new ArgumentNullException(nameof(transcriber));
+        this.segmentOptions = segmentOptions ?? SegmentOptions.Default;
     }
 
     public int Priority => 460;
@@ -86,6 +96,69 @@ public async Task<DocumentConverterResult> ConvertAsync(Stream stream, StreamInf
         var bytes = memory.ToArray();
 
         var metadata = await metadataExtractor.ExtractAsync(bytes, streamInfo, cancellationToken).ConfigureAwait(false);
+        var transcript = await TryTranscribeAsync(bytes, streamInfo, cancellationToken).ConfigureAwait(false);
+
+        var segments = BuildSegments(metadata, transcript, streamInfo);
+        var markdown = segments.Count > 0
+            ? SegmentMarkdownComposer.Compose(segments, segmentOptions)
+            : "*No audio metadata available.*";
+
+        var title = metadata.TryGetValue("Title", out var t) && !string.IsNullOrWhiteSpace(t)
+            ? t.Trim()
+            : streamInfo.FileName is not null ? Path.GetFileNameWithoutExtension(streamInfo.FileName) : null;
+
+        return new DocumentConverterResult(markdown, title, segments);
+    }
+
+    private async Task<string?> TryTranscribeAsync(byte[] audioBytes, StreamInfo streamInfo, CancellationToken cancellationToken)
+    {
+        try
+        {
+            return await transcriber.TranscribeAsync(audioBytes, streamInfo, cancellationToken).ConfigureAwait(false);
+        }
+        catch
+        {
+            return null;
+        }
+    }
+
+    private IReadOnlyList<DocumentSegment> BuildSegments(
+        IReadOnlyDictionary<string, string> metadata,
+        string? transcript,
+        StreamInfo streamInfo)
+    {
+        var segments = new List<DocumentSegment>();
+        var source = streamInfo.FileName;
+
+        var metadataMarkdown = BuildMetadataMarkdown(metadata);
+        if (!string.IsNullOrWhiteSpace(metadataMarkdown))
+        {
+            segments.Add(new DocumentSegment(
+                markdown: metadataMarkdown,
+                type: SegmentType.Metadata,
+                label: "Metadata",
+                source: source));
+        }
+
+        var audioDuration = ParseAudioDuration(metadata);
+        var transcriptSegments = CreateAudioSegments(transcript, audioDuration, streamInfo);
+
+        if (transcriptSegments.Count > 0)
+        {
+            segments.Add(new DocumentSegment(
+                markdown: "### Audio Transcript",
+                type: SegmentType.Section,
+                label: "Audio Transcript",
+                source: source));
+
+            segments.AddRange(transcriptSegments);
+        }
+
+        return segments;
+    }
+
+    private static string BuildMetadataMarkdown(IReadOnlyDictionary<string, string> metadata)
+    {
         var builder = new StringBuilder();
 
         foreach (var field in MetadataFields)
@@ -96,39 +169,191 @@ public async Task<DocumentConverterResult> ConvertAsync(Stream stream, StreamInf
             }
         }
 
-        var transcript = await TryTranscribeAsync(bytes, streamInfo, cancellationToken).ConfigureAwait(false);
-        if (!string.IsNullOrWhiteSpace(transcript))
+        return builder.ToString().TrimEnd();
+    }
+
+    private IReadOnlyList<DocumentSegment> CreateAudioSegments(string? transcript, TimeSpan? totalDuration, StreamInfo streamInfo)
+    {
+        if (string.IsNullOrWhiteSpace(transcript))
+        {
+            return Array.Empty<DocumentSegment>();
+        }
+
+        var cleanedTranscript = transcript.Trim();
+        var duration = totalDuration.GetValueOrDefault();
+        var segmentDuration = segmentOptions.Audio.SegmentDuration;
+
+        if (!totalDuration.HasValue || duration <= TimeSpan.Zero || segmentDuration <= TimeSpan.Zero)
+        {
+            var metadata = totalDuration.HasValue
+                ? new Dictionary<string, string> { ["totalDuration"] = FormatDuration(duration) }
+                : null;
+
+            return new List<DocumentSegment>
+            {
+                new DocumentSegment(
+                    markdown: cleanedTranscript,
+                    type: SegmentType.Audio,
+                    number: 1,
+                    label: "Segment 1",
+                    startTime: totalDuration.HasValue ? TimeSpan.Zero : null,
+                    endTime: totalDuration,
+                    source: streamInfo.FileName,
+                    additionalMetadata: metadata)
+            };
+        }
+
+        var segmentCount = Math.Max(1, (int)Math.Ceiling(duration.TotalSeconds / segmentDuration.TotalSeconds));
+        var lengthBasedCap = Math.Max(1, cleanedTranscript.Length / 500);
+        segmentCount = Math.Min(segmentCount, lengthBasedCap);
+
+        var chunks = SplitTranscriptIntoChunks(cleanedTranscript, segmentCount);
+        var segments = new List<DocumentSegment>(chunks.Count);
+
+        for (var i = 0; i < chunks.Count; i++)
+        {
+            var start = segmentDuration * i;
+            if (start > duration)
+            {
+                start = duration;
+            }
+
+            var end = segmentDuration * (i + 1);
+            if (end > duration)
+            {
+                end = duration;
+            }
+
+            if (i == chunks.Count - 1 && end < duration)
+            {
+                end = duration;
+            }
+
+            var metadata = new Dictionary<string, string>
+            {
+                ["segment"] = (i + 1).ToString(CultureInfo.InvariantCulture),
+                ["totalDuration"] = FormatDuration(duration)
+            };
+
+            segments.Add(new DocumentSegment(
+                markdown: chunks[i],
+                type: SegmentType.Audio,
+                number: i + 1,
+                label: $"Segment {i + 1}",
+                startTime: start,
+                endTime: end,
+                source: streamInfo.FileName,
+                additionalMetadata: metadata));
+        }
+
+        return segments;
+    }
+
+    private static List<string> SplitTranscriptIntoChunks(string transcript, int segmentCount)
+    {
+        if (segmentCount <= 1)
         {
-            if (builder.Length > 0)
+            return new List<string> { transcript };
+        }
+
+        var segments = new List<string>(segmentCount);
+        var length = transcript.Length;
+        var chunkSize = Math.Max(1, (int)Math.Ceiling((double)length / segmentCount));
+        var position = 0;
+
+        var breakChars = new[] { '.', '!', '?', '\n', '\r', ' ' };
+
+        while (position < length)
+        {
+            var end = Math.Min(position + chunkSize, length);
+            if (end < length)
             {
-                builder.AppendLine();
+                var searchLength = Math.Min(chunkSize, 200);
+                var splitIndex = transcript.LastIndexOfAny(breakChars, end - 1, searchLength);
+                if (splitIndex > position)
+                {
+                    end = splitIndex + 1;
+                }
             }
 
-            builder.AppendLine("### Audio Transcript");
-            builder.AppendLine();
-            builder.AppendLine(transcript.Trim());
+            var chunk = transcript[position..end].Trim();
+            if (!string.IsNullOrEmpty(chunk))
+            {
+                segments.Add(chunk);
+            }
+
+            position = end;
         }
 
-        var markdown = builder.Length > 0 ? builder.ToString().TrimEnd() : "*No audio metadata available.*";
-        var title = metadata.TryGetValue("Title", out var t) && !string.IsNullOrWhiteSpace(t)
-            ? t.Trim()
-            : streamInfo.FileName is not null ? Path.GetFileNameWithoutExtension(streamInfo.FileName) : null;
+        if (segments.Count == 0)
+        {
+            segments.Add(transcript);
+        }
 
-        return new DocumentConverterResult(markdown, title);
+        return segments;
     }
 
-    private async Task<string?> TryTranscribeAsync(byte[] audioBytes, StreamInfo streamInfo, CancellationToken cancellationToken)
+    private static TimeSpan? ParseAudioDuration(IReadOnlyDictionary<string, string> metadata)
     {
-        try
+        if (metadata.TryGetValue("Duration", out var value) && TryParseDuration(value, out var duration))
         {
-            return await transcriber.TranscribeAsync(audioBytes, streamInfo, cancellationToken).ConfigureAwait(false);
+            return duration;
         }
-        catch
+
+        if (metadata.TryGetValue("MediaDuration", out value) && TryParseDuration(value, out duration))
         {
-            return null;
+            return duration;
         }
+
+        return null;
     }
 
+    private static bool TryParseDuration(string rawValue, out TimeSpan duration)
+    {
+        var value = rawValue.Trim();
+        var colonCount = 0;
+        foreach (var ch in value)
+        {
+            if (ch == ':')
+            {
+                colonCount++;
+            }
+        }
+
+        if (colonCount == 1 && TimeSpan.TryParseExact(value, @"mm\:ss", CultureInfo.InvariantCulture, out duration))
+        {
+            return true;
+        }
+
+        if (colonCount == 2 && TimeSpan.TryParseExact(value, @"hh\:mm\:ss", CultureInfo.InvariantCulture, out duration))
+        {
+            return true;
+        }
+
+        if (TimeSpan.TryParse(value, CultureInfo.InvariantCulture, out duration))
+        {
+            return true;
+        }
+
+        var sanitized = value.Replace(',', '.');
+        if (sanitized.EndsWith("s", StringComparison.OrdinalIgnoreCase))
+        {
+            sanitized = sanitized[..^1];
+        }
+
+        if (double.TryParse(sanitized, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds))
+        {
+            duration = TimeSpan.FromSeconds(seconds);
+            return true;
+        }
+
+        duration = default;
+        return false;
+    }
+
+    private static string FormatDuration(TimeSpan duration)
+        => duration.ToString(duration.TotalHours >= 1 ? @"hh\:mm\:ss" : @"mm\:ss", CultureInfo.InvariantCulture);
+
     internal interface IAudioMetadataExtractor
     {
         Task<IReadOnlyDictionary<string, string>> ExtractAsync(byte[] audioBytes, StreamInfo streamInfo, CancellationToken cancellationToken);