From 99519db27c15322ea20860c761ba40611991807b Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:03:42 +0200 Subject: [PATCH 01/12] Fix (keepspaces): Keep non-breaking spaces between words as spaces With keepspaces = 1, every non-breaking space was removed. Web-clipped content often has a non-breaking space as the only space between two words (e.g. 'press theWindowskey'), so words were glued together ('Windowskey'). Now non-breaking spaces are removed only at the start (keeping the line's indentation) and end of a line (incl. lines that only contain them), where converting them could create an indented code block or a trailing-double-space hard line break. Any other run of them becomes a single space. Co-Authored-By: Claude Opus 5.5 --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 8 +++++++- ConvertOneNote2MarkDown-v2.ps1 | 19 ++++++++++++++++--- README.md | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..4ab10d7 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -1201,6 +1201,9 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { $fakeMarkdownContent = @" hello world$( [char]0x00A0 ) +press the$( [char]0x00A0 ) Windows$( [char]0x00A0 )key +$( [char]0x00A0 )$( [char]0x00A0 )no leading non-breaking spaces + $( [char]0x00A0 )keeps its indentation - foo @@ -1244,11 +1247,14 @@ some more } } - # Should remove extra newline between unordered and ordered lists, remove non-breaking spaces, and '>' from ordered lists. Ignore first 8 lines for page header + # Should remove extra newline between unordered and ordered lists, remove non-breaking spaces at line boundaries but keep them as spaces between words, and '>' from ordered lists. Ignore first 8 lines for page header $split = $mutated -split "`n" $expectedBody = $split[8..($split.Count - 1)] -join "`n" $expectedBody | Should -Be $( @" hello world +press the Windows key +no leading non-breaking spaces + keeps its indentation - foo - foo1 diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..abf9bc3 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -152,7 +152,7 @@ Whether to include page timestamp and separator at top of document } keepspaces = @{ description = @' -Whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines, and `>` after unordered lists +Whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines and line ends (other non-breaking spaces become normal spaces), and `>` after unordered lists 1: Clear - Default 2: Don't clear '@ @@ -933,11 +933,24 @@ Function New-SectionGroupConversionConfig { @{ description = 'Clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines, and `>` after unordered lists' replacements = @( - # Remove non-breaking spaces + # Non-breaking spaces: web-clipped content is full of them, often as the only space between two words + # (e.g. 'theWindowskey'). Deleting them all glued those words together ('Windowskey'), so: + # remove them at the end of a line (incl. lines that only contain them), since converting them there could + # create a trailing-double-space hard line break, @{ - searchRegex = [regex]::Escape([char]0x00A0) + searchRegex = '(?m)[ \t\u00A0]*\u00A0[ \t\u00A0]*$' replacement = '' } + # remove them at the start of a line, keeping its indentation, since converting them there could turn the line into an indented code block, + @{ + searchRegex = '(?m)^([ \t]*)\u00A0[ \t\u00A0]*' + replacement = '$1' + } + # and turn any other run of them (with adjacent spaces) into a single space + @{ + searchRegex = '[ \t]*\u00A0[ \t\u00A0]*' + replacement = ' ' + } # Remove an extra newline between each occurrence of '- some unordered list item' @{ searchRegex = '(\s*)- ([^\r\n]*)\r*\n\r*\n(?=\s*-)' diff --git a/README.md b/README.md index c23006f..5816860 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote * `markdown_phpextra` (PHP Markdown Extra) * `markdown_strict` (original unextended Markdown) * Improved headers, with title now as a `#` heading, standardized `DateTime` format for created and modified dates, and horizontal line to separate from rest of document -* Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines, and `>` after unordered lists, which are created when converting with Pandoc +* Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines and line ends (other non-breaking spaces become normal spaces), and `>` after unordered lists, which are created when converting with Pandoc * Choose whether to remove `\` escape symbol that are created when converting with Pandoc * Choose whether to use Line Feed (`LF`) or Carriage Return + Line Feed (`CRLF`) for new lines * Choose whether to include a `.pdf` export alongside the `.md` file. `.md` does not preserve `InkDrawing` (i.e. overlayed drawings, highlights, pen marks) absolute positions within a page, but a `.pdf` export is a complete page snapshot that preserves `InkDrawing` absolute positions within a page. From d2ebc34ed72c7bcb67585616550f0de0bbea00d0 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:04:09 +0200 Subject: [PATCH 02/12] Fix (convert): Remove the page's title, date and time lines only when present The first 6 lines of pandoc's output were always discarded, assuming they are the page's title, date and time. Pages without a date and time (e.g. emails sent to OneNote) lost real content, e.g. a table and a Subject line, or a page's first command. The new Remove-PageHeader removes the header only when there is a date line (text with a 4-digit year, any language) followed by a time line (e.g. '09:50', '9:19 AM'); the title may be empty. Without such a header, only a first line repeating the page's name is removed. Also fix the Created/Modified offset: the 'zz00' format printed only the offset's hours, which is wrong for e.g. +05:30. The output format stays the same (e.g. '+0200'). Co-Authored-By: Claude Opus 5.5 --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 49 +++++++++++++++++++++ ConvertOneNote2MarkDown-v2.ps1 | 64 +++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..76a6a0d 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -407,6 +407,55 @@ Describe "Encode-Markdown" -Tag 'Unit' { } +Describe "Remove-PageHeader" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should remove the title, date and time lines ()" -TestCases @( + @{ Name = 'German'; Lines = @('Docker', '', 'Donnerstag, 17. Oktober 2024', '', '09:50', '', 'body', 'more') } + @{ Name = 'English, AM/PM'; Lines = @('Max Planck', '', 'Monday, September 8, 2014', '', '9:19 AM', '', 'body', 'more') } + @{ Name = 'title with a year'; Lines = @('Pressemeetings 2011', '', 'Saturday, October 29, 2011', '', '6:22 AM', '', 'body', 'more') } + @{ Name = 'untitled page'; Lines = @([string][char]0x00A0, '', 'Dienstag, 19. August 2025', '', '12:34', '', 'body', 'more') } + ) { + param ($Name, $Lines) + + $result = @( Remove-PageHeader -Lines $Lines ) + + ($result -join "`n").Trim() | Should -Be "body`nmore" + } + + It "Should keep all lines if there is no date and time (e.g. an email sent to OneNote)" { + $lines = @('Linksammlung', '', '| **Tags:** #Teufel |', '|-------------------|', '', '**Subject:** Linksammlung', 'body') + + $result = @( Remove-PageHeader -Lines $lines ) + + $result | Should -Be $lines + } + + It "Should remove only a leading title line if there is no date and time, but a -Title" { + $lines = @('curl \> /volume1/extern', '', 'curl \> /volume1/external_ip.log', 'body') + + $result = @( Remove-PageHeader -Lines $lines -Title 'curl http://ifconfig.me/ip > /volume1/extern' ) + + ($result -join "`n").Trim() | Should -Be "curl \> /volume1/external_ip.log`nbody" + } + + It "Should return an empty string for a page that only has a header" { + $result = @( Remove-PageHeader -Lines @('Docker', '', 'Donnerstag, 17. Oktober 2024', '', '09:50') ) + + $result | Should -Be @('') + } + + It "Should accept empty input" { + $result = @( Remove-PageHeader -Lines @() ) + + $result.Count | Should -Be 0 + } + + } + +} + Describe "New-OneNoteConnection" -Tag 'Unit' { Context 'Behavior' { diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..eb62542 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -497,6 +497,54 @@ Function Encode-Markdown { $Name } +Function Remove-PageHeader { + [CmdletBinding()] + param ( + # The markdown lines produced by pandoc from a published page + [Parameter(Mandatory)] + [AllowEmptyCollection()] + [AllowEmptyString()] + [string[]] + $Lines + , + # The page's name. Without a date/time header, a first line matching it is removed too (we add our own heading) + [Parameter()] + [string] + $Title = '' + ) + + # A page published to docx starts with its title, date and time paragraphs (e.g. 'Docker', 'Donnerstag, 17. Oktober 2024', '09:50'), + # which pandoc renders as the first lines of the markdown. We add our own heading, so remove them - but only when they are really + # there: some pages (e.g. emails sent to OneNote) have no date/time paragraphs, and blindly dropping the first 6 lines removed real content. + # The title may be empty (an untitled page yields a line with just a non-breaking space). + $datePattern = '^(?=.*\p{L})(?=.*\b\d{4}\b)[^|#>*]{1,80}$' # Some text with a 4-digit year, in any locale + $timePattern = '^\d{1,2}:\d{2}(:\d{2})?(\s*[AaPp]\.?\s?[Mm]\.?)?$' # E.g. '09:50', '9:19 AM' + $clean = { param($s) $s.Replace([string][char]0x00A0, ' ').Trim() } + $lettersAndDigits = { param($s) $s -replace '[^\p{L}\p{N}]', '' } # Pandoc escapes/autolinks the title text, so compare only letters and digits + $nonBlank = @( + for ($i = 0; $i -lt $Lines.Count -and $i -lt 8; $i++) { + if (& $clean $Lines[$i]) { + $i + } + } + ) + $lastHeaderLine = -1 + if ($nonBlank.Count -ge 3 -and (& $clean $Lines[$nonBlank[1]]) -match $datePattern -and (& $clean $Lines[$nonBlank[2]]) -match $timePattern) { + $lastHeaderLine = $nonBlank[2] # Title, date, time + }elseif ($nonBlank.Count -ge 2 -and (& $clean $Lines[$nonBlank[0]]) -match $datePattern -and (& $clean $Lines[$nonBlank[1]]) -match $timePattern) { + $lastHeaderLine = $nonBlank[1] # Empty title, date, time + }elseif ($Title -and $nonBlank.Count -ge 1 -and (& $lettersAndDigits $Lines[$nonBlank[0]]) -and (& $lettersAndDigits $Lines[$nonBlank[0]]) -eq (& $lettersAndDigits $Title)) { + $lastHeaderLine = $nonBlank[0] # Title only + } + if ($lastHeaderLine -lt 0) { + $Lines + }elseif ($lastHeaderLine + 1 -lt $Lines.Count) { + $Lines[($lastHeaderLine + 1)..($Lines.Count - 1)] + }else { + '' + } +} + Function Set-ContentNoBom { [CmdletBinding()] param ( @@ -920,8 +968,9 @@ Function New-SectionGroupConversionConfig { replacement = & { $heading = "# $( $pageCfg['object'].name )" if ($config['headerTimestampEnabled']['value'] -eq 1) { - $heading += "`n`nCreated: $( $pageCfg['dateTime'].ToString('yyyy-MM-dd HH:mm:ss zz00') )" - $heading += "`n`nModified: $( $pageCfg['lastModifiedTime'].ToString('yyyy-MM-dd HH:mm:ss zz00') )" + # 'zzz' without its colon (e.g. '+0200'). The former 'zz00' printed only the offset's hours, which is wrong for e.g. +05:30 + $heading += "`n`nCreated: $( $pageCfg['dateTime'].ToString('yyyy-MM-dd HH:mm:ss ') + $pageCfg['dateTime'].ToString('zzz').Replace(':', '') )" + $heading += "`n`nModified: $( $pageCfg['lastModifiedTime'].ToString('yyyy-MM-dd HH:mm:ss ') + $pageCfg['lastModifiedTime'].ToString('zzz').Replace(':', '') )" $heading += "`n`n---`n`n" } $heading @@ -1273,15 +1322,8 @@ Function Convert-OneNotePage { if (!$config['dryRun']['value']) { # Get markdown content $content = @( Get-Content -LiteralPath $pageCfg['filePath'] -ErrorAction Stop ) # Use -LiteralPath so that characters like '(', ')', '[', ']', '`', "'", '"' are supported. Or else we will get an error "Cannot find path 'xxx' because it does not exist" - $content = @( - if ($content.Count -gt 6) { - # Discard first 6 lines which contain a header, created date, and time. We are going to add our own header - $content[6..($content.Count - 1)] - }else { - # Empty page - '' - } - ) -join "`n" + # Discard the page's title, date and time lines (if present). We are going to add our own header + $content = @( Remove-PageHeader -Lines $content -Title $pageCfg['object'].name ) -join "`n" } # Mutate From 079b0eb510fdd092740af717b51532780e231737 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:04:36 +0200 Subject: [PATCH 03/12] Fix (file names): Keep duplicate page postfixes and avoid broken attachment and image links - Duplicate page names: the '-1', '-2' postfix was appended before truncating to mdFileNameAndFolderNameMaxLength, so for long names it was cut off again, and the pages overwrote each other. Duplicates are now detected on the truncated name (which also catches long names that only differ after the cut-off), and the postfix survives truncation (new Truncate-PathFileName -KeepSuffix). - Inserted attachments: references were replaced with a plain search, so with attachments '11-36-00002.pdf' and 'Datenblatt_11-36-00002.pdf' on one page, the first name also matched inside the second, nesting one link inside the other. Only whole file names are matched now. - Images: image file names are prefixed with the page path, which can contain characters like '#' and "'" (e.g. 'c't-expert-community--Newsletter-#10-image1.jpg'). A '#' in a link is read as a URL fragment, breaking the image. Image names now go through Remove-InvalidFileNameCharsInsertedFiles, like inserted attachments. Co-Authored-By: Claude Opus 5.5 --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 72 ++++++++++++++++++++++++++++ ConvertOneNote2MarkDown-v2.ps1 | 36 +++++++++----- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..8645573 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -313,6 +313,24 @@ Describe "Truncate-PathFileName" -Tag 'Unit' { $path | Truncate-PathFileName -Length $length | Should -Be $expectedPath } + It "Should keep a given -KeepSuffix when truncating" { + $drive = "C:$( [io.path]::DirectorySeparatorChar )" # E.g. C:\ + $path = $drive + ("a" * 1000) + '-1' + $length = 123 + $expectedPath = $drive + ("a" * 121) + '-1' + + $path | Truncate-PathFileName -Length $length -KeepSuffix '-1' | Should -Be $expectedPath + } + + It "Should ignore -KeepSuffix if the filename does not end with it" { + $drive = "C:$( [io.path]::DirectorySeparatorChar )" # E.g. C:\ + $path = $drive + ("a" * 1000) + $length = 123 + $expectedPath = $drive + ("a" * 123) + + $path | Truncate-PathFileName -Length $length -KeepSuffix '-1' | Should -Be $expectedPath + } + } } @@ -833,6 +851,32 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { } } + It "Should not match an attachment's name inside a longer attachment name" { + Mock Get-OneNotePageContent { + @' + + + + + + + + + +'@ -as [xml] + } + + $pageCfg = @( New-SectionGroupConversionConfig @params 6>$null ) | Select-Object -First 1 + $content = '\<\\> and \<\<11-36-00002.pdf\>\>' + foreach ($m in ($pageCfg['mutations'] | Where-Object { $_['description'] -eq 'Change inserted attachment(s) filename references' })) { + foreach ($r in $m['replacements']) { + $content = $content -replace $r['searchRegex'], $r['replacement'] + } + } + + $content | Should -Be "\<\<[Datenblatt_11-36-00002.pdf]($( $pageCfg['mediaPathPandoc'] )/Datenblatt_11-36-00002.pdf)\>\> and \<\<[11-36-00002.pdf]($( $pageCfg['mediaPathPandoc'] )/11-36-00002.pdf)\>\>" + } + It "Should generate fully unique docx file for each page, even for identically-named pages in a section" { $result = @( New-SectionGroupConversionConfig @params 6>$null ) @@ -885,6 +929,34 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { } } + It "Should keep the unique postfix of identically named pages when truncating long names" { + $longName = 'Some long name ' * 20 + $fakeHierarchy = @" + + + + + + + + + + +"@ -as [xml] + $params['SectionGroups'] = $fakeHierarchy.Notebooks.Notebook + $params['Config']['mdFileNameAndFolderNameMaxLength']['value'] = 100 + + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + + $result.Count | Should -Be 3 + $result[1]['fileName'] | Should -Match '-1\.md$' + $result[2]['fileName'] | Should -Match '-2\.md$' + @( $result | ForEach-Object { $_['filePathNormal'] } | Select-Object -Unique ).Count | Should -Be 3 + foreach ($pageCfg in $result) { + $pageCfg['fileName'].Length | Should -BeLessOrEqual 100 + } + } + It "Should honor config notesdestpath" { if ($env:OS -match 'windows') { $params['Config']['notesdestpath']['value'] = 'D:\foo\bar' diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..bb5a62e 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -386,6 +386,11 @@ Function Truncate-PathFileName { [ValidateRange(0,255)] [int] $Length + , + # A suffix (e.g. the '-1' postfix of a duplicate page name) that must survive truncation + [Parameter()] + [string] + $KeepSuffix = '' ) process { @@ -404,7 +409,11 @@ Function Truncate-PathFileName { $name = Split-Path $Path -Leaf if ($name.Length -gt $maxLength) { $parent = Split-Path $Path -Parent - $truncatedName = $name.Substring(0, $maxLength) + $truncatedName = if ($KeepSuffix -and $name.EndsWith($KeepSuffix) -and $maxLength -gt $KeepSuffix.Length) { + $name.Substring(0, $maxLength - $KeepSuffix.Length) + $KeepSuffix + }else { + $name.Substring(0, $maxLength) + } [io.path]::combine( $parent, $truncatedName ) }else { $Path @@ -800,17 +809,18 @@ Function New-SectionGroupConversionConfig { $pageCfg['filePathRel'] = & { $filePathRel = "$( $pageCfg['pagePrefix'] )$( $pageCfg['nameCompat'] )" - # in case multiple pages with the same name exist in a section, postfix the filename + # In case multiple pages with the same name exist in a section, postfix the filename. + # Compare the names as they will be written (truncated to fit '.md'), so two long names that only differ + # after the cut-off are also recognized as duplicates. The postfix is kept when truncating below, so it can't be cut off again. + $pageCfg['filePathRelKey'] = $filePathRel | Truncate-PathFileName -Length ($config['mdFileNameAndFolderNameMaxLength']['value'] - 3) $recurrence = 0 foreach ($p in $sectionCfg['pages']) { - if ($p['pagePrefix'] -eq $pageCfg['pagePrefix'] -and $p['pathFromRoot'] -eq $pageCfg['pathFromRoot']) { + if ($p['filePathRelKey'] -eq $pageCfg['filePathRelKey']) { $recurrence++ } } - if ($recurrence -gt 0) { - $filePathRel = "$filePathRel-$recurrence" - } - $filePathRel | Truncate-PathFileName -Length $config['mdFileNameAndFolderNameMaxLength']['value'] # Truncate to no more than 255 characters so we don't hit the folder name limit on most file systems on Windows / Linux + $pageCfg['filePathRelPostfix'] = if ($recurrence -gt 0) { "-$recurrence" } else { '' } + "$filePathRel$( $pageCfg['filePathRelPostfix'] )" | Truncate-PathFileName -Length $config['mdFileNameAndFolderNameMaxLength']['value'] -KeepSuffix $pageCfg['filePathRelPostfix'] # Truncate to no more than 255 characters so we don't hit the folder name limit on most file systems on Windows / Linux } $pageCfg['filePathRelUnderscore'] = $pageCfg['filePathRel'].Replace( [io.path]::DirectorySeparatorChar, '_' ) $pageCfg['filePathNormal'] = & { @@ -819,7 +829,7 @@ Function New-SectionGroupConversionConfig { }else { [io.path]::combine( $cfg['notesDirectory'], $sectionCfg['nameCompat'], "$( $pageCfg['filePathRel'] )" ) } - "$( $pathWithoutExtension | Truncate-PathFileName -Length ($config['mdFileNameAndFolderNameMaxLength']['value'] - 3) ).md" # Truncate to no more than 255 characters so we don't hit the file name limit on Windows / Linux + "$( $pathWithoutExtension | Truncate-PathFileName -Length ($config['mdFileNameAndFolderNameMaxLength']['value'] - 3) -KeepSuffix $pageCfg['filePathRelPostfix'] ).md" # Truncate to no more than 255 characters so we don't hit the file name limit on Windows / Linux } $pageCfg['filePathLong'] = "\\?\$( $pageCfg['filePathNormal'] )" # A non-Win32 path. Prefixing with '\\?\' allows Windows Powershell <= 5 (based on Win32) to support long absolute paths. $pageCfg['filePath'] = if ($PSVersionTable.PSVersion.Major -le 5) { @@ -896,7 +906,9 @@ Function New-SectionGroupConversionConfig { description = 'Change inserted attachment(s) filename references' replacements = @( @{ - searchRegex = [regex]::Escape( $attachmentCfg['object'].preferredName ) + # Only where the name is not part of a longer file name: with attachments '11-36-00002.pdf' and 'Datenblatt_11-36-00002.pdf' + # on one page, a plain search for the first one also matched inside the second, nesting one link inside the other + searchRegex = '(? Date: Thu, 24 Sep 2026 15:05:03 +0200 Subject: [PATCH 04/12] Fix (file names): Decode OneNote's escape codes in section names OneNote reports characters that aren't allowed in section file names as '^' codes in the section's name, e.g. 'AI ^M ML' for a section named 'AI / ML', and 'DECT^J Bluetooth' for 'DECT & Bluetooth'. These codes ended up in folder names ('AI-^M--ML'). Decode '^M' to '/' and '^J' to '&' (the codes observed so far) before removing invalid file name characters, for notebook, section group and section names. Co-Authored-By: Claude Opus 5.5 --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 18 ++++++++++++++++++ ConvertOneNote2MarkDown-v2.ps1 | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..0fc7b37 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -372,6 +372,24 @@ Describe "Remove-InvalidFileNameCharsInsertedFiles" -Tag 'Unit' { } +Describe "ConvertFrom-OneNoteEscapedName" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should decode OneNote's '^' codes in section names" { + 'AI ^M ML' | ConvertFrom-OneNoteEscapedName | Should -Be 'AI / ML' + 'Messen^J Konferenzen^J Seminare' | ConvertFrom-OneNoteEscapedName | Should -Be 'Messen& Konferenzen& Seminare' + 'plain name' | ConvertFrom-OneNoteEscapedName | Should -Be 'plain name' + } + + It "Should turn a decoded '/' into a dash in the directory name" { + 'AI ^M ML' | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars | Should -Not -Match '\^' + } + + } + +} + Describe "Encode-Markdown" -Tag 'Unit' { Context 'Parameters' { diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..04b1cc5 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -465,6 +465,22 @@ Function Remove-InvalidFileNameCharsInsertedFiles { return $newName } +Function ConvertFrom-OneNoteEscapedName { + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true,Position = 0,ValueFromPipeline = $true)] + [AllowEmptyString()] + [string] + $Name + ) + + process { + # OneNote stores characters that aren't allowed in its section file names as '^' codes in the section's name, + # e.g. a section named 'AI / ML' is reported as 'AI ^M ML', and 'DECT & Bluetooth' as 'DECT^J Bluetooth' + $Name.Replace('^M', '/').Replace('^J', '&') + } +} + Function Encode-Markdown { [CmdletBinding()] param ( @@ -681,7 +697,7 @@ Function New-SectionGroupConversionConfig { $cfg['object'] = $sectionGroup # Keep a reference to the SectionGroup object $cfg['kind'] = 'SectionGroup' $cfg['id'] = $sectionGroup.ID # E.g. {9570CCF6-17C2-4DCE-83A0-F58AE8914E29}{1}{B0} - $cfg['nameCompat'] = $sectionGroup.name | Remove-InvalidFileNameChars + $cfg['nameCompat'] = $sectionGroup.name | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars $cfg['levelsFromRoot'] = $LevelsFromRoot $cfg['uri'] = $sectionGroup.path # E.g. https://d.docs.live.net/0123456789abcdef/Skydrive Notebooks/mynotebook/mysectiongroup $cfg['notesDirectory'] = [io.path]::combine( $NotesDestination.TrimEnd('/').TrimEnd('\').Replace('\', [io.path]::DirectorySeparatorChar), $cfg['nameCompat'] ) # No need to truncate. Section Group and Section names have a max length of 50, so we should never hit the absolute path, file name, or directory name limits on Windows @@ -720,7 +736,7 @@ Function New-SectionGroupConversionConfig { $sectionCfg['object'] = $section # Keep a reference to the Section object $sectionCfg['kind'] = 'Section' $sectionCfg['id'] = $section.ID # E.g {BE566C4F-73DC-43BD-AE7A-1954F8B22C2A}{1}{B0} - $sectionCfg['nameCompat'] = $section.name | Remove-InvalidFileNameChars + $sectionCfg['nameCompat'] = $section.name | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars $sectionCfg['levelsFromRoot'] = $cfg['levelsFromRoot'] + 1 $sectionCfg['pathFromRoot'] = "$( $cfg['pathFromRoot'] )$( [io.path]::DirectorySeparatorChar )$( $sectionCfg['nameCompat'] )".Trim([io.path]::DirectorySeparatorChar) # No need to truncate. Section Group and Section names have a max length of 50, so we should never hit the absolute path, file name, or directory name limits on Windows $sectionCfg['pathFromRootCompat'] = $sectionCfg['pathFromRoot'] | Remove-InvalidFileNameChars From 672457ef0c0c18b9bc11a1a3f33fa93531f0f3c2 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:05:29 +0200 Subject: [PATCH 05/12] Enhancement (configuration): Add option overwrite to remove a notebook's previous output When converting into an existing notes folder, files of the same name are overwritten, but notes, folders and media that were renamed or deleted in OneNote since the last run are left behind. With overwrite = 2, a notebook's previous output is removed before converting it. The notebook's docx folder is kept, so usedocx = 2 can still reuse it. Only the notebook's own folder under notesdestpath is touched. Read-only files (e.g. inserted attachments) and paths beyond 260 characters on Windows Powershell 5 are handled. The default (1) keeps the current behavior. Co-Authored-By: Claude Opus 5.5 --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 38 ++++++++++++++++++++ ConvertOneNote2MarkDown-v2.ps1 | 53 ++++++++++++++++++++++++++++ README.md | 1 + 3 files changed, 92 insertions(+) diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..ae96aef 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -1812,6 +1812,44 @@ Describe 'Convert-OneNotePage' -Tag 'Unit' { } +Describe "Remove-PreviousNotebookOutput" -Tag 'Unit' { + + Context 'Behavior' { + + BeforeEach { + $notebookDirectory = Join-Path $TestDrive 'my-notebook' + if (Test-Path -LiteralPath $notebookDirectory) { + Remove-Item -LiteralPath $notebookDirectory -Recurse -Force # Left over from a previous test (e.g. the dry run), incl. the read-only file + } + New-Item -ItemType Directory -Path (Join-Path $notebookDirectory 'docx') -Force > $null + New-Item -ItemType Directory -Path (Join-Path $notebookDirectory 'section\media') -Force > $null + 'docx' | Set-Content -LiteralPath (Join-Path $notebookDirectory 'docx\page.docx') + 'note' | Set-Content -LiteralPath (Join-Path $notebookDirectory 'section\stale-page.md') + 'pdf' | Set-Content -LiteralPath (Join-Path $notebookDirectory 'section\media\attachment.pdf') + (Get-Item -LiteralPath (Join-Path $notebookDirectory 'section\media\attachment.pdf')).IsReadOnly = $true # OneNote's inserted files are read-only + } + + It "Should remove everything in the notebook's directory except the docx folder" { + Remove-PreviousNotebookOutput -NotesDestination $TestDrive -NotebookName 'my notebook' + + Test-Path -LiteralPath (Join-Path $notebookDirectory 'section') | Should -Be $false + Test-Path -LiteralPath (Join-Path $notebookDirectory 'docx\page.docx') | Should -Be $true + } + + It "Should not remove anything on a dry run" { + Remove-PreviousNotebookOutput -NotesDestination $TestDrive -NotebookName 'my notebook' -DryRun + + Test-Path -LiteralPath (Join-Path $notebookDirectory 'section\stale-page.md') | Should -Be $true + } + + It "Should do nothing if the notebook's directory does not exist" { + { Remove-PreviousNotebookOutput -NotesDestination $TestDrive -NotebookName 'other notebook' } | Should -Not -Throw + } + + } + +} + Describe "Print-ConversionErrors" -Tag 'Unit' { Context 'Behavior' { diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..0b0b3ff 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -186,6 +186,16 @@ Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new line Whether to include a PDF export alongside the markdown file 1: Don't include PDF - Default 2: Include PDF +'@ + default = 1 + value = 1 + validateRange = 1,2 + } + overwrite = @{ + description = @' +Whether to remove a notebook's previous output before converting it. Without this, files of the same name are overwritten, but notes, folders and media that were renamed or deleted in OneNote since the last run are left behind. The notebook's docx folder is always kept +1: Keep previous output - Default +2: Remove previous output (except the docx folder) '@ default = 1 value = 1 @@ -528,6 +538,45 @@ Function Set-ContentNoBom { } } +Function Remove-PreviousNotebookOutput { + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] + $NotesDestination + , + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] + $NotebookName + , + [Parameter()] + [switch] + $DryRun + ) + + # The same directory New-SectionGroupConversionConfig converts this notebook into + $nameCompat = $NotebookName | Remove-InvalidFileNameChars + if (! $nameCompat.Trim('-. ')) { + throw "Refusing to remove previous output of notebook '$NotebookName': its directory name would be empty" + } + $notebookDirectory = [io.path]::combine( $NotesDestination.TrimEnd('/').TrimEnd('\').Replace('\', [io.path]::DirectorySeparatorChar), $nameCompat ) + if (! (Test-Path -LiteralPath $notebookDirectory -PathType Container)) { + return + } + + # Keep the docx folder, so that 'usedocx = 2' can still reuse its files + foreach ($item in @( Get-ChildItem -LiteralPath $notebookDirectory -Force | Where-Object { $_.Name -ne 'docx' } )) { + "Removing previous output: $( $item.FullName )" | Write-Verbose + if (! $DryRun) { + # A non-Win32 path, so that Windows Powershell <= 5 can also remove files beyond the 260-character path limit + $literalPath = if ($PSVersionTable.PSVersion.Major -le 5 -and $env:OS -match 'windows') { "\\?\$( $item.FullName )" } else { $item.FullName } + Remove-Item -LiteralPath $literalPath -Recurse -Force -ErrorAction Stop + } + } +} + Function New-OneNoteConnection { [CmdletBinding()] param () @@ -1432,6 +1481,10 @@ Function Convert-OneNote2MarkDown { $pageConversionConfigsAll = @() foreach ($notebook in $notebooks) { "`nConverting notebook '$( $notebook.name )'... (Ignoring deleted notes)" | Write-Host -ForegroundColor Cyan + if ($config['overwrite']['value'] -eq 2) { + "Removing previous output of notebook '$( $notebook.name )' (keeping its docx folder)" | Write-Host -ForegroundColor Cyan + Remove-PreviousNotebookOutput -NotesDestination $config['notesdestpath']['value'] -NotebookName $notebook.name -DryRun:([bool]$config['dryRun']['value']) + } New-SectionGroupConversionConfig -OneNoteConnection $OneNote -NotesDestination $config['notesdestpath']['value'] -Config $config -SectionGroups $notebook -LevelsFromRoot 0 -ErrorVariable +totalerr | Tee-Object -Variable pageConversionConfigs | Convert-OneNotePage -OneNoteConnection $OneNote -Config $config -ErrorVariable +totalerr "`nDone converting notebook '$( $notebook.name )' with $( ($pageConversionConfigs | Measure-object).Count ) notes." | Write-Host -ForegroundColor Cyan $pageConversionConfigsAll += $pageConversionConfigs diff --git a/README.md b/README.md index c23006f..857296d 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote * Choose whether to remove `\` escape symbol that are created when converting with Pandoc * Choose whether to use Line Feed (`LF`) or Carriage Return + Line Feed (`CRLF`) for new lines * Choose whether to include a `.pdf` export alongside the `.md` file. `.md` does not preserve `InkDrawing` (i.e. overlayed drawings, highlights, pen marks) absolute positions within a page, but a `.pdf` export is a complete page snapshot that preserves `InkDrawing` absolute positions within a page. +* Choose whether to **remove a notebook's previous output** before converting it (`$overwrite = 2`), so notes, folders and media that were renamed or deleted in OneNote since the last run don't linger. The notebook's `docx` folder is always kept, so existing `.docx` files can still be reused. * Detailed logs. Run the script with `-Verbose` to see detailed logs of each page's conversion. ## Known Issues From 0058d6288baf401d8932715972a5d17b79c6bba3 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:14:00 +0100 Subject: [PATCH 06/12] Local: Add LOCAL-CHANGES.md, README pointer and docs; decode escape codes in notebook names - LOCAL-CHANGES.md records the differences to upstream and the PR status. - docs/ holds the PR texts and the 2026-11-01 transcription quality report. - Remove-PreviousNotebookOutput now also decodes OneNote escape codes in the notebook name (not part of PR #195). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- ConvertOneNote2MarkDown-v2.ps1 | 2 +- LOCAL-CHANGES.md | 84 ++++++++++++++++++++ README.md | 2 + docs/TRANSCRIPTION-QUALITY-REPORT.md | 81 +++++++++++++++++++ docs/pr-descriptions.md | 113 +++++++++++++++++++++++++++ 5 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 LOCAL-CHANGES.md create mode 100644 docs/TRANSCRIPTION-QUALITY-REPORT.md create mode 100644 docs/pr-descriptions.md diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 4a829cf..c2e1b56 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -630,7 +630,7 @@ Function Remove-PreviousNotebookOutput { ) # The same directory New-SectionGroupConversionConfig converts this notebook into - $nameCompat = $NotebookName | Remove-InvalidFileNameChars + $nameCompat = $NotebookName | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars if (! $nameCompat.Trim('-. ')) { throw "Refusing to remove previous output of notebook '$NotebookName': its directory name would be empty" } diff --git a/LOCAL-CHANGES.md b/LOCAL-CHANGES.md new file mode 100644 index 0000000..d97e30e --- /dev/null +++ b/LOCAL-CHANGES.md @@ -0,0 +1,84 @@ +# Local changes + +This copy of [ConvertOneNote2MarkDown](https://github.com/theohbrothers/ConvertOneNote2MarkDown) is based on upstream `master` as of 2026-09-24. Since 2026-11-01 the folder is a git repository (`origin` = the fork, `upstream` = the original project). `main` is upstream `master` plus the five PR branches below, merged; compare with `git diff upstream/master`. The former `.orig` copies are no longer needed. + +All changes were checked against the real notebooks by converting them from the kept `.docx` files and comparing the result with an export from the original script: + +- 138 of 197 pages were byte-identical. +- 57 pages differed only in spaces or image names. +- 2 pages got back content that the original script had deleted. + +## Changes + +### Fixes + +- **Words were glued together** (`keepspaces = 1`). Non-breaking spaces were deleted, but web-clipped content often has them as the only space between two words: `theWindowskey` became `Windowskey`. 193 of 197 pages contained some. + - Now they are removed only at the start and end of a line, keeping the line's indentation. + - Anywhere else they become a single normal space. +- **Page content was deleted.** The first 6 lines of Pandoc's output were always dropped, on the assumption that they are title, date and time. Pages without those lines, such as emails sent to OneNote, lost real content: a Tags table, a Subject line, the first command of a page. + - The new `Remove-PageHeader` removes the header only when there really is a date and time line (in any language, including `9:19 AM`). + - Without that header, it only removes a first line that repeats the page title. +- **Duplicate page names overwrote each other.** The `-1`, `-2` postfix was added before the name was truncated to `mdFileNameAndFolderNameMaxLength`, so for long titles it was cut off again and the pages were written to the same file. + - Duplicates are now detected on the truncated name, which also catches long names that only differ after the cut-off. + - The postfix survives truncation (`Truncate-PathFileName -KeepSuffix`). +- **Attachment links were nested inside each other.** Inserted-file links were created with a plain text search. With attachments `11-36-00002.pdf` and `Datenblatt_11-36-00002.pdf` on one page, the first name also matched inside the second. The search now only matches a name that isn't part of a longer file name. +- **Image names broke Markdown links.** Image file names are built from the page path, which kept characters like `#` and `'` (e.g. `c't-expert-community--Newsletter-#10-image1.jpg`). A `#` in a link is read as a URL fragment. Image names now go through the same clean-up as inserted attachments (`Remove-InvalidFileNameCharsInsertedFiles`). +- **Section names contained OneNote escape codes.** OneNote reports characters that aren't allowed in section file names as `^` codes: `AI ^M ML` for `AI / ML`, `DECT^J Bluetooth` for `DECT & Bluetooth`. They ended up literally in folder names. `ConvertFrom-OneNoteEscapedName` decodes `^M` to `/` and `^J` to `&` for section, section group and notebook names; `/` then becomes `-` as usual. These are the only two codes seen so far. Others would still show up as `^`. +- **Wrong time zone offset.** `Created:` / `Modified:` used the format `zz00`, which only printed the hours of the offset (wrong for e.g. +05:30). The output format stays the same (`+0200`). + +### New option + +- **`overwrite`**: `2` removes a notebook's previous output before converting it (`Remove-PreviousNotebookOutput`), so notes, folders and media renamed or deleted in OneNote since the last run don't linger. + - The notebook's `docx` folder is always kept, so `usedocx = 2` can still reuse it. + - Only the notebook's own folder under `notesdestpath` is touched. + - Read-only files and paths over 260 characters are handled. + - The default is `1` (keep previous output), which is the old behaviour. + +### Not changed, for the record + +- **Wrong-looking Created/Modified times:** on some pages "Modified" is 1 to 2 hours *before* "Created". That's OneNote's own data, not a conversion error: those pages (probably created by the web clipper) store local time labelled as UTC in `dateTime`. The script converts correctly. +- **`>` removed at the start of lines** (`keepspaces = 1`): deliberate. Pandoc renders OneNote's indented paragraphs as block quotes, and this un-indents them. + +## Sent upstream + +On 2026-09-24 all of the changes above were offered to the original project as five independent pull requests, #191 to #195. They come from the fork https://github.com/klangme1ster/ConvertOneNote2MarkDown, whose branches are `origin/` in this repository. Longer PR texts, with test details, are in `docs/pr-descriptions.md`. + +| PR | Branch | Contains | +|---|---|---| +| [#191](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/191) | `fix/keepspaces-keep-non-breaking-spaces-between-words-as-spaces` | Words glued together | +| [#192](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/192) | `fix/convert-remove-page-header-only-when-present` | Page content deleted, wrong time zone offset | +| [#193](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/193) | `fix/file-names-keep-duplicate-postfix-and-avoid-broken-links` | Duplicate page names, nested attachment links, image names | +| [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | `fix/file-names-decode-onenote-escape-codes-in-section-names` | Section names with OneNote escape codes | +| [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | `enhancement/configuration-add-option-overwrite-to-remove-previous-output` | The `overwrite` option | + +The PR branches differ from this copy in two small ways, so that each branch works and merges on its own: + +- The new functions and tests are placed at different spots in the files. +- In the `overwrite` branch, `Remove-PreviousNotebookOutput` doesn't call `ConvertFrom-OneNoteEscapedName`. + +Not in any PR: this file, the README pointer to it, and `config.ps1`. + +If upstream merges the PRs, this copy can be replaced by upstream's version (keep `config.ps1`). + +## Recommended configuration + +In `config.ps1`: + +- `$keepescape = 3`: keep all backslashes. The default `1` also deletes the ones in Windows paths (`C:\Program Files` → `C:Program Files`), and `2` turns escaped `\*` and `\` back into formatting and hidden HTML. +- `$conversion = 'markdown-simple_tables-multiline_tables-grid_tables+pipe_tables-smart'`: `-smart` stops Pandoc from escaping every quote (`\"`). Don't add `-link_attributes` or `-bracketed_spans`: Pandoc then writes raw `` and `` HTML instead. +- `$overwrite = 2` when re-exporting into an existing folder. + +## Development notes + +- **Keep both `.ps1` files pure ASCII.** Windows PowerShell 5.1 reads scripts without a BOM as ANSI, so a UTF-8 character (e.g. a literal non-breaking space in a regex) breaks parsing or silently changes the pattern. Write `\u00A0` in regexes and `[char]0x00A0` in strings. Check with `grep -c '[^ -~ ]' *.ps1`, which should print `0`. +- **Running the tests without installing anything:** `test/test.ps1` installs Pester 4 into the user profile. Instead: + + ```powershell + Save-PSResource -Name Pester -Version 4.10.1 -Path -TrustRepository # PowerShell 7 + # then, in Windows PowerShell 5.1: + Import-Module \Pester\4.10.1\Pester.psd1 -Force + Invoke-Pester -Script . -Tag Unit + ``` + + Don't set `$ErrorActionPreference = 'Stop'` for the run: several tests expect non-terminating errors. +- **Test status:** 92 unit tests. The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes. diff --git a/README.md b/README.md index 5edc630..236e788 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![github-actions](https://github.com/theohbrothers/ConvertOneNote2MarkDown/actions/workflows/ci-master-pr.yml/badge.svg?branch=master)](https://github.com/theohbrothers/ConvertOneNote2MarkDown/actions/workflows/ci-master-pr.yml) [![github-release](https://img.shields.io/github/v/release/theohbrothers/ConvertOneNote2MarkDown?style=flat-square)](https://github.com/theohbrothers/ConvertOneNote2MarkDown/releases/) +> **Local copy:** this copy has fixes and an extra `overwrite` option on top of upstream; see [LOCAL-CHANGES.md](LOCAL-CHANGES.md). + Ready to make the step to Markdown and saying farewell to your OneNote, EverNote or whatever proprietary note taking tool you are using? Nothing beats clear text, right? Read on! The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote Object Model on your workstation to convert all OneNote pages to Word documents and then utilizes Pandoc to convert the Word documents to Markdown (`.md`) format. diff --git a/docs/TRANSCRIPTION-QUALITY-REPORT.md b/docs/TRANSCRIPTION-QUALITY-REPORT.md new file mode 100644 index 0000000..733b8b3 --- /dev/null +++ b/docs/TRANSCRIPTION-QUALITY-REPORT.md @@ -0,0 +1,81 @@ +# Problem report: Markdown transcription quality + +Written 2026-11-01 from the Markdown cleanup side (`c:\Users\roman\SynologyDrive\Code\py\markdown_cleaner`). +This report is evidence plus a proposal. Nothing in this folder has been changed. + +## Setup the findings come from + +- `config.ps1`: `$keepescape = 3` (keep all Pandoc escapes), `$keepspaces = 1`, + `$conversion = 'markdown-simple_tables-multiline_tables-grid_tables+pipe_tables-smart'`. +- Output scanned: 392 `.md` notes in `c:\Users\roman\SynologyDrive\Dokumente\onenote2md` (read only). +- Example page: `Notizbuch-von-Roman\Teufel\JIRA-Prozess-Wiki.md` (list-heavy, badly affected). +- Not verified: whether the `.docx` of these pages still exists (`usedocx = 2`); I could not find one. + The Pandoc statements below are from reading the converter code and from Pandoc's known writer behaviour, + not from re-running a conversion. + +## Findings + +Counts are pages affected / occurrences, from a regex scan of the 392 notes. + +| # | Artifact in the output | Pages / occurrences | Effect when rendered | +|---|---|---|---| +| 1 | Escaped list marker at line start: `\- item` | 45 / 597 | Literal "- item" paragraphs, not a list | +| 2 | List items separated by a blank line (loose, mostly because of #1) | 65 / 369 | Wide, spaced list; in source one blank line per item | +| 3 | Line-end backslash (Pandoc `escaped_line_breaks`) | 89 / 1810 | `\` is a hard break only in CommonMark-style renderers; at the end of a paragraph it renders as a literal `\` | +| 4 | Lone `-` or `\-` line | 23 / 35 | Empty bullet; a bare `-` directly below a text line is a setext underline and turns that line into a heading | +| 5 | Escaped number: `7\.` | 22 / 133 | Not a numbered list; in `JIRA-Prozess-Wiki.md` the number even sits alone in a paragraph, with the item text in the next one | +| 6 | Needless escapes: `\>` (142 pages / 598), `\*` (33 / 489), `\_` (19 / 182) | | Noise in the source (`--\>` for `-->`, `snake\_case`) | + +## Cause (read from the code) + +1. **`keepspaces = 1` cannot see escaped markers.** + The list regexes in `ConvertOneNote2MarkDown-v2.ps1` (around line 932) are + `(\s*)- ([^\r\n]*)\r*\n\r*\n(?=\s*-)` and the numbered variant. With `keepescape = 3` the next item starts with `\-`, + so the look-ahead `(?=\s*-)` fails and the blank line stays. The same applies to `\d+\.` vs `\d+\\\.`. +2. **`\-` and `7\.` come from paragraphs that start with literal text "- " or "7."**, not from real OneNote lists + (a real list would come out of Pandoc as `- item`). Pandoc is faithful here and escapes the text so it does not + turn into a list. The converter could recognise this pattern and rebuild a real list. This is the main transcription gap. + To confirm, run Pandoc on the `.docx` of one of the affected pages and look at the paragraph styles. +3. **`keepescape` offers only coarse choices.** + `1` deletes every `\` in the file (`[regex]::Escape('\')` → empty). That also removes backslashes that carry + meaning: Windows paths, UNC paths (`\\server\share`), a deliberately escaped `\*`, and the `\` hard breaks. + `2` keeps escapes before alphanumerics only. `3` keeps everything, which is where the artifacts above come from. +4. **The `\` hard break** is Pandoc's `escaped_line_breaks` extension, enabled in the default `markdown` flavour. + Pandoc documents that disabling it (`markdown-escaped_line_breaks`) makes the writer emit two trailing spaces + instead. I did not test this against a real page. + +## Proposal + +Add a fourth `keepescape` mode (or make it the default) that removes only escapes which cannot change the rendering, +and rebuilds lists from escaped markers. A working Python reference implementation with 24 unit tests exists: + +- `c:\Users\roman\SynologyDrive\Code\py\markdown_cleaner\qon_helper.py` + - `escapes`: unescapes `_` `*` `.` `-` `+` `=` `#` `>` `!` and a few more only where that cannot change the + rendering (not at line start, not before `[`, `*` only when it cannot pair). + - `fix_lists`: `\- ` → `- `, `7\.` → `7.` (also when the number stands alone before its text), + drops lone `-` lines, drops blank lines between list items, turns line-end `\` into two trailing spaces + or removes it at paragraph end. Code fences, inline code and front matter are skipped. + - `repair` = `fix_lists` + escapes + collapse blank lines + hard breaks. +- Dry run of `repair` over the 392 notes: 245 pages changed, 0 errors; `repair` run twice differs in 4 pages + (whitespace only). +- Porting the rules to PowerShell (as post-processing in the existing replacement list) is the minimal change. + Alternative: run the Python filter as a post-step. That adds a Python dependency to this tool, so the + PowerShell port is probably the better fit. + +Also worth checking in the converter: + +- The first lines of many pages are `| **Tags:** #Teufel #Prozess |` followed by `|---|`, a one-cell table + with only a header row. I did not find where it is produced. If it is meant to be a tags line, a plain line + would render better and QOwnNotes could read the tags. +- `keepspaces` also strips `>` after lists. That is documented as deliberate; leave it. + +## Acceptance criteria + +- Re-export of the 392-page sample (or at least `JIRA-Prozess-Wiki.md`): + the scan counts for findings 1, 2, 4, 5 drop to zero; #3 and #6 drop to the genuinely needed cases. +- No intended backslash is lost (Windows paths, `\\server\share`, `\*` in math or code-like text). + Add Pester cases for these. +- Existing Pester tests stay unchanged; new behaviour gets new tests. +- Default behaviour stays the same unless the user picks the new mode (it changes rendering for pages that + start lines with "- "). +- Record the change in `LOCAL-CHANGES.md`. diff --git a/docs/pr-descriptions.md b/docs/pr-descriptions.md new file mode 100644 index 0000000..91d475a --- /dev/null +++ b/docs/pr-descriptions.md @@ -0,0 +1,113 @@ +# Pull request descriptions + +Paste each title and description into GitHub's "Open a pull request" form for the matching branch. + +All five PRs are independent and merge cleanly in any order and combination. On a German Windows, one existing test ("Throw if -Length is greater than 255") fails with or without these changes: it expects the English .NET error message. + +--- + +## 1. Branch `fix/keepspaces-keep-non-breaking-spaces-between-words-as-spaces` + +**Title:** Fix (keepspaces): Keep non-breaking spaces between words as spaces + +**Description:** + +With `keepspaces = 1` (the default), every non-breaking space is removed. Content clipped from web pages often has a non-breaking space as the *only* space between two words, so words get glued together. For example, a page containing `press the Windowskey at the same timeto open` came out as `press the Windowskey at the same timeto open`. In my notebooks, 193 of 197 pages contained non-breaking spaces (4306 in total). + +This PR changes the rule: + +- **Start of a line:** removed, keeping the line's indentation. Turning them into spaces there could create an indented code block. +- **End of a line, and lines that contain only non-breaking spaces:** removed. Turning them into spaces there could create a trailing-double-space hard line break. +- **Anywhere else:** a run of them, together with adjacent spaces, becomes a single space. + +The option's description and the README are updated to match. The `keepspaces` test now also covers words separated by non-breaking spaces, a line that starts with them, and an indented line. + +--- + +## 2. Branch `fix/convert-remove-page-header-only-when-present` + +**Title:** Fix (convert): Remove the page's title, date and time lines only when present + +**Description:** + +`Convert-OneNotePage` always discards the first 6 lines of pandoc's output, assuming they are the page's title, date and time. Some pages don't have these, for example emails sent to OneNote from Outlook. On those pages, real content was lost. In my notebooks this happened to two pages: + +- one lost a `Tags` table and its `Subject:` line +- the other lost its first line, a `curl` command + +This PR adds `Remove-PageHeader`, which only removes the header when it's really there: + +- **Title, date, time:** removed when there is a date line (any text with a 4-digit year, so it works in any language, e.g. `Donnerstag, 17. Oktober 2024` or `Monday, September 8, 2014`) followed by a time line (`09:50`, `9:19 AM`). The title may be empty; an untitled page yields a line with only a non-breaking space. +- **No date/time:** only a first line that repeats the page's name is removed. The comparison uses letters and digits only, because pandoc escapes and autolinks the title. +- **Anything else:** kept as is. + +I checked this against pandoc's output for all 197 pages of one of my notebooks. The header was recognized on all pages that have one, and the two pages without one kept their content. + +**Also included**, since it's about the same header: the `Created:` / `Modified:` lines used the format `zz00`, which only prints the hours of the UTC offset. That is wrong for time zones like +05:30. The output format doesn't change (`+0200`). + +New unit tests cover `Remove-PageHeader`: German and English headers, a title containing a year, an untitled page, a page without a header, a title-only page, a header-only page, and empty input. + +--- + +## 3. Branch `fix/file-names-keep-duplicate-postfix-and-avoid-broken-links` + +**Title:** Fix (file names): Keep duplicate page postfixes and avoid broken attachment and image links + +**Description:** + +Three related problems with generated file names and links. + +**Duplicate page names overwrite each other.** The `-1`, `-2` postfix for identically named pages in a section is appended *before* the name is truncated to `mdFileNameAndFolderNameMaxLength`. For long page names, truncation cuts the postfix off again, and the pages are written to the same file. Now: + +- duplicates are detected on the truncated name, which also catches long names that only differ after the cut-off +- the postfix survives truncation, via a new `Truncate-PathFileName -KeepSuffix` parameter + +**Nested attachment links.** Inserted-file references are replaced with a plain text search. A page with the attachments `11-36-00002.pdf` and `Datenblatt_11-36-00002.pdf` ended up with a broken, nested link: `[Datenblatt_[11-36-00002.pdf](media/11-36-00002.pdf)](media/Datenblatt_[11-36-00002.pdf](media/11-36-00002.pdf))`. The search now only matches a name that isn't part of a longer file name. + +**Broken image links.** Image file names are prefixed with the page path, which can contain characters like `#` and `'`. For example, a page named `c't expert community: Newsletter #10` produces `c't-expert-community--Newsletter-#10-image1.jpg`. In a Markdown link, `#` starts a URL fragment, so the image doesn't load. Image names now go through `Remove-InvalidFileNameCharsInsertedFiles`, like inserted attachments already do. + +New unit tests: `-KeepSuffix` in both cases, long duplicate page names (including two names that only differ after the cut-off), and an attachment name contained in another attachment's name. + +--- + +## 4. Branch `fix/file-names-decode-onenote-escape-codes-in-section-names` + +**Title:** Fix (file names): Decode OneNote's escape codes in section names + +**Description:** + +OneNote reports characters that aren't allowed in section file names as `^` codes in the section's name, for example: + +- `AI ^M ML` for a section named `AI / ML` +- `DECT^J Bluetooth` for `DECT & Bluetooth` + +These codes ended up literally in folder names (`AI-^M--ML`, `DECT^J-Bluetooth`). + +This PR adds `ConvertFrom-OneNoteEscapedName`, which decodes `^M` to `/` and `^J` to `&`, the two codes I've seen. It runs before `Remove-InvalidFileNameChars` for notebook, section group and section names, so `/` then becomes `-` as usual. Other codes, if they exist, are left as they are. + +If the `overwrite` PR (#5 in this series) is merged too, `Remove-PreviousNotebookOutput` should use the same decoding for the notebook's directory name, so both agree for notebook names that contain these codes. + +New unit tests cover the decoding and the resulting directory name. + +--- + +## 5. Branch `enhancement/configuration-add-option-overwrite-to-remove-previous-output` + +**Title:** Enhancement (configuration): Add option `overwrite` to remove a notebook's previous output + +**Description:** + +When converting into an existing notes folder, files of the same name are overwritten. But notes, folders and media that were renamed or deleted in OneNote since the last run are left behind, and they're easy to mistake for current notes. + +This PR adds the option `overwrite`: + +- **`1` (default):** current behavior. +- **`2`:** before converting a notebook, its previous output is removed (`Remove-PreviousNotebookOutput`). + - The notebook's `docx` folder is kept, so `usedocx = 2` can still reuse it. + - Only the notebook's own folder under `notesdestpath` is touched, and nothing happens on a dry run. + - Read-only files (e.g. inserted attachments, which OneNote marks read-only) are removed too. + - On Windows PowerShell 5, paths longer than 260 characters are handled via the `\\?\` prefix. + +This overlaps in purpose with #190 (`skipIfExists`), which is about re-running conversions too. The two options don't conflict. + +New unit tests run against real files in `TestDrive`: removal while keeping `docx`, including a read-only file; a dry run; and a notebook folder that doesn't exist. I also used it on two notebooks (393 pages, with long paths): a planted stale note was removed, and both `docx` folders were kept. From ee23b44841dd34b256415d063f1e9cb97587ffc4 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:33:49 +0100 Subject: [PATCH 07/12] Enhancement (configuration): Add keepescape mode 4 to repair pandoc escapes OneNote pages often start paragraphs with a literal '- ' or '7.'. Pandoc escapes them ('\- item', '7\. item'), so the notes lose their lists and carry loose blank lines, line-end backslashes and needless escapes. Mode 4 (Repair-MarkdownEscapes) rebuilds those lists (also inside block quotes and after non-breaking spaces), turns a line-end backslash into two spaces, and removes only escapes that cannot change the rendering. Code fences and inline code are untouched. Modes 1 to 3 and the default are unchanged. Also adds docs (ideas, known issues, ADR 0001) and the quality report. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 108 +++++++++++++++++++++++++++ ConvertOneNote2MarkDown-v2.ps1 | 107 +++++++++++++++++++++++++- LOCAL-CHANGES.md | 12 ++- config.example.ps1 | 1 + docs/TRANSCRIPTION-QUALITY-REPORT.md | 11 +++ docs/adr/0001-keepescape-mode-4.md | 20 +++++ docs/ideas.md | 5 ++ docs/known-issues.md | 3 + 8 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 docs/adr/0001-keepescape-mode-4.md create mode 100644 docs/ideas.md create mode 100644 docs/known-issues.md diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 91e9f6e..e506dea 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -492,6 +492,93 @@ Describe "Remove-PageHeader" -Tag 'Unit' { } +Describe "Repair-MarkdownEscapes" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should turn escaped list markers into lists ()" -TestCases @( + @{ Name = 'bullet'; Content = '\- one'; Expected = '- one' } + @{ Name = 'indented bullet'; Content = ' \- one'; Expected = ' - one' } + @{ Name = 'number'; Content = '7\. one'; Expected = '7. one' } + @{ Name = 'number alone before its text'; Content = "7\.`n`nitem text"; Expected = '7. item text' } + ) { + param ($Name, $Content, $Expected) + + Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Expected + } + + It "Should accept a non-breaking space after an escaped list marker" { + $nbsp = [char]0x00A0 + Repair-MarkdownEscapes -Content "\-$nbsp one`n7\.$nbsp two" | Should -BeExactly "- one`n7. two" + } + + It "Should repair lists inside block quotes (keepspaces removes the '>' later)" { + Repair-MarkdownEscapes -Content "> 1\. one`n>`n> 2\. two`n>`n> \- x" | Should -BeExactly "> 1. one`n> 2. two`n> - x" + } + + It "Should drop a lone dash line that only had a line-end backslash" { + Repair-MarkdownEscapes -Content "- one`n-\`n`n`n" | Should -BeExactly "- one`n`n`n" + } + + It "Should remove blank lines between list items" { + Repair-MarkdownEscapes -Content "\- a`n`n\- b`n`n\- c" | Should -BeExactly "- a`n- b`n- c" + Repair-MarkdownEscapes -Content "1\. a`n`n2\. b" | Should -BeExactly "1. a`n2. b" + } + + It "Should remove several blank lines, also with non-breaking spaces, between list items" { + Repair-MarkdownEscapes -Content "- a`n`n`n$( [char]0x00A0 )`n`n- b" | Should -BeExactly "- a`n- b" + Repair-MarkdownEscapes -Content "1. a`n`n`n`n2. b`n`ntext" | Should -BeExactly "1. a`n2. b`n`ntext" + Repair-MarkdownEscapes -Content "- a`n`n> $( [char]0x00A0 ) `n`n- b" | Should -BeExactly "- a`n- b" + } + + It "Should keep blank lines between a list and a paragraph" { + Repair-MarkdownEscapes -Content "- a`n`ntext" | Should -BeExactly "- a`n`ntext" + } + + It "Should remove lone dash lines, also with non-breaking spaces" { + Repair-MarkdownEscapes -Content "text`n-`n\-`n-$( [char]0x00A0 )`nmore" | Should -BeExactly "text`nmore" + } + + It "Should turn a backslash at the end of a line into two spaces, and drop it at the end of a paragraph" { + Repair-MarkdownEscapes -Content "one\`ntwo\`n`nnext" | Should -BeExactly "one `ntwo`n`nnext" + } + + It "Should remove escapes that can't change the rendering" { + Repair-MarkdownEscapes -Content 'a --\> b, snake\_case, 1\. no, end\.' | Should -BeExactly 'a --> b, snake_case, 1. no, end.' + } + + It "Should keep escapes that change the rendering ()" -TestCases @( + @{ Name = 'quote at line start'; Content = '\> not a quote' } + @{ Name = 'underscore at line start'; Content = '\_not emphasis_' } + @{ Name = 'image-like'; Content = 'a \![b](c)' } + @{ Name = 'pair of asterisks'; Content = 'a \*b\* c' } + @{ Name = 'brackets'; Content = '\[x\]' } + ) { + param ($Name, $Content) + + Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Content + } + + It "Should keep Windows and UNC paths" { + $content = 'C:\Program Files\App and \\server\share\dir' + Repair-MarkdownEscapes -Content $content | Should -BeExactly $content + } + + It "Should not touch code" { + $content = "``\- keep \_this`` and`n`n``````text`n\- keep`n7\. keep\`n``````" + Repair-MarkdownEscapes -Content $content | Should -BeExactly $content + } + + It "Should give the same result when run twice" { + $content = "\- a`n`n\- b\`nc`n`n7\. x --\> y\_z\`n`nend\" + $once = Repair-MarkdownEscapes -Content $content + Repair-MarkdownEscapes -Content $once | Should -BeExactly $once + } + + } + +} + Describe "New-OneNoteConnection" -Tag 'Unit' { Context 'Behavior' { @@ -1525,6 +1612,27 @@ foo\bar\0 } } + It "Should leave backslashes to Repair-MarkdownEscapes if config keepescape is 4" { + $params['Config']['keepescape']['value'] = 4 + + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + $result.Count | Should -Be 48 + + foreach ($pageCfg in $result) { + $fakeMarkdownContent = 'hello\$\^\\\*\_\[\]world' + + # Mutate + foreach ($m in $pageCfg['mutations']) { + foreach ($r in $m['replacements']) { + $fakeMarkdownContent = $fakeMarkdownContent -replace $r['searchRegex'], $r['replacement'] + } + } + + # None of the mutations may touch backslashes + $fakeMarkdownContent | Should -Match ([regex]::Escape('hello\$\^\\\*\_\[\]world')) + } + } + It "Should honor config newlineCharacter" { $params['Config']['newlineCharacter']['value'] = 1 diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index c2e1b56..130a21d 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -166,10 +166,11 @@ Whether to clear escape symbols from md files. See: https://pandoc.org/MANUAL.ht 1: Clear all '\' characters - Default 2: Clear all '\' characters except those preceding alphanumeric characters 3: Keep '\' symbol escape +4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter '@ default = 1 value = 1 - validateRange = 1,3 + validateRange = 1,4 } newlineCharacter = @{ description = @' @@ -580,6 +581,107 @@ Function Remove-PageHeader { } } +Function Repair-MarkdownEscapes { + [CmdletBinding()] + param ( + # The markdown content produced by pandoc (any newline style) + [Parameter(Mandatory,Position = 0,ValueFromPipeline = $true)] + [AllowEmptyString()] + [string] + $Content + ) + + process { + # Pandoc escapes text that merely looks like markdown. OneNote pages often start paragraphs with a literal '- ' or '7.' (no real list), + # which ends up as '\- item' or '7\. item'. Rebuild those lists, and remove only the escapes that can't change how the text is rendered. + # Fenced code blocks and inline code are never touched. + $saved = New-Object 'System.Collections.Generic.List[string]' + $shield = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + $saved.Add($m.Value) + "MDSHIELD$( $saved.Count - 1 )END" + } + $text = $Content -replace '\r\n', "`n" + $text = [regex]::Replace($text, '(?ms)(?:^[ \t]*)?(`{3,})[^\n`]*\n.*?^[ \t]*\1`*[ \t]*$', $shield) + $text = [regex]::Replace($text, '(`+)(?!`)(.+?)(?]*\\?-[ \t\u00A0]*$\n?', '' # A lone '-' or '\-' line is an empty bullet (below a text line, '-' would even make a heading) + $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*)\\-[ \t\u00A0]+', '$1- ' + $text = $text -replace '(?m)^(\d+)\\\.[ \t]*\n(?:[ \t]*\n)+(?=\S)', '$1. ' # A number alone on its line, with the item text in the next paragraph + $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*\d+)\\\.[ \t\u00A0]+', '$1. ' + $listStart = '^[ \t\u00A0>]*(?:[-*+]|\d+[.)])\s' + $lines = $text -split "`n" + $isBlank = '^[ \t\u00A0>]*$' # Lines with only non-breaking spaces or an empty block quote ('> ') become blank later on (keepspaces) + $text = @( + for ($i = 0; $i -lt $lines.Count; $i++) { + # Drop the blank lines between two list items + if ($lines[$i] -match $isBlank -and $i -gt 0) { + $prev = $i - 1 + while ($prev -gt 0 -and $lines[$prev] -match $isBlank) { $prev-- } + $next = $i + 1 + while ($next -lt $lines.Count - 1 -and $lines[$next] -match $isBlank) { $next++ } + if ($lines[$prev] -match $listStart -and $next -lt $lines.Count -and $lines[$next] -match $listStart) { + continue + } + } + $lines[$i] + } + ) -join "`n" + + # Escapes that can't change the rendering + $unescape = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + if ($m.Value -eq '\\') { return $m.Value } # An escaped backslash is matched as a pair, so it can't hide the character after it + $c = $m.Value[1] + $start = $m.Index + $lineStart = if ($start -gt 0) { $text.LastIndexOf([char]10, $start - 1) + 1 } else { 0 } + $prefix = $text.Substring($lineStart, $start - $lineStart) + $before = if ($start -gt 0) { [string]$text[$start - 1] } else { '' } + $afterIndex = $start + 2 + $after = if ($afterIndex -lt $text.Length) { [string]$text[$afterIndex] } else { '' } + $atLineStart = $prefix.Trim(" `t>").Length -eq 0 + if (',."''%@/?!=+-#>'.Contains($c)) { + $keep = ('=+-#>'.Contains($c) -and $atLineStart) -or ($c -eq '.' -and $prefix -match '^[ \t>]*\d+$') -or ($c -eq '!' -and $after -eq '[') + if (!$keep -and $c -eq '#') { + $lineEnd = $text.IndexOf([char]10, $start) + $line = if ($lineEnd -lt 0) { $text.Substring($lineStart) } else { $text.Substring($lineStart, $lineEnd - $lineStart) } + $keep = $line -match '^\s*#{1,6}(?:\s|$)' # A closing '\#' in a heading stays + } + }elseif ($c -eq '*' -or $c -eq '_') { + $loneGap = ($before -eq '' -or $before -match '^\s$') -and ($after -eq '' -or $after -match '^\s$') + $intraword = $before -match '^[\p{L}\p{N}]$' -and $after -match '^[\p{L}\p{N}]$' + if ($c -eq '*') { + $pStart = if ($start -ge 2) { $text.LastIndexOf("`n`n", $start - 1, [System.StringComparison]::Ordinal) } else { -1 } + $pEnd = $text.IndexOf("`n`n", $start, [System.StringComparison]::Ordinal) + $from = if ($pStart -ge 0) { $pStart + 2 } else { 0 } + $to = if ($pEnd -ge 0) { $pEnd } else { $text.Length } + $paragraph = $text.Substring($from, $to - $from) + $keep = $atLineStart -or !($loneGap -or ($paragraph.Length - $paragraph.Replace('*', '').Length) -eq 1) + }else { + $keep = $atLineStart -or !($loneGap -or $intraword) + } + }else { + $keep = $true + } + if ($keep) { $m.Value } else { [string]$c } + } + $text = [regex]::Replace($text, '\\\\|\\.', $unescape) + + # Restore code (fences can contain inline code and vice versa, so repeat until nothing is left) + $restore = [System.Text.RegularExpressions.MatchEvaluator]{ param ($m) $saved[[int]$m.Groups[1].Value] } + while ($text -match 'MDSHIELD\d+END') { + $text = [regex]::Replace($text, 'MDSHIELD(\d+)END', $restore) + } + $text + } +} + Function Set-ContentNoBom { [CmdletBinding()] param ( @@ -1416,6 +1518,9 @@ Function Convert-OneNotePage { $content = @( Get-Content -LiteralPath $pageCfg['filePath'] -ErrorAction Stop ) # Use -LiteralPath so that characters like '(', ')', '[', ']', '`', "'", '"' are supported. Or else we will get an error "Cannot find path 'xxx' because it does not exist" # Discard the page's title, date and time lines (if present). We are going to add our own header $content = @( Remove-PageHeader -Lines $content -Title $pageCfg['object'].name ) -join "`n" + if ($config['keepescape']['value'] -eq 4) { + $content = Repair-MarkdownEscapes -Content $content + } } # Mutate diff --git a/LOCAL-CHANGES.md b/LOCAL-CHANGES.md index d97e30e..00f05d3 100644 --- a/LOCAL-CHANGES.md +++ b/LOCAL-CHANGES.md @@ -34,6 +34,14 @@ All changes were checked against the real notebooks by converting them from the - Read-only files and paths over 260 characters are handled. - The default is `1` (keep previous output), which is the old behaviour. +- **`keepescape = 4`**: keeps a `\` only where it changes the rendering (`Repair-MarkdownEscapes`). Default behaviour (`1`) is unchanged. + - Pandoc escapes paragraphs that only look like lists. OneNote pages often start lines with a literal `- ` or `7.` (no real list: the `.docx` has no list numbering), which came out as `\- item` and `7\. item`. These become real lists again (`- item`, `7. item`), blank lines between the items are removed, and lone `-` lines (empty bullets, or an accidental heading underline) are dropped. + - A `\` at the end of a line (Pandoc's hard break) becomes two trailing spaces, or is dropped at the end of a paragraph. + - Escapes that cannot change the rendering (`\>` in `--\>`, `\_` in `snake\_case`, `\.`, `\-` in the middle of a line) are removed. Kept: escapes at the start of a line, `\[`, `\]`, `\*` pairs, `\\`, Windows and UNC paths. + - Code fences and inline code are never touched. + - Port of `fix_lists` and `escapes` from `markdown_cleaner/qon_helper.py`, extended for what real OneNote exports showed: lists inside block quotes (`> 1. item`; `keepspaces` removes the `>` only later), non-breaking spaces after the marker, and several blank or nbsp-only lines between items. + - Checked with a real run of the script against OneNote (notebook with 197 pages, `keepescape` 3 vs 4, same `.docx`): escaped bullets 503 to 0, blank lines between list items 45 to 0, line-end backslashes 1039 to 0, lone dash lines 25 to 0, escaped numbers 87 to 0. `\>` 316 to 25, `\*` 366 to 330 and `\_` 9 to 9 remain on purpose (they sit at line starts or pair up). Letters and digits are identical on all 197 pages, so no text was lost. + ### Not changed, for the record - **Wrong-looking Created/Modified times:** on some pages "Modified" is 1 to 2 hours *before* "Created". That's OneNote's own data, not a conversion error: those pages (probably created by the web clipper) store local time labelled as UTC in `dateTime`. The script converts correctly. @@ -64,7 +72,7 @@ If upstream merges the PRs, this copy can be replaced by upstream's version (kee In `config.ps1`: -- `$keepescape = 3`: keep all backslashes. The default `1` also deletes the ones in Windows paths (`C:\Program Files` → `C:Program Files`), and `2` turns escaped `\*` and `\` back into formatting and hidden HTML. +- `$keepescape = 4` (new, see above) keeps only the backslashes that matter. Before that: `3` keeps all backslashes, but leaves `\- item`, `7\. item` and `\` line ends in the output. The default `1` also deletes the ones in Windows paths (`C:\Program Files` → `C:Program Files`), and `2` turns escaped `\*` and `\` back into formatting and hidden HTML. - `$conversion = 'markdown-simple_tables-multiline_tables-grid_tables+pipe_tables-smart'`: `-smart` stops Pandoc from escaping every quote (`\"`). Don't add `-link_attributes` or `-bracketed_spans`: Pandoc then writes raw `` and `` HTML instead. - `$overwrite = 2` when re-exporting into an existing folder. @@ -81,4 +89,4 @@ In `config.ps1`: ``` Don't set `$ErrorActionPreference = 'Stop'` for the run: several tests expect non-terminating errors. -- **Test status:** 92 unit tests. The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes. +- **Test status:** 114 unit tests (22 of them new for `keepescape = 4`). The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes. diff --git a/config.example.ps1 b/config.example.ps1 index 6077bdf..72d6a8f 100644 --- a/config.example.ps1 +++ b/config.example.ps1 @@ -77,6 +77,7 @@ $keepspaces = 1 # 1: Clear all '\' characters - Default # 2: Clear all '\' characters except those preceding alphanumeric characters # 3: Keep '\' symbol escape +# 4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter $keepescape = 1 # Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new lines diff --git a/docs/TRANSCRIPTION-QUALITY-REPORT.md b/docs/TRANSCRIPTION-QUALITY-REPORT.md index 733b8b3..94470a8 100644 --- a/docs/TRANSCRIPTION-QUALITY-REPORT.md +++ b/docs/TRANSCRIPTION-QUALITY-REPORT.md @@ -79,3 +79,14 @@ Also worth checking in the converter: - Default behaviour stays the same unless the user picks the new mode (it changes rendering for pages that start lines with "- "). - Record the change in `LOCAL-CHANGES.md`. + +## Addendum 2026-11-01: verification and result + +Checked with Pandoc 3.11 on the `.docx` of `JIRA-Prozess-Wiki` (kept in `c:\temp\notes\\docx`) and on all 197 `.docx` of the notebook: + +- The `.docx` has no list numbering (`w:numPr` count 0, all paragraphs style `StandardWeb`). The `\-` and `7\.` are therefore literal text typed in OneNote; Pandoc is faithful and escapes it. This confirms cause 2. +- `-escaped_line_breaks` in `$conversion` does turn the line-end `\` into two trailing spaces (confirmed). Not needed any more with `keepescape = 4`. +- The one-cell `Tags` table is part of the OneNote content (it is in the Pandoc output of the `.docx`); the script does not produce it. +- Implemented as `keepescape = 4` (`Repair-MarkdownEscapes`, Pester tests added). First version was identical to `qon_helper.escapes(fix_lists(...))` on all 197 `.docx` pages. A real run against OneNote then showed three gaps, now fixed: lists inside block quotes (`> 1\. item`), non-breaking spaces after the marker, and several blank or nbsp-only lines between items. +- Real run (script + OneNote, same notebook, `keepescape` 3 vs 4, 197 pages, no errors), scan counts before/after: escaped bullets 503 to 0, blank lines between list items 45 to 0, line-end backslashes 1039 to 0, lone dash lines 25 to 0, escaped numbers 87 to 0, `\>` 316 to 25, `\*` 366 to 330, `\_` 9 to 9 (the rest is kept on purpose). Letters and digits are identical on every page. +- Not yet done: re-export into the real destination (`onenote2md`) and a look at the rendering in QOwnNotes; the `Tags` table is unchanged. diff --git a/docs/adr/0001-keepescape-mode-4.md b/docs/adr/0001-keepescape-mode-4.md new file mode 100644 index 0000000..9e691f0 --- /dev/null +++ b/docs/adr/0001-keepescape-mode-4.md @@ -0,0 +1,20 @@ +# 0001: keepescape mode 4 (repair pandoc escapes) + +## Status +Proposed (2026-11-01) + +## Context +With `keepescape = 3` the exported Markdown keeps every escape that Pandoc writes. OneNote pages often contain paragraphs that start with a literal `- ` or `7.`; Pandoc escapes them (`\- item`, `7\. item`), so the notes render as plain paragraphs instead of lists, with blank lines between the items. Pandoc's line-end `\` and many needless escapes (`\>`, `\_`) add noise. `keepescape = 1` removes every backslash, which also destroys Windows paths and intended escapes; `2` and `3` don't fix the list problem. See `docs/TRANSCRIPTION-QUALITY-REPORT.md`. + +## Decision +Add `keepescape = 4`, implemented as the function `Repair-MarkdownEscapes` that runs on the Pandoc output before the existing mutations. It rebuilds lists, converts line-end backslashes to two spaces, and removes only escapes that cannot change the rendering. Code is shielded. The default stays `1`; modes 1 to 3 are untouched. + +Rejected alternatives: +- Regex entries in the mutation list like modes 1 and 2: they cannot skip code blocks and inline code. +- Running the Python `markdown_cleaner` as a post-step: adds a Python dependency to this tool. +- Only `-escaped_line_breaks` in `$conversion`: fixes the line-end backslash, nothing else. + +## Consequences +- Opt-in, so no breaking change for existing users. Notes exported with mode 4 render lists as intended. +- The rules exist twice (here and in `qon_helper.py`); on all 197 pages of the test notebook both give identical output. Changes to one should be mirrored. +- Pages where a literal `- ` was meant as text, not as a list, become lists. diff --git a/docs/ideas.md b/docs/ideas.md new file mode 100644 index 0000000..c3b6f5d --- /dev/null +++ b/docs/ideas.md @@ -0,0 +1,5 @@ +# Ideas + +- [ ] 2026-11-01: One-cell `| **Tags:** ... |` table at page start - comes from the OneNote content itself (pandoc output of the .docx), not from this script; decide whether to turn it into a plain tags line +- [ ] 2026-11-01: Re-export the notebooks with `$keepescape = 4` and compare against the 392 notes in onenote2md +- [ ] 2026-11-01: Offer keepescape mode 4 upstream as a further PR (#196) once it has proven itself locally diff --git a/docs/known-issues.md b/docs/known-issues.md new file mode 100644 index 0000000..a905cde --- /dev/null +++ b/docs/known-issues.md @@ -0,0 +1,3 @@ +# Known issues + +- 2026-09-24 | Tests: Pester test "Throw if -Length is greater than 255" expects the English .NET error message and fails on a German Windows, with or without local changes | Workaround: ignore this single failure (114 unit tests, 1 expected failure) From f1ab93cea7e56647d740a80550c2bbf8e4a9c958 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:53:40 +0100 Subject: [PATCH 08/12] Docs: Explain the fork's changes for other users Replace LOCAL-CHANGES.md with FORK-CHANGES.md, written for people who only know the upstream project: a "do I need this" table by symptom, one section per change with cause, fix and before/after examples, the recommended configuration, PR status and development notes. README gets a fork banner and points to keepescape mode 4. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- FORK-CHANGES.md | 152 +++++++++++++++++++++++++++ LOCAL-CHANGES.md | 92 ---------------- README.md | 4 +- docs/TRANSCRIPTION-QUALITY-REPORT.md | 2 +- 4 files changed, 155 insertions(+), 95 deletions(-) create mode 100644 FORK-CHANGES.md delete mode 100644 LOCAL-CHANGES.md diff --git a/FORK-CHANGES.md b/FORK-CHANGES.md new file mode 100644 index 0000000..6e3e098 --- /dev/null +++ b/FORK-CHANGES.md @@ -0,0 +1,152 @@ +# What this fork changes + +This is a fork of [theohbrothers/ConvertOneNote2MarkDown](https://github.com/theohbrothers/ConvertOneNote2MarkDown) (the PowerShell script that exports OneNote pages through Pandoc to Markdown). It is based on upstream `master` v2.23.1 (2026-09-24). Everything not listed here works as in upstream, and the defaults are unchanged: you only get a different result where a fix removes a bug, or when you switch on a new option. + +The fixes were found by converting two real notebooks (about 200 pages each, German and English, many web clippings) and reading the result. Five of them were offered upstream as pull requests, which are still open. + +## Do I need this? + +Look at your existing export. If you see one of these, this fork probably helps: + +| What you see in the exported notes | Fix | Where | +|---|---|---| +| Words run together: `Windowskey` instead of `Windows key` | [Non-breaking spaces](#words-were-glued-together) | PR [#191](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/191) | +| The first lines of some pages are missing (a table, a `Subject:` line, the first command) | [Page header](#page-content-was-deleted) | PR [#192](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/192) | +| Pages with the same title overwrite each other; attachment links like `[a.pdf]([b-a.pdf](...))`; images with `#` in the name show as broken | [File names](#file-names-and-links) | PR [#193](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/193) | +| Folder names like `AI ^M ML` or `DECT^J Bluetooth` | [Section names](#section-names-with-onenote-escape-codes) | PR [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | +| Old notes, folders or images stay after you renamed or deleted them in OneNote | [`overwrite = 2`](#new-option-overwrite) | PR [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | +| Lists show as `\- item` or `7\. item`, backslashes at line ends, loose lists, `--\>` | [`keepescape = 4`](#new-option-keepescape--4) | this fork only | +| `Created:` / `Modified:` has the wrong time zone offset (for example +05:30) | [Time zone](#page-content-was-deleted) | PR #192 | + +## How to use the fork + +1. Get the script: `git clone https://github.com/klangme1ster/ConvertOneNote2MarkDown` and use branch `main`. It contains all fixes below. Each pull-request branch (`fix/...`, `enhancement/...`) contains one fix on top of upstream, if you only want some. +2. Copy `config.example.ps1` to `config.ps1` next to the script and edit it. The rest (requirements, running, FAQ) is as in the [README](README.md). + +Recommended settings: + +```powershell +$keepescape = 4 # rebuild lists, drop escapes that don't matter (see below). 3 keeps every backslash +$conversion = 'markdown-simple_tables-multiline_tables-grid_tables+pipe_tables-smart' # '-smart' stops Pandoc escaping every quote as \" +$overwrite = 2 # when you re-export into an existing folder +``` + +Do not add `-link_attributes` or `-bracketed_spans` to `$conversion`: Pandoc then writes raw `` and `` HTML instead of Markdown. + +## The changes + +### Words were glued together + +*Symptom:* `the Windows key` became `Windowskey`. In one notebook 193 of 197 pages were affected. + +*Cause:* with `keepspaces = 1` (the default) every non-breaking space was deleted. Web-clipped content often has a non-breaking space as the only space between two words. + +*Fix:* they are now removed only at the start and end of a line (keeping the indentation, because a leading space could turn the line into a code block, a trailing double space into a hard line break). Anywhere else they become one normal space. + +### Page content was deleted + +*Symptom:* pages without a date line, such as emails sent to OneNote, lost real content (a Tags table, a `Subject:` line, the first command of the page). + +*Cause:* the script always dropped the first 6 lines of Pandoc's output, assuming they are title, date and time. + +*Fix:* the new function `Remove-PageHeader` removes the header only if a date and a time line are really there (any language, `9:19 AM` works too). Otherwise it removes a first line only if it repeats the page title. + +Also in this PR: `Created:` / `Modified:` used the format `zz00`, which printed only the hours of the offset (wrong for +05:30). The output format stays `+0200`, but the value is now right. + +### File names and links + +Three bugs with the same cause, names built from the page path: + +- **Duplicate page names overwrote each other.** The `-1`, `-2` postfix was added before the name was cut to `mdFileNameAndFolderNameMaxLength`, so the postfix could be cut off again and two pages were written to the same file. Duplicates are now detected on the cut name, and the postfix survives the cut. +- **Attachment links were nested.** With `11-36-00002.pdf` and `Datenblatt_11-36-00002.pdf` on one page, the first name also matched inside the second and the link was inserted into the other link. Now a name only matches if it is not part of a longer file name. +- **Image names broke links.** Names such as `c't-expert-community--Newsletter-#10-image1.jpg` contain `#`, which Markdown reads as a URL fragment. Image names now go through the same clean-up as attachments. + +### Section names with OneNote escape codes + +OneNote stores characters that are not allowed in a section file name as `^` codes: `AI / ML` is reported as `AI ^M ML`, `DECT & Bluetooth` as `DECT^J Bluetooth`. These ended up in folder names. `ConvertFrom-OneNoteEscapedName` decodes `^M` to `/` (then `-`, as for any `/`) and `^J` to `&` for section, section group and notebook names. Only these two codes have been seen; others would still show as `^`. + +*Note:* if you already exported before, these folders get new names (`AI ^M ML` becomes `AI-ML`), so links that point to the old folder names need updating. + +### New option: `overwrite` + +`overwrite = 2` removes a notebook's previous output before converting it, so notes, folders and media that were renamed or deleted in OneNote since the last run do not linger. The notebook's `docx` folder is always kept, so `usedocx = 2` still reuses it. Only the notebook's own folder under `notesdestpath` is touched. Read-only files and paths over 260 characters are handled. The default `1` keeps the previous output (old behaviour). + +### New option: `keepescape = 4` + +*Symptom:* in a typical export many lists are broken. Many people type `- item` or `7. item` in OneNote without using real lists. Pandoc is faithful and escapes this text so it does not turn into a list: + +```markdown +\- first point + +\- second point + +7\. numbered step\ +next line --\> result +``` + +That renders as plain paragraphs with a hyphen in front, a literal `\` at line ends, and loose spacing. Setting `keepescape = 1` removes every backslash, but also those in Windows paths (`C:\Program Files`) and in text you meant to escape. `2` and `3` do not repair the lists. + +*Fix:* `keepescape = 4` runs `Repair-MarkdownEscapes` on the Pandoc output: + +```markdown +- first point +- second point + +7. numbered step +next line --> result +``` + +- Rebuilds lists from `\- item` and `7\. item` (also inside block quotes, and when a non-breaking space follows the marker). Removes the blank lines between list items and lone `-` lines (empty bullets, or an accidental heading underline). +- A `\` at the end of a line (Pandoc's hard line break) becomes two trailing spaces, or is dropped at the end of a paragraph. +- Removes escapes that cannot change the rendering (`--\>`, `snake\_case`, `\.` in the middle of a line). +- Keeps escapes that matter: at the start of a line, `\[`, `\]`, `\*` pairs, `\\`, Windows and UNC paths. +- Code fences and inline code are never touched. +- Modes 1 to 3 and the default (`1`) are unchanged. + +*Result on two real notebooks (393 pages, same `.docx` files, mode 3 against mode 4):* + +| | mode 3 | mode 4 | +|---|---|---| +| escaped bullets `\-` | 601 | 0 | +| blank lines between list items | 67 | 0 | +| line-end backslashes | 1804 | 0 | +| lone `-` lines | 29 | 0 | +| escaped numbers `7\.` | 133 | 0 | + +The letters and digits of every page are identical in both modes, so no text is lost. The remaining escapes (`\>`, `\*`, `\_`) are the ones kept on purpose. Details and the decision: [docs/adr/0001-keepescape-mode-4.md](docs/adr/0001-keepescape-mode-4.md) and [docs/TRANSCRIPTION-QUALITY-REPORT.md](docs/TRANSCRIPTION-QUALITY-REPORT.md). + +*Caveat:* a literal `- ` at the start of a line is now treated as a list. If you meant it as plain text, keep `keepescape = 3`. + +### Not changed on purpose + +- **`Modified` before `Created`:** on some pages the modified time is 1 to 2 hours before the created time. That is OneNote's own data (probably pages made by the web clipper, which store local time labelled as UTC); the script converts it correctly. +- **`>` removed at the start of lines** (`keepspaces = 1`): deliberate. Pandoc renders OneNote's indented paragraphs as block quotes, and this removes the indentation. + +## Status of the pull requests + +| PR | Branch | Contains | +|---|---|---| +| [#191](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/191) | `fix/keepspaces-keep-non-breaking-spaces-between-words-as-spaces` | Words glued together | +| [#192](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/192) | `fix/convert-remove-page-header-only-when-present` | Page content deleted, time zone offset | +| [#193](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/193) | `fix/file-names-keep-duplicate-postfix-and-avoid-broken-links` | Duplicate page names, nested attachment links, image names | +| [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | `fix/file-names-decode-onenote-escape-codes-in-section-names` | Section names with escape codes | +| [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | `enhancement/configuration-add-option-overwrite-to-remove-previous-output` | The `overwrite` option | + +The five PRs are independent and merge cleanly in any order. `keepescape = 4` (branch `enhancement/keepescape-repair-markdown`) is not offered upstream yet. The PR texts are in [docs/pr-descriptions.md](docs/pr-descriptions.md). If upstream merges the PRs, `main` can simply be replaced by upstream's version. + +`main` differs from the merged PR branches in one line: `Remove-PreviousNotebookOutput` also decodes the escape codes in the notebook name. + +## Development notes + +- **Keep both `.ps1` files pure ASCII.** Windows PowerShell 5.1 reads scripts without a BOM as ANSI, so a UTF-8 character (for example a literal non-breaking space in a regex) breaks parsing or silently changes the pattern. Write `\u00A0` in regexes and `[char]0x00A0` in strings. Check: `grep -c '[^ -~ ]' *.ps1` must print `0`. +- **Run the tests without installing anything:** `test/test.ps1` installs Pester 4 into the user profile. Instead: + + ```powershell + Save-PSResource -Name Pester -Version 4.10.1 -Path -TrustRepository # PowerShell 7 + # then, in Windows PowerShell 5.1: + Import-Module \Pester\4.10.1\Pester.psd1 -Force + Invoke-Pester -Script . -Tag Unit + ``` + + Do not set `$ErrorActionPreference = 'Stop'` for the run: several tests expect non-terminating errors. +- **Test status:** 114 unit tests. The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes (see [docs/known-issues.md](docs/known-issues.md)). diff --git a/LOCAL-CHANGES.md b/LOCAL-CHANGES.md deleted file mode 100644 index 00f05d3..0000000 --- a/LOCAL-CHANGES.md +++ /dev/null @@ -1,92 +0,0 @@ -# Local changes - -This copy of [ConvertOneNote2MarkDown](https://github.com/theohbrothers/ConvertOneNote2MarkDown) is based on upstream `master` as of 2026-09-24. Since 2026-11-01 the folder is a git repository (`origin` = the fork, `upstream` = the original project). `main` is upstream `master` plus the five PR branches below, merged; compare with `git diff upstream/master`. The former `.orig` copies are no longer needed. - -All changes were checked against the real notebooks by converting them from the kept `.docx` files and comparing the result with an export from the original script: - -- 138 of 197 pages were byte-identical. -- 57 pages differed only in spaces or image names. -- 2 pages got back content that the original script had deleted. - -## Changes - -### Fixes - -- **Words were glued together** (`keepspaces = 1`). Non-breaking spaces were deleted, but web-clipped content often has them as the only space between two words: `theWindowskey` became `Windowskey`. 193 of 197 pages contained some. - - Now they are removed only at the start and end of a line, keeping the line's indentation. - - Anywhere else they become a single normal space. -- **Page content was deleted.** The first 6 lines of Pandoc's output were always dropped, on the assumption that they are title, date and time. Pages without those lines, such as emails sent to OneNote, lost real content: a Tags table, a Subject line, the first command of a page. - - The new `Remove-PageHeader` removes the header only when there really is a date and time line (in any language, including `9:19 AM`). - - Without that header, it only removes a first line that repeats the page title. -- **Duplicate page names overwrote each other.** The `-1`, `-2` postfix was added before the name was truncated to `mdFileNameAndFolderNameMaxLength`, so for long titles it was cut off again and the pages were written to the same file. - - Duplicates are now detected on the truncated name, which also catches long names that only differ after the cut-off. - - The postfix survives truncation (`Truncate-PathFileName -KeepSuffix`). -- **Attachment links were nested inside each other.** Inserted-file links were created with a plain text search. With attachments `11-36-00002.pdf` and `Datenblatt_11-36-00002.pdf` on one page, the first name also matched inside the second. The search now only matches a name that isn't part of a longer file name. -- **Image names broke Markdown links.** Image file names are built from the page path, which kept characters like `#` and `'` (e.g. `c't-expert-community--Newsletter-#10-image1.jpg`). A `#` in a link is read as a URL fragment. Image names now go through the same clean-up as inserted attachments (`Remove-InvalidFileNameCharsInsertedFiles`). -- **Section names contained OneNote escape codes.** OneNote reports characters that aren't allowed in section file names as `^` codes: `AI ^M ML` for `AI / ML`, `DECT^J Bluetooth` for `DECT & Bluetooth`. They ended up literally in folder names. `ConvertFrom-OneNoteEscapedName` decodes `^M` to `/` and `^J` to `&` for section, section group and notebook names; `/` then becomes `-` as usual. These are the only two codes seen so far. Others would still show up as `^`. -- **Wrong time zone offset.** `Created:` / `Modified:` used the format `zz00`, which only printed the hours of the offset (wrong for e.g. +05:30). The output format stays the same (`+0200`). - -### New option - -- **`overwrite`**: `2` removes a notebook's previous output before converting it (`Remove-PreviousNotebookOutput`), so notes, folders and media renamed or deleted in OneNote since the last run don't linger. - - The notebook's `docx` folder is always kept, so `usedocx = 2` can still reuse it. - - Only the notebook's own folder under `notesdestpath` is touched. - - Read-only files and paths over 260 characters are handled. - - The default is `1` (keep previous output), which is the old behaviour. - -- **`keepescape = 4`**: keeps a `\` only where it changes the rendering (`Repair-MarkdownEscapes`). Default behaviour (`1`) is unchanged. - - Pandoc escapes paragraphs that only look like lists. OneNote pages often start lines with a literal `- ` or `7.` (no real list: the `.docx` has no list numbering), which came out as `\- item` and `7\. item`. These become real lists again (`- item`, `7. item`), blank lines between the items are removed, and lone `-` lines (empty bullets, or an accidental heading underline) are dropped. - - A `\` at the end of a line (Pandoc's hard break) becomes two trailing spaces, or is dropped at the end of a paragraph. - - Escapes that cannot change the rendering (`\>` in `--\>`, `\_` in `snake\_case`, `\.`, `\-` in the middle of a line) are removed. Kept: escapes at the start of a line, `\[`, `\]`, `\*` pairs, `\\`, Windows and UNC paths. - - Code fences and inline code are never touched. - - Port of `fix_lists` and `escapes` from `markdown_cleaner/qon_helper.py`, extended for what real OneNote exports showed: lists inside block quotes (`> 1. item`; `keepspaces` removes the `>` only later), non-breaking spaces after the marker, and several blank or nbsp-only lines between items. - - Checked with a real run of the script against OneNote (notebook with 197 pages, `keepescape` 3 vs 4, same `.docx`): escaped bullets 503 to 0, blank lines between list items 45 to 0, line-end backslashes 1039 to 0, lone dash lines 25 to 0, escaped numbers 87 to 0. `\>` 316 to 25, `\*` 366 to 330 and `\_` 9 to 9 remain on purpose (they sit at line starts or pair up). Letters and digits are identical on all 197 pages, so no text was lost. - -### Not changed, for the record - -- **Wrong-looking Created/Modified times:** on some pages "Modified" is 1 to 2 hours *before* "Created". That's OneNote's own data, not a conversion error: those pages (probably created by the web clipper) store local time labelled as UTC in `dateTime`. The script converts correctly. -- **`>` removed at the start of lines** (`keepspaces = 1`): deliberate. Pandoc renders OneNote's indented paragraphs as block quotes, and this un-indents them. - -## Sent upstream - -On 2026-09-24 all of the changes above were offered to the original project as five independent pull requests, #191 to #195. They come from the fork https://github.com/klangme1ster/ConvertOneNote2MarkDown, whose branches are `origin/` in this repository. Longer PR texts, with test details, are in `docs/pr-descriptions.md`. - -| PR | Branch | Contains | -|---|---|---| -| [#191](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/191) | `fix/keepspaces-keep-non-breaking-spaces-between-words-as-spaces` | Words glued together | -| [#192](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/192) | `fix/convert-remove-page-header-only-when-present` | Page content deleted, wrong time zone offset | -| [#193](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/193) | `fix/file-names-keep-duplicate-postfix-and-avoid-broken-links` | Duplicate page names, nested attachment links, image names | -| [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | `fix/file-names-decode-onenote-escape-codes-in-section-names` | Section names with OneNote escape codes | -| [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | `enhancement/configuration-add-option-overwrite-to-remove-previous-output` | The `overwrite` option | - -The PR branches differ from this copy in two small ways, so that each branch works and merges on its own: - -- The new functions and tests are placed at different spots in the files. -- In the `overwrite` branch, `Remove-PreviousNotebookOutput` doesn't call `ConvertFrom-OneNoteEscapedName`. - -Not in any PR: this file, the README pointer to it, and `config.ps1`. - -If upstream merges the PRs, this copy can be replaced by upstream's version (keep `config.ps1`). - -## Recommended configuration - -In `config.ps1`: - -- `$keepescape = 4` (new, see above) keeps only the backslashes that matter. Before that: `3` keeps all backslashes, but leaves `\- item`, `7\. item` and `\` line ends in the output. The default `1` also deletes the ones in Windows paths (`C:\Program Files` → `C:Program Files`), and `2` turns escaped `\*` and `\` back into formatting and hidden HTML. -- `$conversion = 'markdown-simple_tables-multiline_tables-grid_tables+pipe_tables-smart'`: `-smart` stops Pandoc from escaping every quote (`\"`). Don't add `-link_attributes` or `-bracketed_spans`: Pandoc then writes raw `` and `` HTML instead. -- `$overwrite = 2` when re-exporting into an existing folder. - -## Development notes - -- **Keep both `.ps1` files pure ASCII.** Windows PowerShell 5.1 reads scripts without a BOM as ANSI, so a UTF-8 character (e.g. a literal non-breaking space in a regex) breaks parsing or silently changes the pattern. Write `\u00A0` in regexes and `[char]0x00A0` in strings. Check with `grep -c '[^ -~ ]' *.ps1`, which should print `0`. -- **Running the tests without installing anything:** `test/test.ps1` installs Pester 4 into the user profile. Instead: - - ```powershell - Save-PSResource -Name Pester -Version 4.10.1 -Path -TrustRepository # PowerShell 7 - # then, in Windows PowerShell 5.1: - Import-Module \Pester\4.10.1\Pester.psd1 -Force - Invoke-Pester -Script . -Tag Unit - ``` - - Don't set `$ErrorActionPreference = 'Stop'` for the run: several tests expect non-terminating errors. -- **Test status:** 114 unit tests (22 of them new for `keepescape = 4`). The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes. diff --git a/README.md b/README.md index 236e788..2ec05d2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![github-actions](https://github.com/theohbrothers/ConvertOneNote2MarkDown/actions/workflows/ci-master-pr.yml/badge.svg?branch=master)](https://github.com/theohbrothers/ConvertOneNote2MarkDown/actions/workflows/ci-master-pr.yml) [![github-release](https://img.shields.io/github/v/release/theohbrothers/ConvertOneNote2MarkDown?style=flat-square)](https://github.com/theohbrothers/ConvertOneNote2MarkDown/releases/) -> **Local copy:** this copy has fixes and an extra `overwrite` option on top of upstream; see [LOCAL-CHANGES.md](LOCAL-CHANGES.md). +> **This is a fork** of [theohbrothers/ConvertOneNote2MarkDown](https://github.com/theohbrothers/ConvertOneNote2MarkDown) with bug fixes (words glued together, missing page content, overwritten duplicate pages, broken links) and two new options: `overwrite` (remove a notebook's old output first) and `keepescape = 4` (repair lists and escapes that Pandoc writes as `\- item`, `7\. item`). See [FORK-CHANGES.md](FORK-CHANGES.md) for what changed, with examples and when you need it. Ready to make the step to Markdown and saying farewell to your OneNote, EverNote or whatever proprietary note taking tool you are using? Nothing beats clear text, right? Read on! @@ -32,7 +32,7 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote * `markdown_strict` (original unextended Markdown) * Improved headers, with title now as a `#` heading, standardized `DateTime` format for created and modified dates, and horizontal line to separate from rest of document * Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines and line ends (other non-breaking spaces become normal spaces), and `>` after unordered lists, which are created when converting with Pandoc -* Choose whether to remove `\` escape symbol that are created when converting with Pandoc +* Choose whether to remove `\` escape symbol that are created when converting with Pandoc. Mode `4` (`$keepescape = 4`) keeps only the escapes that matter and rebuilds lists that Pandoc escaped (`\- item`, `7\. item`); see [FORK-CHANGES.md](FORK-CHANGES.md#new-option-keepescape--4) * Choose whether to use Line Feed (`LF`) or Carriage Return + Line Feed (`CRLF`) for new lines * Choose whether to include a `.pdf` export alongside the `.md` file. `.md` does not preserve `InkDrawing` (i.e. overlayed drawings, highlights, pen marks) absolute positions within a page, but a `.pdf` export is a complete page snapshot that preserves `InkDrawing` absolute positions within a page. * Choose whether to **remove a notebook's previous output** before converting it (`$overwrite = 2`), so notes, folders and media that were renamed or deleted in OneNote since the last run don't linger. The notebook's `docx` folder is always kept, so existing `.docx` files can still be reused. diff --git a/docs/TRANSCRIPTION-QUALITY-REPORT.md b/docs/TRANSCRIPTION-QUALITY-REPORT.md index 94470a8..2a38778 100644 --- a/docs/TRANSCRIPTION-QUALITY-REPORT.md +++ b/docs/TRANSCRIPTION-QUALITY-REPORT.md @@ -78,7 +78,7 @@ Also worth checking in the converter: - Existing Pester tests stay unchanged; new behaviour gets new tests. - Default behaviour stays the same unless the user picks the new mode (it changes rendering for pages that start lines with "- "). -- Record the change in `LOCAL-CHANGES.md`. +- Record the change in `FORK-CHANGES.md`. ## Addendum 2026-11-01: verification and result From 58ebd68ab610400e54719031a9b78f2359cca832 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:24:52 +0100 Subject: [PATCH 09/12] CI: Run on main, drop retired and unusable test jobs, update checkout This fork's workflow differs from upstream: - Remove windows-2019: GitHub retired the image, the job never starts. - Remove the PowerShell 6.0, 6.1, 6.2 and 7.0 container jobs: their Ubuntu 16.04/18.04 images are too old (glibc < 2.28) for the Node runtime GitHub forces on JavaScript actions, so actions/checkout fails before any test runs. - actions/checkout v3 -> v4. - Also run for pushes and pull requests on main. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- .github/workflows/ci-master-pr.yml | 85 +++--------------------------- 1 file changed, 7 insertions(+), 78 deletions(-) diff --git a/.github/workflows/ci-master-pr.yml b/.github/workflows/ci-master-pr.yml index d7c6b6f..86d5446 100644 --- a/.github/workflows/ci-master-pr.yml +++ b/.github/workflows/ci-master-pr.yml @@ -4,28 +4,19 @@ on: push: branches: - master + - main tags: - '**' pull_request: branches: - master + - main jobs: - test-powershell-5-1-windows-2019: - runs-on: windows-2019 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - powershell -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - test-powershell-5-1-windows-2022: runs-on: windows-2022 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -36,64 +27,12 @@ jobs: ########## # Docker # ########## - test-powershell-6-0: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.0.2-ubuntu-16.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-6-1: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.1.3-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-6-2: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.2.4-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-7-0: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:7.0.3-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - test-powershell-7-1: runs-on: ubuntu-latest container: image: mcr.microsoft.com/powershell:7.1.5-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -106,7 +45,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.2.2-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -119,7 +58,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.3-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -132,7 +71,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.4-ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -142,12 +81,7 @@ jobs: update-draft-release: needs: - - test-powershell-5-1-windows-2019 - test-powershell-5-1-windows-2022 - - test-powershell-6-0 - - test-powershell-6-1 - - test-powershell-6-2 - - test-powershell-7-0 - test-powershell-7-1 - test-powershell-7-2 - test-powershell-7-3 @@ -162,12 +96,7 @@ jobs: publish-draft-release: needs: - - test-powershell-5-1-windows-2019 - test-powershell-5-1-windows-2022 - - test-powershell-6-0 - - test-powershell-6-1 - - test-powershell-6-2 - - test-powershell-7-0 - test-powershell-7-1 - test-powershell-7-2 - test-powershell-7-3 From 255e5cac27a8902959da7d1e2380fe0895fe3f79 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:50:57 +0100 Subject: [PATCH 10/12] Docs: Add PR texts for keepescape mode 4 and the CI fix Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- docs/pr-descriptions.md | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/pr-descriptions.md b/docs/pr-descriptions.md index 91d475a..af7e84d 100644 --- a/docs/pr-descriptions.md +++ b/docs/pr-descriptions.md @@ -111,3 +111,54 @@ This PR adds the option `overwrite`: This overlaps in purpose with #190 (`skipIfExists`), which is about re-running conversions too. The two options don't conflict. New unit tests run against real files in `TestDrive`: removal while keeping `docx`, including a read-only file; a dry run; and a notebook folder that doesn't exist. I also used it on two notebooks (393 pages, with long paths): a planted stale note was removed, and both `docx` folders were kept. + +--- + +## 6. Branch `enhancement/configuration-add-keepescape-mode-4-to-repair-pandoc-escapes` + +Based on upstream `master`, independent of PRs 1 to 5. Not opened upstream yet. + +**Title:** Enhancement (configuration): Add keepescape mode 4 to repair Pandoc escapes + +**Description:** + +OneNote pages often start paragraphs with a literal `- ` or `7.` instead of using real lists. Pandoc escapes such text so it does not turn into a list (`\- item`, `7\. item`). In an export of two notebooks (393 pages) this left the notes with plain paragraphs instead of lists, a blank line between every item, a literal `\` at the end of many lines (Pandoc's hard line break) and many needless escapes (`--\>`, `snake\_case`). + +The existing `keepescape` modes cannot fix this: `1` deletes every backslash (also in Windows paths such as `C:\Program Files`), `2` and `3` keep the list escapes. + +This PR adds `keepescape = 4`, implemented as the function `Repair-MarkdownEscapes`, run on Pandoc's output before the existing replacements: + +- Rebuilds lists from `\- item` and `7\. item` (also inside block quotes and when a non-breaking space follows the marker), removes the blank lines between list items, and drops lone `-` lines (empty bullets, or an accidental heading underline). +- Turns a `\` at the end of a line into two trailing spaces, or drops it at the end of a paragraph. +- Removes escapes that cannot change the rendering (`\>` in `--\>`, `\_` in `snake\_case`, `\.` and `\-` in the middle of a line). +- Keeps escapes that matter: at the start of a line, `\[`, `\]`, `\*` pairs, `\`, Windows and UNC paths. +- Never touches code fences and inline code. + +The default (`1`) and modes 1 to 3 are unchanged. A literal `- ` at the start of a line is treated as a list in mode 4; use `3` if that is not wanted. + +**Tested** + +- 22 new unit tests: each rule, cases that must stay untouched (Windows and UNC paths, `\*`, code), block quotes, non-breaking spaces, running it twice gives the same result, and that mode 4 adds no replacement that touches backslashes. +- Real run against OneNote with two notebooks (393 pages, same `.docx` files, mode 3 against mode 4): escaped bullets 601 to 0, blank lines between list items 67 to 0, line-end backslashes 1804 to 0, lone `-` lines 29 to 0, escaped numbers 133 to 0. The letters and digits of every page are identical in both modes, so no text is lost. +- Unit tests: 96 passed in Windows PowerShell 5.1; the one failure ("Throw if -Length is greater than 255") expects the English .NET error message and also fails on `master` on a German Windows. +- Not tested here: PowerShell 6.x and 7.0 (see the CI PR below). + +--- + +## 7. Branch `fix/ci-remove-retired-windows-2019-and-unusable-container-jobs` + +Based on upstream `master`. Not opened upstream yet. Best merged before any other open PR, so their CI can finish. + +**Title:** CI: Remove the retired windows-2019 job and the unusable PowerShell 6.x/7.0 container jobs + +**Description:** + +Running the workflow on this repository's pull requests (in a fork) shows three problems that have nothing to do with the code: + +- `test-powershell-5-1-windows-2019` stays queued forever because GitHub retired the `windows-2019` runner image (June 2025). The whole workflow run never completes. +- `test-powershell-6-0`, `6-1`, `6-2` and `7-0` run in Ubuntu 16.04/18.04 containers. `actions/checkout` fails there before any test runs: `/__e/node24/bin/node: ... GLIBC_2.28 not found`, because GitHub now forces a newer Node runtime on JavaScript actions. +- `actions/checkout@v3` is outdated. + +This PR removes those five jobs (and their entries in `needs:`) and updates `actions/checkout` to `v4`. Windows PowerShell 5.1 on `windows-2022` and PowerShell 7.1 to 7.4 remain and pass. The tests are not changed. + +The trade-off: PowerShell 6.x and 7.0 are no longer exercised by CI. They could be added back with a newer container image or a different way to install them, if wanted. From 9e5d9821964e2e1ef33fc2ba2382f9f1bce11348 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:04:54 +0100 Subject: [PATCH 11/12] Docs: Add upstream PR numbers #216 and #217 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- FORK-CHANGES.md | 4 ++-- docs/pr-descriptions.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/FORK-CHANGES.md b/FORK-CHANGES.md index 6e3e098..f160b10 100644 --- a/FORK-CHANGES.md +++ b/FORK-CHANGES.md @@ -15,7 +15,7 @@ Look at your existing export. If you see one of these, this fork probably helps: | Pages with the same title overwrite each other; attachment links like `[a.pdf]([b-a.pdf](...))`; images with `#` in the name show as broken | [File names](#file-names-and-links) | PR [#193](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/193) | | Folder names like `AI ^M ML` or `DECT^J Bluetooth` | [Section names](#section-names-with-onenote-escape-codes) | PR [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | | Old notes, folders or images stay after you renamed or deleted them in OneNote | [`overwrite = 2`](#new-option-overwrite) | PR [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | -| Lists show as `\- item` or `7\. item`, backslashes at line ends, loose lists, `--\>` | [`keepescape = 4`](#new-option-keepescape--4) | this fork only | +| Lists show as `\- item` or `7\. item`, backslashes at line ends, loose lists, `--\>` | [`keepescape = 4`](#new-option-keepescape--4) | PR [#217](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/217) | | `Created:` / `Modified:` has the wrong time zone offset (for example +05:30) | [Time zone](#page-content-was-deleted) | PR #192 | ## How to use the fork @@ -132,7 +132,7 @@ The letters and digits of every page are identical in both modes, so no text is | [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | `fix/file-names-decode-onenote-escape-codes-in-section-names` | Section names with escape codes | | [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | `enhancement/configuration-add-option-overwrite-to-remove-previous-output` | The `overwrite` option | -The five PRs are independent and merge cleanly in any order. `keepescape = 4` (branch `enhancement/keepescape-repair-markdown`) is not offered upstream yet. The PR texts are in [docs/pr-descriptions.md](docs/pr-descriptions.md). If upstream merges the PRs, `main` can simply be replaced by upstream's version. +The five PRs are independent and merge cleanly in any order. Two more PRs were opened on 2026-11-01: [#217](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/217) for `keepescape = 4` and [#216](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/216), which removes the retired `windows-2019` job and the unusable PowerShell 6.x/7.0 container jobs from the CI workflow (this fork's `main` has the same CI change). #216 should be merged first, so the CI of the other PRs can finish. The PR texts are in [docs/pr-descriptions.md](docs/pr-descriptions.md). If upstream merges the PRs, `main` can simply be replaced by upstream's version. `main` differs from the merged PR branches in one line: `Remove-PreviousNotebookOutput` also decodes the escape codes in the notebook name. diff --git a/docs/pr-descriptions.md b/docs/pr-descriptions.md index af7e84d..2ea9efc 100644 --- a/docs/pr-descriptions.md +++ b/docs/pr-descriptions.md @@ -116,7 +116,7 @@ New unit tests run against real files in `TestDrive`: removal while keeping `doc ## 6. Branch `enhancement/configuration-add-keepescape-mode-4-to-repair-pandoc-escapes` -Based on upstream `master`, independent of PRs 1 to 5. Not opened upstream yet. +Based on upstream `master`, independent of PRs 1 to 5. Opened upstream as [#217](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/217) on 2026-11-01. **Title:** Enhancement (configuration): Add keepescape mode 4 to repair Pandoc escapes @@ -147,7 +147,7 @@ The default (`1`) and modes 1 to 3 are unchanged. A literal `- ` at the start of ## 7. Branch `fix/ci-remove-retired-windows-2019-and-unusable-container-jobs` -Based on upstream `master`. Not opened upstream yet. Best merged before any other open PR, so their CI can finish. +Based on upstream `master`. Opened upstream as [#216](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/216) on 2026-11-01. Best merged before any other open PR, so their CI can finish. **Title:** CI: Remove the retired windows-2019 job and the unusable PowerShell 6.x/7.0 container jobs From 85ff6c5d7eebdc18876ea69d7261062c99d25aa3 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:24:00 +0100 Subject: [PATCH 12/12] Enhancement (configuration): Add option hardbreaks to force hard line breaks A single newline inside a paragraph is a soft break: some programs show it as a line break, strict CommonMark renderers join the lines. Pandoc writes OneNote's line breaks as a backslash at the line end, which is not rendered the same everywhere either, and keepescape 1 and 2 delete it, so the line break is lost. hardbreaks = 2 (Add-MarkdownHardBreaks) turns that backslash into two spaces and ends every other line that is followed by a line of the same paragraph with two spaces. Blank lines, headings, tables, rules, HTML, code and the line before a list item are left alone. Default 1 changes nothing; the option works with every keepescape mode. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 76 ++++++++++++++++++++++++++++ ConvertOneNote2MarkDown-v2.ps1 | 71 ++++++++++++++++++++++++++ FORK-CHANGES.md | 30 ++++++++++- README.md | 1 + config.example.ps1 | 5 ++ docs/known-issues.md | 2 +- 6 files changed, 183 insertions(+), 2 deletions(-) diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index e506dea..8289751 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -579,6 +579,75 @@ Describe "Repair-MarkdownEscapes" -Tag 'Unit' { } +Describe "Add-MarkdownHardBreaks" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should end every line that is followed by a line of the same paragraph with two spaces" { + Add-MarkdownHardBreaks -Content "one`ntwo`nthree`n`nnext`nline" | Should -BeExactly "one `ntwo `nthree`n`nnext `nline" + } + + It "Should turn pandoc's backslash line break into two spaces" { + Add-MarkdownHardBreaks -Content "one\`ntwo" | Should -BeExactly "one `ntwo" + Add-MarkdownHardBreaks -Content "one \`ntwo" | Should -BeExactly "one `ntwo" + } + + It "Should treat a literal backslash (written as two) at the end of a line as text and still end the line with two spaces" { + Add-MarkdownHardBreaks -Content "C:\\dir\\`nnext" | Should -BeExactly "C:\\dir\\ `nnext" + Add-MarkdownHardBreaks -Content "one\\\`nnext" | Should -BeExactly "one\\ `nnext" + } + + It "Should leave a backslash at the end of a paragraph to keepescape" { + Add-MarkdownHardBreaks -Content "one\`n`ntwo" | Should -BeExactly "one\`n`ntwo" + } + + It "Should keep lines that already end with two spaces" { + Add-MarkdownHardBreaks -Content "one `ntwo" | Should -BeExactly "one `ntwo" + } + + It "Should leave alone lines before a list item and the list items' own paragraph ends" { + Add-MarkdownHardBreaks -Content "text`n- item`n- item two" | Should -BeExactly "text`n- item`n- item two" + Add-MarkdownHardBreaks -Content "1. one`n2. two" | Should -BeExactly "1. one`n2. two" + } + + It "Should leave headings, tables, rules and HTML alone ()" -TestCases @( + @{ Name = 'heading'; Content = "# Title`ntext" } + @{ Name = 'before heading'; Content = "text`n# Title" } + @{ Name = 'table'; Content = "| a | b |`n|---|---|`n| 1 | 2 |" } + @{ Name = 'rule'; Content = "text`n---`ntext" } + @{ Name = 'html'; Content = "`n`n
" } + ) { + param ($Name, $Content) + + Add-MarkdownHardBreaks -Content $Content | Should -BeExactly $Content + } + + It "Should treat lines with only non-breaking spaces or a bare '>' as blank" { + Add-MarkdownHardBreaks -Content "one`n$( [char]0x00A0 )`ntwo" | Should -BeExactly "one`n$( [char]0x00A0 )`ntwo" + Add-MarkdownHardBreaks -Content "> one`n>`n> two" | Should -BeExactly "> one`n>`n> two" + Add-MarkdownHardBreaks -Content "> one`n> two" | Should -BeExactly "> one `n> two" + } + + It "Should not touch code" { + $content = @('text', '```sh', 'line one', 'line two\', '```', 'see `a b\` here', 'next') -join "`n" + $expected = @('text', '```sh', 'line one', 'line two\', '```', 'see `a b\` here ', 'next') -join "`n" + Add-MarkdownHardBreaks -Content $content | Should -BeExactly $expected + } + + It "Should give the same result when run twice" { + $content = "one\`ntwo`nthree`n`n- a`n- b`ntext`n> q`n> r" + $once = Add-MarkdownHardBreaks -Content $content + Add-MarkdownHardBreaks -Content $once | Should -BeExactly $once + } + + It "Should accept CRLF input and return LF" { + Add-MarkdownHardBreaks -Content "one`r`ntwo" | Should -BeExactly "one `ntwo" + } + + } + +} + Describe "New-OneNoteConnection" -Tag 'Unit' { Context 'Behavior' { @@ -1633,6 +1702,13 @@ foo\bar\0 } } + It "Should leave line breaks alone by default and accept 1 and 2 for config hardbreaks" { + $config = Get-DefaultConfiguration + $config['hardbreaks']['default'] | Should -Be 1 + $config['hardbreaks']['value'] | Should -Be 1 + $config['hardbreaks']['validateRange'] | Should -Be @(1, 2) + } + It "Should honor config newlineCharacter" { $params['Config']['newlineCharacter']['value'] = 1 diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 130a21d..fc25228 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -172,6 +172,16 @@ Whether to clear escape symbols from md files. See: https://pandoc.org/MANUAL.ht value = 1 validateRange = 1,4 } + hardbreaks = @{ + description = @' +Whether to end lines that are followed by another line of the same paragraph with two spaces, which is a hard line break in every markdown program. Without it a single newline is a 'soft' break, which some programs show as a line break and others join into one line; and keepescape 1 and 2 delete the backslash that pandoc writes for a line break. Headings, tables, rules, HTML, code and the lines before a list item are left alone +1: Leave line breaks as they are - Default +2: Force hard line breaks (two spaces at the end of the line) +'@ + default = 1 + value = 1 + validateRange = 1,2 + } newlineCharacter = @{ description = @' Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new lines @@ -682,6 +692,64 @@ Function Repair-MarkdownEscapes { } } +Function Add-MarkdownHardBreaks { + [CmdletBinding()] + param ( + # The markdown content produced by pandoc (any newline style) + [Parameter(Mandatory,Position = 0,ValueFromPipeline = $true)] + [AllowEmptyString()] + [string] + $Content + ) + + process { + # A single newline inside a paragraph is a 'soft' break: some programs show it as a line break, strict CommonMark renderers join the lines. + # A backslash at the end of a line (what pandoc writes for a line break) is not rendered the same way everywhere either, and + # 'keepescape' 1 and 2 delete it. Two trailing spaces are invisible and are rendered as a line break by every program. + # Fenced code blocks and inline code are never touched. + $saved = New-Object 'System.Collections.Generic.List[string]' + $shield = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + $saved.Add($m.Value) + "MDSHIELD$( $saved.Count - 1 )END" + } + $text = $Content -replace '\r\n', "`n" + $text = [regex]::Replace($text, '(?ms)(?:^[ \t]*)?(`{3,})[^\n`]*\n.*?^[ \t]*\1`*[ \t]*$', $shield) + $text = [regex]::Replace($text, '(`+)(?!`)(.+?)(?$( [char]0x00A0 )" + $heading = '^\s*#{1,6}(?:\s|$)' + $table = '^\s*\|' + $rule = '^\s*(?:[-*_=]\s*){3,}$' + $html = '^\s*<' + $code = '^\s*MDSHIELD\d+END\s*$' # A fenced code block on its own lines + $listStart = '^\s*(?:[-*+]|\d+[.)])\s' + $lines = $text -split "`n" + for ($i = 0; $i -lt $lines.Count - 1; $i++) { + $line = $lines[$i] + $next = $lines[$i + 1] + if ($line.Trim($blank.ToCharArray()).Length -eq 0 -or $next.Trim($blank.ToCharArray()).Length -eq 0) { continue } # Paragraph end. A bare '>' is a blank line in a block quote + if ($line.EndsWith(' ')) { continue } # Already a line break + if ($line -match $heading -or $line -match $table -or $line -match $html -or $line -match $code) { continue } + if ($line -match $rule -or $next -match $rule) { continue } + if ($next -match $listStart -or $next -match $table -or $next -match $heading -or $next -match $html -or $next -match $code) { continue } + $lines[$i] = $line.TrimEnd() + ' ' + } + $text = $lines -join "`n" + + # Restore code (fences can contain inline code and vice versa, so repeat until nothing is left) + $restore = [System.Text.RegularExpressions.MatchEvaluator]{ param ($m) $saved[[int]$m.Groups[1].Value] } + while ($text -match 'MDSHIELD\d+END') { + $text = [regex]::Replace($text, 'MDSHIELD(\d+)END', $restore) + } + $text + } +} + Function Set-ContentNoBom { [CmdletBinding()] param ( @@ -1518,6 +1586,9 @@ Function Convert-OneNotePage { $content = @( Get-Content -LiteralPath $pageCfg['filePath'] -ErrorAction Stop ) # Use -LiteralPath so that characters like '(', ')', '[', ']', '`', "'", '"' are supported. Or else we will get an error "Cannot find path 'xxx' because it does not exist" # Discard the page's title, date and time lines (if present). We are going to add our own header $content = @( Remove-PageHeader -Lines $content -Title $pageCfg['object'].name ) -join "`n" + if ($config['hardbreaks']['value'] -eq 2) { + $content = Add-MarkdownHardBreaks -Content $content + } if ($config['keepescape']['value'] -eq 4) { $content = Repair-MarkdownEscapes -Content $content } diff --git a/FORK-CHANGES.md b/FORK-CHANGES.md index f160b10..b8c7b7e 100644 --- a/FORK-CHANGES.md +++ b/FORK-CHANGES.md @@ -16,6 +16,7 @@ Look at your existing export. If you see one of these, this fork probably helps: | Folder names like `AI ^M ML` or `DECT^J Bluetooth` | [Section names](#section-names-with-onenote-escape-codes) | PR [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | | Old notes, folders or images stay after you renamed or deleted them in OneNote | [`overwrite = 2`](#new-option-overwrite) | PR [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | | Lists show as `\- item` or `7\. item`, backslashes at line ends, loose lists, `--\>` | [`keepescape = 4`](#new-option-keepescape--4) | PR [#217](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/217) | +| Lines of a paragraph run together in some programs, or show a `\` at the line end | [`hardbreaks = 2`](#new-option-hardbreaks) | this fork only | | `Created:` / `Modified:` has the wrong time zone offset (for example +05:30) | [Time zone](#page-content-was-deleted) | PR #192 | ## How to use the fork @@ -117,6 +118,33 @@ The letters and digits of every page are identical in both modes, so no text is *Caveat:* a literal `- ` at the start of a line is now treated as a list. If you meant it as plain text, keep `keepescape = 3`. +### New option: `hardbreaks` + +*Symptom:* in some programs lines of one paragraph run together into a single line, in others they appear as separate lines. Markdown knows two kinds of line break: a single newline is a *soft* break (rendered as a space by strict CommonMark renderers, as a line break by others), and two spaces at the end of the line are a *hard* break that every program renders as a break. OneNote's line breaks (Shift+Enter) are written by Pandoc as a backslash at the line end, which is not rendered the same way everywhere either. `keepescape = 1` and `2` even delete that backslash, so the line break is lost. + +*Fix:* `hardbreaks = 2` runs `Add-MarkdownHardBreaks` before the `keepescape` replacements: + +```markdown +first line\ +second line +third line +``` + +becomes (the two trailing spaces are invisible) + +```markdown +first line +second line +third line +``` + +- A backslash at the end of a line (Pandoc's line break) becomes two spaces. A literal backslash is written by Pandoc as `\\` and stays as text. +- Every other line that is directly followed by another line of the same paragraph gets two spaces. +- Left alone: blank lines and paragraph ends, headings, tables, rules, HTML, code blocks and inline code, and a line before a list item. +- Default `1` changes nothing. The option works with every `keepescape` mode; with `keepescape = 4` the line-end backslash is converted anyway. +- Checked on one real notebook (196 pages, `keepescape = 1`): with `hardbreaks = 1` all 418 line breaks inside paragraphs are soft (the backslashes are gone); with `hardbreaks = 2` 415 are hard and 3 stay soft. The letters and digits of every page are identical. +- Limitation: a few lines that were separated only by a line of non-breaking spaces or a bare `>` end up next to each other after `keepspaces` removes that line; they stay soft. + ### Not changed on purpose - **`Modified` before `Created`:** on some pages the modified time is 1 to 2 hours before the created time. That is OneNote's own data (probably pages made by the web clipper, which store local time labelled as UTC); the script converts it correctly. @@ -149,4 +177,4 @@ The five PRs are independent and merge cleanly in any order. Two more PRs were o ``` Do not set `$ErrorActionPreference = 'Stop'` for the run: several tests expect non-terminating errors. -- **Test status:** 114 unit tests. The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes (see [docs/known-issues.md](docs/known-issues.md)). +- **Test status:** 130 unit tests. The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes (see [docs/known-issues.md](docs/known-issues.md)). diff --git a/README.md b/README.md index 2ec05d2..c857e75 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote * Improved headers, with title now as a `#` heading, standardized `DateTime` format for created and modified dates, and horizontal line to separate from rest of document * Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines and line ends (other non-breaking spaces become normal spaces), and `>` after unordered lists, which are created when converting with Pandoc * Choose whether to remove `\` escape symbol that are created when converting with Pandoc. Mode `4` (`$keepescape = 4`) keeps only the escapes that matter and rebuilds lists that Pandoc escaped (`\- item`, `7\. item`); see [FORK-CHANGES.md](FORK-CHANGES.md#new-option-keepescape--4) +* Choose whether to force hard line breaks (`$hardbreaks = 2`): lines that are followed by another line of the same paragraph end with two spaces, which every markdown program renders as a line break. Invisible, unlike a `\` at the line end, and it survives `$keepescape = 1` * Choose whether to use Line Feed (`LF`) or Carriage Return + Line Feed (`CRLF`) for new lines * Choose whether to include a `.pdf` export alongside the `.md` file. `.md` does not preserve `InkDrawing` (i.e. overlayed drawings, highlights, pen marks) absolute positions within a page, but a `.pdf` export is a complete page snapshot that preserves `InkDrawing` absolute positions within a page. * Choose whether to **remove a notebook's previous output** before converting it (`$overwrite = 2`), so notes, folders and media that were renamed or deleted in OneNote since the last run don't linger. The notebook's `docx` folder is always kept, so existing `.docx` files can still be reused. diff --git a/config.example.ps1 b/config.example.ps1 index 72d6a8f..30b8c03 100644 --- a/config.example.ps1 +++ b/config.example.ps1 @@ -80,6 +80,11 @@ $keepspaces = 1 # 4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter $keepescape = 1 +# Whether to end lines that are followed by another line of the same paragraph with two spaces, which is a hard line break in every markdown program. Without it a single newline is a 'soft' break, which some programs show as a line break and others join into one line; and keepescape 1 and 2 delete the backslash that pandoc writes for a line break. Headings, tables, rules, HTML, code and the lines before a list item are left alone +# 1: Leave line breaks as they are - Default +# 2: Force hard line breaks (two spaces at the end of the line) +$hardbreaks = 1 + # Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new lines # 1: LF (unix) - Default # 2: CRLF (windows) diff --git a/docs/known-issues.md b/docs/known-issues.md index a905cde..43fc313 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -1,3 +1,3 @@ # Known issues -- 2026-09-24 | Tests: Pester test "Throw if -Length is greater than 255" expects the English .NET error message and fails on a German Windows, with or without local changes | Workaround: ignore this single failure (114 unit tests, 1 expected failure) +- 2026-09-24 | Tests: Pester test "Throw if -Length is greater than 255" expects the English .NET error message and fails on a German Windows, with or without local changes | Workaround: ignore this single failure (130 unit tests, 1 expected failure)