From 3636a085512d36ab74a10b64f4d23356d1e68d7f Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:47:22 +0100 Subject: [PATCH 1/2] CI: Remove the retired windows-2019 job and the unusable PowerShell 6.x/7.0 container jobs - windows-2019: GitHub retired the runner image, so the job stays queued forever and the whole workflow run never completes. - PowerShell 6.0, 6.1, 6.2 and 7.0 containers (Ubuntu 16.04/18.04): their glibc is older than 2.28, which the Node runtime that GitHub now forces on JavaScript actions needs, so actions/checkout fails before any test runs. - actions/checkout v3 -> v4. The remaining jobs (Windows PowerShell 5.1 on windows-2022, PowerShell 7.1 to 7.4) pass. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 --- .github/workflows/ci-master-pr.yml | 83 ++---------------------------- 1 file changed, 5 insertions(+), 78 deletions(-) diff --git a/.github/workflows/ci-master-pr.yml b/.github/workflows/ci-master-pr.yml index d7c6b6f..c5508b9 100644 --- a/.github/workflows/ci-master-pr.yml +++ b/.github/workflows/ci-master-pr.yml @@ -11,21 +11,10 @@ on: - master jobs: - test-powershell-5-1-windows-2019: - runs-on: windows-2019 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - powershell -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - test-powershell-5-1-windows-2022: runs-on: windows-2022 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -36,64 +25,12 @@ jobs: ########## # Docker # ########## - test-powershell-6-0: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.0.2-ubuntu-16.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-6-1: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.1.3-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-6-2: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.2.4-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-7-0: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:7.0.3-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - test-powershell-7-1: runs-on: ubuntu-latest container: image: mcr.microsoft.com/powershell:7.1.5-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -106,7 +43,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.2.2-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -119,7 +56,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.3-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -132,7 +69,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.4-ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -142,12 +79,7 @@ jobs: update-draft-release: needs: - - test-powershell-5-1-windows-2019 - test-powershell-5-1-windows-2022 - - test-powershell-6-0 - - test-powershell-6-1 - - test-powershell-6-2 - - test-powershell-7-0 - test-powershell-7-1 - test-powershell-7-2 - test-powershell-7-3 @@ -162,12 +94,7 @@ jobs: publish-draft-release: needs: - - test-powershell-5-1-windows-2019 - test-powershell-5-1-windows-2022 - - test-powershell-6-0 - - test-powershell-6-1 - - test-powershell-6-2 - - test-powershell-7-0 - test-powershell-7-1 - test-powershell-7-2 - test-powershell-7-3 From f3e67c319e27aeb59f91eacc7465af79e66ecab4 Mon Sep 17 00:00:00 2001 From: klangme1ster <126453713+klangme1ster@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:46:58 +0100 Subject: [PATCH 2/2] Enhancement (configuration): Add keepescape mode 4 to repair Pandoc escapes OneNote pages often start paragraphs with a literal '- ' or '7.'. Pandoc escapes them ('\- item', '7\. item'), so the notes lose their lists and carry loose blank lines, line-end backslashes and needless escapes. The existing modes cannot fix that: 1 deletes every backslash (also in Windows paths), 2 and 3 keep the list escapes. Mode 4 (Repair-MarkdownEscapes) rebuilds those lists (also inside block quotes and after non-breaking spaces), turns a line-end backslash into two spaces, and removes only escapes that cannot change the rendering. Code fences and inline code are untouched. Modes 1 to 3 and the default are unchanged. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011zGMypgzLxRQFiU563LLb5 (cherry picked from commit 3438fc2e94aa14357ec8e044020edbb2e7614ca0) --- ConvertOneNote2MarkDown-v2.Tests.ps1 | 108 +++++++++++++++++++++++++++ ConvertOneNote2MarkDown-v2.ps1 | 107 +++++++++++++++++++++++++- README.md | 2 +- config.example.ps1 | 1 + 4 files changed, 216 insertions(+), 2 deletions(-) diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..787b737 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -407,6 +407,93 @@ Describe "Encode-Markdown" -Tag 'Unit' { } +Describe "Repair-MarkdownEscapes" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should turn escaped list markers into lists ()" -TestCases @( + @{ Name = 'bullet'; Content = '\- one'; Expected = '- one' } + @{ Name = 'indented bullet'; Content = ' \- one'; Expected = ' - one' } + @{ Name = 'number'; Content = '7\. one'; Expected = '7. one' } + @{ Name = 'number alone before its text'; Content = "7\.`n`nitem text"; Expected = '7. item text' } + ) { + param ($Name, $Content, $Expected) + + Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Expected + } + + It "Should accept a non-breaking space after an escaped list marker" { + $nbsp = [char]0x00A0 + Repair-MarkdownEscapes -Content "\-$nbsp one`n7\.$nbsp two" | Should -BeExactly "- one`n7. two" + } + + It "Should repair lists inside block quotes (keepspaces removes the '>' later)" { + Repair-MarkdownEscapes -Content "> 1\. one`n>`n> 2\. two`n>`n> \- x" | Should -BeExactly "> 1. one`n> 2. two`n> - x" + } + + It "Should drop a lone dash line that only had a line-end backslash" { + Repair-MarkdownEscapes -Content "- one`n-\`n`n`n" | Should -BeExactly "- one`n`n`n" + } + + It "Should remove blank lines between list items" { + Repair-MarkdownEscapes -Content "\- a`n`n\- b`n`n\- c" | Should -BeExactly "- a`n- b`n- c" + Repair-MarkdownEscapes -Content "1\. a`n`n2\. b" | Should -BeExactly "1. a`n2. b" + } + + It "Should remove several blank lines, also with non-breaking spaces, between list items" { + Repair-MarkdownEscapes -Content "- a`n`n`n$( [char]0x00A0 )`n`n- b" | Should -BeExactly "- a`n- b" + Repair-MarkdownEscapes -Content "1. a`n`n`n`n2. b`n`ntext" | Should -BeExactly "1. a`n2. b`n`ntext" + Repair-MarkdownEscapes -Content "- a`n`n> $( [char]0x00A0 ) `n`n- b" | Should -BeExactly "- a`n- b" + } + + It "Should keep blank lines between a list and a paragraph" { + Repair-MarkdownEscapes -Content "- a`n`ntext" | Should -BeExactly "- a`n`ntext" + } + + It "Should remove lone dash lines, also with non-breaking spaces" { + Repair-MarkdownEscapes -Content "text`n-`n\-`n-$( [char]0x00A0 )`nmore" | Should -BeExactly "text`nmore" + } + + It "Should turn a backslash at the end of a line into two spaces, and drop it at the end of a paragraph" { + Repair-MarkdownEscapes -Content "one\`ntwo\`n`nnext" | Should -BeExactly "one `ntwo`n`nnext" + } + + It "Should remove escapes that can't change the rendering" { + Repair-MarkdownEscapes -Content 'a --\> b, snake\_case, 1\. no, end\.' | Should -BeExactly 'a --> b, snake_case, 1. no, end.' + } + + It "Should keep escapes that change the rendering ()" -TestCases @( + @{ Name = 'quote at line start'; Content = '\> not a quote' } + @{ Name = 'underscore at line start'; Content = '\_not emphasis_' } + @{ Name = 'image-like'; Content = 'a \![b](c)' } + @{ Name = 'pair of asterisks'; Content = 'a \*b\* c' } + @{ Name = 'brackets'; Content = '\[x\]' } + ) { + param ($Name, $Content) + + Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Content + } + + It "Should keep Windows and UNC paths" { + $content = 'C:\Program Files\App and \\server\share\dir' + Repair-MarkdownEscapes -Content $content | Should -BeExactly $content + } + + It "Should not touch code" { + $content = "``\- keep \_this`` and`n`n``````text`n\- keep`n7\. keep\`n``````" + Repair-MarkdownEscapes -Content $content | Should -BeExactly $content + } + + It "Should give the same result when run twice" { + $content = "\- a`n`n\- b\`nc`n`n7\. x --\> y\_z\`n`nend\" + $once = Repair-MarkdownEscapes -Content $content + Repair-MarkdownEscapes -Content $once | Should -BeExactly $once + } + + } + +} + Describe "New-OneNoteConnection" -Tag 'Unit' { Context 'Behavior' { @@ -1380,6 +1467,27 @@ foo\bar\0 } } + It "Should leave backslashes to Repair-MarkdownEscapes if config keepescape is 4" { + $params['Config']['keepescape']['value'] = 4 + + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + $result.Count | Should -Be 48 + + foreach ($pageCfg in $result) { + $fakeMarkdownContent = 'hello\$\^\\\*\_\[\]world' + + # Mutate + foreach ($m in $pageCfg['mutations']) { + foreach ($r in $m['replacements']) { + $fakeMarkdownContent = $fakeMarkdownContent -replace $r['searchRegex'], $r['replacement'] + } + } + + # None of the mutations may touch backslashes + $fakeMarkdownContent | Should -Match ([regex]::Escape('hello\$\^\\\*\_\[\]world')) + } + } + It "Should honor config newlineCharacter" { $params['Config']['newlineCharacter']['value'] = 1 diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..7b2ad4c 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -166,10 +166,11 @@ Whether to clear escape symbols from md files. See: https://pandoc.org/MANUAL.ht 1: Clear all '\' characters - Default 2: Clear all '\' characters except those preceding alphanumeric characters 3: Keep '\' symbol escape +4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter '@ default = 1 value = 1 - validateRange = 1,3 + validateRange = 1,4 } newlineCharacter = @{ description = @' @@ -497,6 +498,107 @@ Function Encode-Markdown { $Name } +Function Repair-MarkdownEscapes { + [CmdletBinding()] + param ( + # The markdown content produced by pandoc (any newline style) + [Parameter(Mandatory,Position = 0,ValueFromPipeline = $true)] + [AllowEmptyString()] + [string] + $Content + ) + + process { + # Pandoc escapes text that merely looks like markdown. OneNote pages often start paragraphs with a literal '- ' or '7.' (no real list), + # which ends up as '\- item' or '7\. item'. Rebuild those lists, and remove only the escapes that can't change how the text is rendered. + # Fenced code blocks and inline code are never touched. + $saved = New-Object 'System.Collections.Generic.List[string]' + $shield = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + $saved.Add($m.Value) + "MDSHIELD$( $saved.Count - 1 )END" + } + $text = $Content -replace '\r\n', "`n" + $text = [regex]::Replace($text, '(?ms)(?:^[ \t]*)?(`{3,})[^\n`]*\n.*?^[ \t]*\1`*[ \t]*$', $shield) + $text = [regex]::Replace($text, '(`+)(?!`)(.+?)(?]*\\?-[ \t\u00A0]*$\n?', '' # A lone '-' or '\-' line is an empty bullet (below a text line, '-' would even make a heading) + $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*)\\-[ \t\u00A0]+', '$1- ' + $text = $text -replace '(?m)^(\d+)\\\.[ \t]*\n(?:[ \t]*\n)+(?=\S)', '$1. ' # A number alone on its line, with the item text in the next paragraph + $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*\d+)\\\.[ \t\u00A0]+', '$1. ' + $listStart = '^[ \t\u00A0>]*(?:[-*+]|\d+[.)])\s' + $lines = $text -split "`n" + $isBlank = '^[ \t\u00A0>]*$' # Lines with only non-breaking spaces or an empty block quote ('> ') become blank later on (keepspaces) + $text = @( + for ($i = 0; $i -lt $lines.Count; $i++) { + # Drop the blank lines between two list items + if ($lines[$i] -match $isBlank -and $i -gt 0) { + $prev = $i - 1 + while ($prev -gt 0 -and $lines[$prev] -match $isBlank) { $prev-- } + $next = $i + 1 + while ($next -lt $lines.Count - 1 -and $lines[$next] -match $isBlank) { $next++ } + if ($lines[$prev] -match $listStart -and $next -lt $lines.Count -and $lines[$next] -match $listStart) { + continue + } + } + $lines[$i] + } + ) -join "`n" + + # Escapes that can't change the rendering + $unescape = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + if ($m.Value -eq '\\') { return $m.Value } # An escaped backslash is matched as a pair, so it can't hide the character after it + $c = $m.Value[1] + $start = $m.Index + $lineStart = if ($start -gt 0) { $text.LastIndexOf([char]10, $start - 1) + 1 } else { 0 } + $prefix = $text.Substring($lineStart, $start - $lineStart) + $before = if ($start -gt 0) { [string]$text[$start - 1] } else { '' } + $afterIndex = $start + 2 + $after = if ($afterIndex -lt $text.Length) { [string]$text[$afterIndex] } else { '' } + $atLineStart = $prefix.Trim(" `t>").Length -eq 0 + if (',."''%@/?!=+-#>'.Contains($c)) { + $keep = ('=+-#>'.Contains($c) -and $atLineStart) -or ($c -eq '.' -and $prefix -match '^[ \t>]*\d+$') -or ($c -eq '!' -and $after -eq '[') + if (!$keep -and $c -eq '#') { + $lineEnd = $text.IndexOf([char]10, $start) + $line = if ($lineEnd -lt 0) { $text.Substring($lineStart) } else { $text.Substring($lineStart, $lineEnd - $lineStart) } + $keep = $line -match '^\s*#{1,6}(?:\s|$)' # A closing '\#' in a heading stays + } + }elseif ($c -eq '*' -or $c -eq '_') { + $loneGap = ($before -eq '' -or $before -match '^\s$') -and ($after -eq '' -or $after -match '^\s$') + $intraword = $before -match '^[\p{L}\p{N}]$' -and $after -match '^[\p{L}\p{N}]$' + if ($c -eq '*') { + $pStart = if ($start -ge 2) { $text.LastIndexOf("`n`n", $start - 1, [System.StringComparison]::Ordinal) } else { -1 } + $pEnd = $text.IndexOf("`n`n", $start, [System.StringComparison]::Ordinal) + $from = if ($pStart -ge 0) { $pStart + 2 } else { 0 } + $to = if ($pEnd -ge 0) { $pEnd } else { $text.Length } + $paragraph = $text.Substring($from, $to - $from) + $keep = $atLineStart -or !($loneGap -or ($paragraph.Length - $paragraph.Replace('*', '').Length) -eq 1) + }else { + $keep = $atLineStart -or !($loneGap -or $intraword) + } + }else { + $keep = $true + } + if ($keep) { $m.Value } else { [string]$c } + } + $text = [regex]::Replace($text, '\\\\|\\.', $unescape) + + # Restore code (fences can contain inline code and vice versa, so repeat until nothing is left) + $restore = [System.Text.RegularExpressions.MatchEvaluator]{ param ($m) $saved[[int]$m.Groups[1].Value] } + while ($text -match 'MDSHIELD\d+END') { + $text = [regex]::Replace($text, 'MDSHIELD(\d+)END', $restore) + } + $text + } +} + Function Set-ContentNoBom { [CmdletBinding()] param ( @@ -1282,6 +1384,9 @@ Function Convert-OneNotePage { '' } ) -join "`n" + if ($config['keepescape']['value'] -eq 4) { + $content = Repair-MarkdownEscapes -Content $content + } } # Mutate diff --git a/README.md b/README.md index c23006f..6037026 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote * `markdown_strict` (original unextended Markdown) * Improved headers, with title now as a `#` heading, standardized `DateTime` format for created and modified dates, and horizontal line to separate from rest of document * Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines, and `>` after unordered lists, which are created when converting with Pandoc -* Choose whether to remove `\` escape symbol that are created when converting with Pandoc +* Choose whether to remove `\` escape symbol that are created when converting with Pandoc. Mode `4` keeps only the escapes that matter and rebuilds lists that Pandoc escaped (`\- item`, `7\. item`), turns a `\` at the end of a line into a hard line break, and removes escapes that do not change the rendering * Choose whether to use Line Feed (`LF`) or Carriage Return + Line Feed (`CRLF`) for new lines * Choose whether to include a `.pdf` export alongside the `.md` file. `.md` does not preserve `InkDrawing` (i.e. overlayed drawings, highlights, pen marks) absolute positions within a page, but a `.pdf` export is a complete page snapshot that preserves `InkDrawing` absolute positions within a page. * Detailed logs. Run the script with `-Verbose` to see detailed logs of each page's conversion. diff --git a/config.example.ps1 b/config.example.ps1 index 6077bdf..72d6a8f 100644 --- a/config.example.ps1 +++ b/config.example.ps1 @@ -77,6 +77,7 @@ $keepspaces = 1 # 1: Clear all '\' characters - Default # 2: Clear all '\' characters except those preceding alphanumeric characters # 3: Keep '\' symbol escape +# 4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter $keepescape = 1 # Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new lines