diff --git a/.github/workflows/ci-master-pr.yml b/.github/workflows/ci-master-pr.yml index d7c6b6f..86d5446 100644 --- a/.github/workflows/ci-master-pr.yml +++ b/.github/workflows/ci-master-pr.yml @@ -4,28 +4,19 @@ on: push: branches: - master + - main tags: - '**' pull_request: branches: - master + - main jobs: - test-powershell-5-1-windows-2019: - runs-on: windows-2019 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - powershell -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - test-powershell-5-1-windows-2022: runs-on: windows-2022 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -36,64 +27,12 @@ jobs: ########## # Docker # ########## - test-powershell-6-0: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.0.2-ubuntu-16.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-6-1: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.1.3-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-6-2: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:6.2.4-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - - test-powershell-7-0: - runs-on: ubuntu-latest - container: - image: mcr.microsoft.com/powershell:7.0.3-ubuntu-18.04 - steps: - - uses: actions/checkout@v3 - - name: Powershell version - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' - - name: Test - run: | - pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1' - test-powershell-7-1: runs-on: ubuntu-latest container: image: mcr.microsoft.com/powershell:7.1.5-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -106,7 +45,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.2.2-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -119,7 +58,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.3-ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -132,7 +71,7 @@ jobs: container: image: mcr.microsoft.com/powershell:7.4-ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Powershell version run: | pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable' @@ -142,12 +81,7 @@ jobs: update-draft-release: needs: - - test-powershell-5-1-windows-2019 - test-powershell-5-1-windows-2022 - - test-powershell-6-0 - - test-powershell-6-1 - - test-powershell-6-2 - - test-powershell-7-0 - test-powershell-7-1 - test-powershell-7-2 - test-powershell-7-3 @@ -162,12 +96,7 @@ jobs: publish-draft-release: needs: - - test-powershell-5-1-windows-2019 - test-powershell-5-1-windows-2022 - - test-powershell-6-0 - - test-powershell-6-1 - - test-powershell-6-2 - - test-powershell-7-0 - test-powershell-7-1 - test-powershell-7-2 - test-powershell-7-3 diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..8289751 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -313,6 +313,24 @@ Describe "Truncate-PathFileName" -Tag 'Unit' { $path | Truncate-PathFileName -Length $length | Should -Be $expectedPath } + It "Should keep a given -KeepSuffix when truncating" { + $drive = "C:$( [io.path]::DirectorySeparatorChar )" # E.g. C:\ + $path = $drive + ("a" * 1000) + '-1' + $length = 123 + $expectedPath = $drive + ("a" * 121) + '-1' + + $path | Truncate-PathFileName -Length $length -KeepSuffix '-1' | Should -Be $expectedPath + } + + It "Should ignore -KeepSuffix if the filename does not end with it" { + $drive = "C:$( [io.path]::DirectorySeparatorChar )" # E.g. C:\ + $path = $drive + ("a" * 1000) + $length = 123 + $expectedPath = $drive + ("a" * 123) + + $path | Truncate-PathFileName -Length $length -KeepSuffix '-1' | Should -Be $expectedPath + } + } } @@ -372,6 +390,24 @@ Describe "Remove-InvalidFileNameCharsInsertedFiles" -Tag 'Unit' { } +Describe "ConvertFrom-OneNoteEscapedName" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should decode OneNote's '^' codes in section names" { + 'AI ^M ML' | ConvertFrom-OneNoteEscapedName | Should -Be 'AI / ML' + 'Messen^J Konferenzen^J Seminare' | ConvertFrom-OneNoteEscapedName | Should -Be 'Messen& Konferenzen& Seminare' + 'plain name' | ConvertFrom-OneNoteEscapedName | Should -Be 'plain name' + } + + It "Should turn a decoded '/' into a dash in the directory name" { + 'AI ^M ML' | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars | Should -Not -Match '\^' + } + + } + +} + Describe "Encode-Markdown" -Tag 'Unit' { Context 'Parameters' { @@ -407,6 +443,211 @@ Describe "Encode-Markdown" -Tag 'Unit' { } +Describe "Remove-PageHeader" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should remove the title, date and time lines ()" -TestCases @( + @{ Name = 'German'; Lines = @('Docker', '', 'Donnerstag, 17. Oktober 2024', '', '09:50', '', 'body', 'more') } + @{ Name = 'English, AM/PM'; Lines = @('Max Planck', '', 'Monday, September 8, 2014', '', '9:19 AM', '', 'body', 'more') } + @{ Name = 'title with a year'; Lines = @('Pressemeetings 2011', '', 'Saturday, October 29, 2011', '', '6:22 AM', '', 'body', 'more') } + @{ Name = 'untitled page'; Lines = @([string][char]0x00A0, '', 'Dienstag, 19. August 2025', '', '12:34', '', 'body', 'more') } + ) { + param ($Name, $Lines) + + $result = @( Remove-PageHeader -Lines $Lines ) + + ($result -join "`n").Trim() | Should -Be "body`nmore" + } + + It "Should keep all lines if there is no date and time (e.g. an email sent to OneNote)" { + $lines = @('Linksammlung', '', '| **Tags:** #Teufel |', '|-------------------|', '', '**Subject:** Linksammlung', 'body') + + $result = @( Remove-PageHeader -Lines $lines ) + + $result | Should -Be $lines + } + + It "Should remove only a leading title line if there is no date and time, but a -Title" { + $lines = @('curl \> /volume1/extern', '', 'curl \> /volume1/external_ip.log', 'body') + + $result = @( Remove-PageHeader -Lines $lines -Title 'curl http://ifconfig.me/ip > /volume1/extern' ) + + ($result -join "`n").Trim() | Should -Be "curl \> /volume1/external_ip.log`nbody" + } + + It "Should return an empty string for a page that only has a header" { + $result = @( Remove-PageHeader -Lines @('Docker', '', 'Donnerstag, 17. Oktober 2024', '', '09:50') ) + + $result | Should -Be @('') + } + + It "Should accept empty input" { + $result = @( Remove-PageHeader -Lines @() ) + + $result.Count | Should -Be 0 + } + + } + +} + +Describe "Repair-MarkdownEscapes" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should turn escaped list markers into lists ()" -TestCases @( + @{ Name = 'bullet'; Content = '\- one'; Expected = '- one' } + @{ Name = 'indented bullet'; Content = ' \- one'; Expected = ' - one' } + @{ Name = 'number'; Content = '7\. one'; Expected = '7. one' } + @{ Name = 'number alone before its text'; Content = "7\.`n`nitem text"; Expected = '7. item text' } + ) { + param ($Name, $Content, $Expected) + + Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Expected + } + + It "Should accept a non-breaking space after an escaped list marker" { + $nbsp = [char]0x00A0 + Repair-MarkdownEscapes -Content "\-$nbsp one`n7\.$nbsp two" | Should -BeExactly "- one`n7. two" + } + + It "Should repair lists inside block quotes (keepspaces removes the '>' later)" { + Repair-MarkdownEscapes -Content "> 1\. one`n>`n> 2\. two`n>`n> \- x" | Should -BeExactly "> 1. one`n> 2. two`n> - x" + } + + It "Should drop a lone dash line that only had a line-end backslash" { + Repair-MarkdownEscapes -Content "- one`n-\`n`n`n" | Should -BeExactly "- one`n`n`n" + } + + It "Should remove blank lines between list items" { + Repair-MarkdownEscapes -Content "\- a`n`n\- b`n`n\- c" | Should -BeExactly "- a`n- b`n- c" + Repair-MarkdownEscapes -Content "1\. a`n`n2\. b" | Should -BeExactly "1. a`n2. b" + } + + It "Should remove several blank lines, also with non-breaking spaces, between list items" { + Repair-MarkdownEscapes -Content "- a`n`n`n$( [char]0x00A0 )`n`n- b" | Should -BeExactly "- a`n- b" + Repair-MarkdownEscapes -Content "1. a`n`n`n`n2. b`n`ntext" | Should -BeExactly "1. a`n2. b`n`ntext" + Repair-MarkdownEscapes -Content "- a`n`n> $( [char]0x00A0 ) `n`n- b" | Should -BeExactly "- a`n- b" + } + + It "Should keep blank lines between a list and a paragraph" { + Repair-MarkdownEscapes -Content "- a`n`ntext" | Should -BeExactly "- a`n`ntext" + } + + It "Should remove lone dash lines, also with non-breaking spaces" { + Repair-MarkdownEscapes -Content "text`n-`n\-`n-$( [char]0x00A0 )`nmore" | Should -BeExactly "text`nmore" + } + + It "Should turn a backslash at the end of a line into two spaces, and drop it at the end of a paragraph" { + Repair-MarkdownEscapes -Content "one\`ntwo\`n`nnext" | Should -BeExactly "one `ntwo`n`nnext" + } + + It "Should remove escapes that can't change the rendering" { + Repair-MarkdownEscapes -Content 'a --\> b, snake\_case, 1\. no, end\.' | Should -BeExactly 'a --> b, snake_case, 1. no, end.' + } + + It "Should keep escapes that change the rendering ()" -TestCases @( + @{ Name = 'quote at line start'; Content = '\> not a quote' } + @{ Name = 'underscore at line start'; Content = '\_not emphasis_' } + @{ Name = 'image-like'; Content = 'a \![b](c)' } + @{ Name = 'pair of asterisks'; Content = 'a \*b\* c' } + @{ Name = 'brackets'; Content = '\[x\]' } + ) { + param ($Name, $Content) + + Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Content + } + + It "Should keep Windows and UNC paths" { + $content = 'C:\Program Files\App and \\server\share\dir' + Repair-MarkdownEscapes -Content $content | Should -BeExactly $content + } + + It "Should not touch code" { + $content = "``\- keep \_this`` and`n`n``````text`n\- keep`n7\. keep\`n``````" + Repair-MarkdownEscapes -Content $content | Should -BeExactly $content + } + + It "Should give the same result when run twice" { + $content = "\- a`n`n\- b\`nc`n`n7\. x --\> y\_z\`n`nend\" + $once = Repair-MarkdownEscapes -Content $content + Repair-MarkdownEscapes -Content $once | Should -BeExactly $once + } + + } + +} + +Describe "Add-MarkdownHardBreaks" -Tag 'Unit' { + + Context 'Behavior' { + + It "Should end every line that is followed by a line of the same paragraph with two spaces" { + Add-MarkdownHardBreaks -Content "one`ntwo`nthree`n`nnext`nline" | Should -BeExactly "one `ntwo `nthree`n`nnext `nline" + } + + It "Should turn pandoc's backslash line break into two spaces" { + Add-MarkdownHardBreaks -Content "one\`ntwo" | Should -BeExactly "one `ntwo" + Add-MarkdownHardBreaks -Content "one \`ntwo" | Should -BeExactly "one `ntwo" + } + + It "Should treat a literal backslash (written as two) at the end of a line as text and still end the line with two spaces" { + Add-MarkdownHardBreaks -Content "C:\\dir\\`nnext" | Should -BeExactly "C:\\dir\\ `nnext" + Add-MarkdownHardBreaks -Content "one\\\`nnext" | Should -BeExactly "one\\ `nnext" + } + + It "Should leave a backslash at the end of a paragraph to keepescape" { + Add-MarkdownHardBreaks -Content "one\`n`ntwo" | Should -BeExactly "one\`n`ntwo" + } + + It "Should keep lines that already end with two spaces" { + Add-MarkdownHardBreaks -Content "one `ntwo" | Should -BeExactly "one `ntwo" + } + + It "Should leave alone lines before a list item and the list items' own paragraph ends" { + Add-MarkdownHardBreaks -Content "text`n- item`n- item two" | Should -BeExactly "text`n- item`n- item two" + Add-MarkdownHardBreaks -Content "1. one`n2. two" | Should -BeExactly "1. one`n2. two" + } + + It "Should leave headings, tables, rules and HTML alone ()" -TestCases @( + @{ Name = 'heading'; Content = "# Title`ntext" } + @{ Name = 'before heading'; Content = "text`n# Title" } + @{ Name = 'table'; Content = "| a | b |`n|---|---|`n| 1 | 2 |" } + @{ Name = 'rule'; Content = "text`n---`ntext" } + @{ Name = 'html'; Content = "`n`n
" } + ) { + param ($Name, $Content) + + Add-MarkdownHardBreaks -Content $Content | Should -BeExactly $Content + } + + It "Should treat lines with only non-breaking spaces or a bare '>' as blank" { + Add-MarkdownHardBreaks -Content "one`n$( [char]0x00A0 )`ntwo" | Should -BeExactly "one`n$( [char]0x00A0 )`ntwo" + Add-MarkdownHardBreaks -Content "> one`n>`n> two" | Should -BeExactly "> one`n>`n> two" + Add-MarkdownHardBreaks -Content "> one`n> two" | Should -BeExactly "> one `n> two" + } + + It "Should not touch code" { + $content = @('text', '```sh', 'line one', 'line two\', '```', 'see `a b\` here', 'next') -join "`n" + $expected = @('text', '```sh', 'line one', 'line two\', '```', 'see `a b\` here ', 'next') -join "`n" + Add-MarkdownHardBreaks -Content $content | Should -BeExactly $expected + } + + It "Should give the same result when run twice" { + $content = "one\`ntwo`nthree`n`n- a`n- b`ntext`n> q`n> r" + $once = Add-MarkdownHardBreaks -Content $content + Add-MarkdownHardBreaks -Content $once | Should -BeExactly $once + } + + It "Should accept CRLF input and return LF" { + Add-MarkdownHardBreaks -Content "one`r`ntwo" | Should -BeExactly "one `ntwo" + } + + } + +} + Describe "New-OneNoteConnection" -Tag 'Unit' { Context 'Behavior' { @@ -833,6 +1074,32 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { } } + It "Should not match an attachment's name inside a longer attachment name" { + Mock Get-OneNotePageContent { + @' + + + + + + + + + +'@ -as [xml] + } + + $pageCfg = @( New-SectionGroupConversionConfig @params 6>$null ) | Select-Object -First 1 + $content = '\<\\> and \<\<11-36-00002.pdf\>\>' + foreach ($m in ($pageCfg['mutations'] | Where-Object { $_['description'] -eq 'Change inserted attachment(s) filename references' })) { + foreach ($r in $m['replacements']) { + $content = $content -replace $r['searchRegex'], $r['replacement'] + } + } + + $content | Should -Be "\<\<[Datenblatt_11-36-00002.pdf]($( $pageCfg['mediaPathPandoc'] )/Datenblatt_11-36-00002.pdf)\>\> and \<\<[11-36-00002.pdf]($( $pageCfg['mediaPathPandoc'] )/11-36-00002.pdf)\>\>" + } + It "Should generate fully unique docx file for each page, even for identically-named pages in a section" { $result = @( New-SectionGroupConversionConfig @params 6>$null ) @@ -885,6 +1152,34 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { } } + It "Should keep the unique postfix of identically named pages when truncating long names" { + $longName = 'Some long name ' * 20 + $fakeHierarchy = @" + + + + + + + + + + +"@ -as [xml] + $params['SectionGroups'] = $fakeHierarchy.Notebooks.Notebook + $params['Config']['mdFileNameAndFolderNameMaxLength']['value'] = 100 + + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + + $result.Count | Should -Be 3 + $result[1]['fileName'] | Should -Match '-1\.md$' + $result[2]['fileName'] | Should -Match '-2\.md$' + @( $result | ForEach-Object { $_['filePathNormal'] } | Select-Object -Unique ).Count | Should -Be 3 + foreach ($pageCfg in $result) { + $pageCfg['fileName'].Length | Should -BeLessOrEqual 100 + } + } + It "Should honor config notesdestpath" { if ($env:OS -match 'windows') { $params['Config']['notesdestpath']['value'] = 'D:\foo\bar' @@ -1201,6 +1496,9 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { $fakeMarkdownContent = @" hello world$( [char]0x00A0 ) +press the$( [char]0x00A0 ) Windows$( [char]0x00A0 )key +$( [char]0x00A0 )$( [char]0x00A0 )no leading non-breaking spaces + $( [char]0x00A0 )keeps its indentation - foo @@ -1244,11 +1542,14 @@ some more } } - # Should remove extra newline between unordered and ordered lists, remove non-breaking spaces, and '>' from ordered lists. Ignore first 8 lines for page header + # Should remove extra newline between unordered and ordered lists, remove non-breaking spaces at line boundaries but keep them as spaces between words, and '>' from ordered lists. Ignore first 8 lines for page header $split = $mutated -split "`n" $expectedBody = $split[8..($split.Count - 1)] -join "`n" $expectedBody | Should -Be $( @" hello world +press the Windows key +no leading non-breaking spaces + keeps its indentation - foo - foo1 @@ -1380,6 +1681,34 @@ foo\bar\0 } } + It "Should leave backslashes to Repair-MarkdownEscapes if config keepescape is 4" { + $params['Config']['keepescape']['value'] = 4 + + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + $result.Count | Should -Be 48 + + foreach ($pageCfg in $result) { + $fakeMarkdownContent = 'hello\$\^\\\*\_\[\]world' + + # Mutate + foreach ($m in $pageCfg['mutations']) { + foreach ($r in $m['replacements']) { + $fakeMarkdownContent = $fakeMarkdownContent -replace $r['searchRegex'], $r['replacement'] + } + } + + # None of the mutations may touch backslashes + $fakeMarkdownContent | Should -Match ([regex]::Escape('hello\$\^\\\*\_\[\]world')) + } + } + + It "Should leave line breaks alone by default and accept 1 and 2 for config hardbreaks" { + $config = Get-DefaultConfiguration + $config['hardbreaks']['default'] | Should -Be 1 + $config['hardbreaks']['value'] | Should -Be 1 + $config['hardbreaks']['validateRange'] | Should -Be @(1, 2) + } + It "Should honor config newlineCharacter" { $params['Config']['newlineCharacter']['value'] = 1 @@ -1812,6 +2141,44 @@ Describe 'Convert-OneNotePage' -Tag 'Unit' { } +Describe "Remove-PreviousNotebookOutput" -Tag 'Unit' { + + Context 'Behavior' { + + BeforeEach { + $notebookDirectory = Join-Path $TestDrive 'my-notebook' + if (Test-Path -LiteralPath $notebookDirectory) { + Remove-Item -LiteralPath $notebookDirectory -Recurse -Force # Left over from a previous test (e.g. the dry run), incl. the read-only file + } + New-Item -ItemType Directory -Path (Join-Path $notebookDirectory 'docx') -Force > $null + New-Item -ItemType Directory -Path (Join-Path $notebookDirectory 'section\media') -Force > $null + 'docx' | Set-Content -LiteralPath (Join-Path $notebookDirectory 'docx\page.docx') + 'note' | Set-Content -LiteralPath (Join-Path $notebookDirectory 'section\stale-page.md') + 'pdf' | Set-Content -LiteralPath (Join-Path $notebookDirectory 'section\media\attachment.pdf') + (Get-Item -LiteralPath (Join-Path $notebookDirectory 'section\media\attachment.pdf')).IsReadOnly = $true # OneNote's inserted files are read-only + } + + It "Should remove everything in the notebook's directory except the docx folder" { + Remove-PreviousNotebookOutput -NotesDestination $TestDrive -NotebookName 'my notebook' + + Test-Path -LiteralPath (Join-Path $notebookDirectory 'section') | Should -Be $false + Test-Path -LiteralPath (Join-Path $notebookDirectory 'docx\page.docx') | Should -Be $true + } + + It "Should not remove anything on a dry run" { + Remove-PreviousNotebookOutput -NotesDestination $TestDrive -NotebookName 'my notebook' -DryRun + + Test-Path -LiteralPath (Join-Path $notebookDirectory 'section\stale-page.md') | Should -Be $true + } + + It "Should do nothing if the notebook's directory does not exist" { + { Remove-PreviousNotebookOutput -NotesDestination $TestDrive -NotebookName 'other notebook' } | Should -Not -Throw + } + + } + +} + Describe "Print-ConversionErrors" -Tag 'Unit' { Context 'Behavior' { diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..fc25228 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -152,7 +152,7 @@ Whether to include page timestamp and separator at top of document } keepspaces = @{ description = @' -Whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines, and `>` after unordered lists +Whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines and line ends (other non-breaking spaces become normal spaces), and `>` after unordered lists 1: Clear - Default 2: Don't clear '@ @@ -166,10 +166,21 @@ Whether to clear escape symbols from md files. See: https://pandoc.org/MANUAL.ht 1: Clear all '\' characters - Default 2: Clear all '\' characters except those preceding alphanumeric characters 3: Keep '\' symbol escape +4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter '@ default = 1 value = 1 - validateRange = 1,3 + validateRange = 1,4 + } + hardbreaks = @{ + description = @' +Whether to end lines that are followed by another line of the same paragraph with two spaces, which is a hard line break in every markdown program. Without it a single newline is a 'soft' break, which some programs show as a line break and others join into one line; and keepescape 1 and 2 delete the backslash that pandoc writes for a line break. Headings, tables, rules, HTML, code and the lines before a list item are left alone +1: Leave line breaks as they are - Default +2: Force hard line breaks (two spaces at the end of the line) +'@ + default = 1 + value = 1 + validateRange = 1,2 } newlineCharacter = @{ description = @' @@ -186,6 +197,16 @@ Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new line Whether to include a PDF export alongside the markdown file 1: Don't include PDF - Default 2: Include PDF +'@ + default = 1 + value = 1 + validateRange = 1,2 + } + overwrite = @{ + description = @' +Whether to remove a notebook's previous output before converting it. Without this, files of the same name are overwritten, but notes, folders and media that were renamed or deleted in OneNote since the last run are left behind. The notebook's docx folder is always kept +1: Keep previous output - Default +2: Remove previous output (except the docx folder) '@ default = 1 value = 1 @@ -386,6 +407,11 @@ Function Truncate-PathFileName { [ValidateRange(0,255)] [int] $Length + , + # A suffix (e.g. the '-1' postfix of a duplicate page name) that must survive truncation + [Parameter()] + [string] + $KeepSuffix = '' ) process { @@ -404,7 +430,11 @@ Function Truncate-PathFileName { $name = Split-Path $Path -Leaf if ($name.Length -gt $maxLength) { $parent = Split-Path $Path -Parent - $truncatedName = $name.Substring(0, $maxLength) + $truncatedName = if ($KeepSuffix -and $name.EndsWith($KeepSuffix) -and $maxLength -gt $KeepSuffix.Length) { + $name.Substring(0, $maxLength - $KeepSuffix.Length) + $KeepSuffix + }else { + $name.Substring(0, $maxLength) + } [io.path]::combine( $parent, $truncatedName ) }else { $Path @@ -465,6 +495,22 @@ Function Remove-InvalidFileNameCharsInsertedFiles { return $newName } +Function ConvertFrom-OneNoteEscapedName { + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true,Position = 0,ValueFromPipeline = $true)] + [AllowEmptyString()] + [string] + $Name + ) + + process { + # OneNote stores characters that aren't allowed in its section file names as '^' codes in the section's name, + # e.g. a section named 'AI / ML' is reported as 'AI ^M ML', and 'DECT & Bluetooth' as 'DECT^J Bluetooth' + $Name.Replace('^M', '/').Replace('^J', '&') + } +} + Function Encode-Markdown { [CmdletBinding()] param ( @@ -497,6 +543,213 @@ Function Encode-Markdown { $Name } +Function Remove-PageHeader { + [CmdletBinding()] + param ( + # The markdown lines produced by pandoc from a published page + [Parameter(Mandatory)] + [AllowEmptyCollection()] + [AllowEmptyString()] + [string[]] + $Lines + , + # The page's name. Without a date/time header, a first line matching it is removed too (we add our own heading) + [Parameter()] + [string] + $Title = '' + ) + + # A page published to docx starts with its title, date and time paragraphs (e.g. 'Docker', 'Donnerstag, 17. Oktober 2024', '09:50'), + # which pandoc renders as the first lines of the markdown. We add our own heading, so remove them - but only when they are really + # there: some pages (e.g. emails sent to OneNote) have no date/time paragraphs, and blindly dropping the first 6 lines removed real content. + # The title may be empty (an untitled page yields a line with just a non-breaking space). + $datePattern = '^(?=.*\p{L})(?=.*\b\d{4}\b)[^|#>*]{1,80}$' # Some text with a 4-digit year, in any locale + $timePattern = '^\d{1,2}:\d{2}(:\d{2})?(\s*[AaPp]\.?\s?[Mm]\.?)?$' # E.g. '09:50', '9:19 AM' + $clean = { param($s) $s.Replace([string][char]0x00A0, ' ').Trim() } + $lettersAndDigits = { param($s) $s -replace '[^\p{L}\p{N}]', '' } # Pandoc escapes/autolinks the title text, so compare only letters and digits + $nonBlank = @( + for ($i = 0; $i -lt $Lines.Count -and $i -lt 8; $i++) { + if (& $clean $Lines[$i]) { + $i + } + } + ) + $lastHeaderLine = -1 + if ($nonBlank.Count -ge 3 -and (& $clean $Lines[$nonBlank[1]]) -match $datePattern -and (& $clean $Lines[$nonBlank[2]]) -match $timePattern) { + $lastHeaderLine = $nonBlank[2] # Title, date, time + }elseif ($nonBlank.Count -ge 2 -and (& $clean $Lines[$nonBlank[0]]) -match $datePattern -and (& $clean $Lines[$nonBlank[1]]) -match $timePattern) { + $lastHeaderLine = $nonBlank[1] # Empty title, date, time + }elseif ($Title -and $nonBlank.Count -ge 1 -and (& $lettersAndDigits $Lines[$nonBlank[0]]) -and (& $lettersAndDigits $Lines[$nonBlank[0]]) -eq (& $lettersAndDigits $Title)) { + $lastHeaderLine = $nonBlank[0] # Title only + } + if ($lastHeaderLine -lt 0) { + $Lines + }elseif ($lastHeaderLine + 1 -lt $Lines.Count) { + $Lines[($lastHeaderLine + 1)..($Lines.Count - 1)] + }else { + '' + } +} + +Function Repair-MarkdownEscapes { + [CmdletBinding()] + param ( + # The markdown content produced by pandoc (any newline style) + [Parameter(Mandatory,Position = 0,ValueFromPipeline = $true)] + [AllowEmptyString()] + [string] + $Content + ) + + process { + # Pandoc escapes text that merely looks like markdown. OneNote pages often start paragraphs with a literal '- ' or '7.' (no real list), + # which ends up as '\- item' or '7\. item'. Rebuild those lists, and remove only the escapes that can't change how the text is rendered. + # Fenced code blocks and inline code are never touched. + $saved = New-Object 'System.Collections.Generic.List[string]' + $shield = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + $saved.Add($m.Value) + "MDSHIELD$( $saved.Count - 1 )END" + } + $text = $Content -replace '\r\n', "`n" + $text = [regex]::Replace($text, '(?ms)(?:^[ \t]*)?(`{3,})[^\n`]*\n.*?^[ \t]*\1`*[ \t]*$', $shield) + $text = [regex]::Replace($text, '(`+)(?!`)(.+?)(?]*\\?-[ \t\u00A0]*$\n?', '' # A lone '-' or '\-' line is an empty bullet (below a text line, '-' would even make a heading) + $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*)\\-[ \t\u00A0]+', '$1- ' + $text = $text -replace '(?m)^(\d+)\\\.[ \t]*\n(?:[ \t]*\n)+(?=\S)', '$1. ' # A number alone on its line, with the item text in the next paragraph + $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*\d+)\\\.[ \t\u00A0]+', '$1. ' + $listStart = '^[ \t\u00A0>]*(?:[-*+]|\d+[.)])\s' + $lines = $text -split "`n" + $isBlank = '^[ \t\u00A0>]*$' # Lines with only non-breaking spaces or an empty block quote ('> ') become blank later on (keepspaces) + $text = @( + for ($i = 0; $i -lt $lines.Count; $i++) { + # Drop the blank lines between two list items + if ($lines[$i] -match $isBlank -and $i -gt 0) { + $prev = $i - 1 + while ($prev -gt 0 -and $lines[$prev] -match $isBlank) { $prev-- } + $next = $i + 1 + while ($next -lt $lines.Count - 1 -and $lines[$next] -match $isBlank) { $next++ } + if ($lines[$prev] -match $listStart -and $next -lt $lines.Count -and $lines[$next] -match $listStart) { + continue + } + } + $lines[$i] + } + ) -join "`n" + + # Escapes that can't change the rendering + $unescape = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + if ($m.Value -eq '\\') { return $m.Value } # An escaped backslash is matched as a pair, so it can't hide the character after it + $c = $m.Value[1] + $start = $m.Index + $lineStart = if ($start -gt 0) { $text.LastIndexOf([char]10, $start - 1) + 1 } else { 0 } + $prefix = $text.Substring($lineStart, $start - $lineStart) + $before = if ($start -gt 0) { [string]$text[$start - 1] } else { '' } + $afterIndex = $start + 2 + $after = if ($afterIndex -lt $text.Length) { [string]$text[$afterIndex] } else { '' } + $atLineStart = $prefix.Trim(" `t>").Length -eq 0 + if (',."''%@/?!=+-#>'.Contains($c)) { + $keep = ('=+-#>'.Contains($c) -and $atLineStart) -or ($c -eq '.' -and $prefix -match '^[ \t>]*\d+$') -or ($c -eq '!' -and $after -eq '[') + if (!$keep -and $c -eq '#') { + $lineEnd = $text.IndexOf([char]10, $start) + $line = if ($lineEnd -lt 0) { $text.Substring($lineStart) } else { $text.Substring($lineStart, $lineEnd - $lineStart) } + $keep = $line -match '^\s*#{1,6}(?:\s|$)' # A closing '\#' in a heading stays + } + }elseif ($c -eq '*' -or $c -eq '_') { + $loneGap = ($before -eq '' -or $before -match '^\s$') -and ($after -eq '' -or $after -match '^\s$') + $intraword = $before -match '^[\p{L}\p{N}]$' -and $after -match '^[\p{L}\p{N}]$' + if ($c -eq '*') { + $pStart = if ($start -ge 2) { $text.LastIndexOf("`n`n", $start - 1, [System.StringComparison]::Ordinal) } else { -1 } + $pEnd = $text.IndexOf("`n`n", $start, [System.StringComparison]::Ordinal) + $from = if ($pStart -ge 0) { $pStart + 2 } else { 0 } + $to = if ($pEnd -ge 0) { $pEnd } else { $text.Length } + $paragraph = $text.Substring($from, $to - $from) + $keep = $atLineStart -or !($loneGap -or ($paragraph.Length - $paragraph.Replace('*', '').Length) -eq 1) + }else { + $keep = $atLineStart -or !($loneGap -or $intraword) + } + }else { + $keep = $true + } + if ($keep) { $m.Value } else { [string]$c } + } + $text = [regex]::Replace($text, '\\\\|\\.', $unescape) + + # Restore code (fences can contain inline code and vice versa, so repeat until nothing is left) + $restore = [System.Text.RegularExpressions.MatchEvaluator]{ param ($m) $saved[[int]$m.Groups[1].Value] } + while ($text -match 'MDSHIELD\d+END') { + $text = [regex]::Replace($text, 'MDSHIELD(\d+)END', $restore) + } + $text + } +} + +Function Add-MarkdownHardBreaks { + [CmdletBinding()] + param ( + # The markdown content produced by pandoc (any newline style) + [Parameter(Mandatory,Position = 0,ValueFromPipeline = $true)] + [AllowEmptyString()] + [string] + $Content + ) + + process { + # A single newline inside a paragraph is a 'soft' break: some programs show it as a line break, strict CommonMark renderers join the lines. + # A backslash at the end of a line (what pandoc writes for a line break) is not rendered the same way everywhere either, and + # 'keepescape' 1 and 2 delete it. Two trailing spaces are invisible and are rendered as a line break by every program. + # Fenced code blocks and inline code are never touched. + $saved = New-Object 'System.Collections.Generic.List[string]' + $shield = [System.Text.RegularExpressions.MatchEvaluator]{ + param ($m) + $saved.Add($m.Value) + "MDSHIELD$( $saved.Count - 1 )END" + } + $text = $Content -replace '\r\n', "`n" + $text = [regex]::Replace($text, '(?ms)(?:^[ \t]*)?(`{3,})[^\n`]*\n.*?^[ \t]*\1`*[ \t]*$', $shield) + $text = [regex]::Replace($text, '(`+)(?!`)(.+?)(?$( [char]0x00A0 )" + $heading = '^\s*#{1,6}(?:\s|$)' + $table = '^\s*\|' + $rule = '^\s*(?:[-*_=]\s*){3,}$' + $html = '^\s*<' + $code = '^\s*MDSHIELD\d+END\s*$' # A fenced code block on its own lines + $listStart = '^\s*(?:[-*+]|\d+[.)])\s' + $lines = $text -split "`n" + for ($i = 0; $i -lt $lines.Count - 1; $i++) { + $line = $lines[$i] + $next = $lines[$i + 1] + if ($line.Trim($blank.ToCharArray()).Length -eq 0 -or $next.Trim($blank.ToCharArray()).Length -eq 0) { continue } # Paragraph end. A bare '>' is a blank line in a block quote + if ($line.EndsWith(' ')) { continue } # Already a line break + if ($line -match $heading -or $line -match $table -or $line -match $html -or $line -match $code) { continue } + if ($line -match $rule -or $next -match $rule) { continue } + if ($next -match $listStart -or $next -match $table -or $next -match $heading -or $next -match $html -or $next -match $code) { continue } + $lines[$i] = $line.TrimEnd() + ' ' + } + $text = $lines -join "`n" + + # Restore code (fences can contain inline code and vice versa, so repeat until nothing is left) + $restore = [System.Text.RegularExpressions.MatchEvaluator]{ param ($m) $saved[[int]$m.Groups[1].Value] } + while ($text -match 'MDSHIELD\d+END') { + $text = [regex]::Replace($text, 'MDSHIELD(\d+)END', $restore) + } + $text + } +} + Function Set-ContentNoBom { [CmdletBinding()] param ( @@ -528,6 +781,45 @@ Function Set-ContentNoBom { } } +Function Remove-PreviousNotebookOutput { + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] + $NotesDestination + , + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] + $NotebookName + , + [Parameter()] + [switch] + $DryRun + ) + + # The same directory New-SectionGroupConversionConfig converts this notebook into + $nameCompat = $NotebookName | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars + if (! $nameCompat.Trim('-. ')) { + throw "Refusing to remove previous output of notebook '$NotebookName': its directory name would be empty" + } + $notebookDirectory = [io.path]::combine( $NotesDestination.TrimEnd('/').TrimEnd('\').Replace('\', [io.path]::DirectorySeparatorChar), $nameCompat ) + if (! (Test-Path -LiteralPath $notebookDirectory -PathType Container)) { + return + } + + # Keep the docx folder, so that 'usedocx = 2' can still reuse its files + foreach ($item in @( Get-ChildItem -LiteralPath $notebookDirectory -Force | Where-Object { $_.Name -ne 'docx' } )) { + "Removing previous output: $( $item.FullName )" | Write-Verbose + if (! $DryRun) { + # A non-Win32 path, so that Windows Powershell <= 5 can also remove files beyond the 260-character path limit + $literalPath = if ($PSVersionTable.PSVersion.Major -le 5 -and $env:OS -match 'windows') { "\\?\$( $item.FullName )" } else { $item.FullName } + Remove-Item -LiteralPath $literalPath -Recurse -Force -ErrorAction Stop + } + } +} + Function New-OneNoteConnection { [CmdletBinding()] param () @@ -681,7 +973,7 @@ Function New-SectionGroupConversionConfig { $cfg['object'] = $sectionGroup # Keep a reference to the SectionGroup object $cfg['kind'] = 'SectionGroup' $cfg['id'] = $sectionGroup.ID # E.g. {9570CCF6-17C2-4DCE-83A0-F58AE8914E29}{1}{B0} - $cfg['nameCompat'] = $sectionGroup.name | Remove-InvalidFileNameChars + $cfg['nameCompat'] = $sectionGroup.name | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars $cfg['levelsFromRoot'] = $LevelsFromRoot $cfg['uri'] = $sectionGroup.path # E.g. https://d.docs.live.net/0123456789abcdef/Skydrive Notebooks/mynotebook/mysectiongroup $cfg['notesDirectory'] = [io.path]::combine( $NotesDestination.TrimEnd('/').TrimEnd('\').Replace('\', [io.path]::DirectorySeparatorChar), $cfg['nameCompat'] ) # No need to truncate. Section Group and Section names have a max length of 50, so we should never hit the absolute path, file name, or directory name limits on Windows @@ -720,7 +1012,7 @@ Function New-SectionGroupConversionConfig { $sectionCfg['object'] = $section # Keep a reference to the Section object $sectionCfg['kind'] = 'Section' $sectionCfg['id'] = $section.ID # E.g {BE566C4F-73DC-43BD-AE7A-1954F8B22C2A}{1}{B0} - $sectionCfg['nameCompat'] = $section.name | Remove-InvalidFileNameChars + $sectionCfg['nameCompat'] = $section.name | ConvertFrom-OneNoteEscapedName | Remove-InvalidFileNameChars $sectionCfg['levelsFromRoot'] = $cfg['levelsFromRoot'] + 1 $sectionCfg['pathFromRoot'] = "$( $cfg['pathFromRoot'] )$( [io.path]::DirectorySeparatorChar )$( $sectionCfg['nameCompat'] )".Trim([io.path]::DirectorySeparatorChar) # No need to truncate. Section Group and Section names have a max length of 50, so we should never hit the absolute path, file name, or directory name limits on Windows $sectionCfg['pathFromRootCompat'] = $sectionCfg['pathFromRoot'] | Remove-InvalidFileNameChars @@ -800,17 +1092,18 @@ Function New-SectionGroupConversionConfig { $pageCfg['filePathRel'] = & { $filePathRel = "$( $pageCfg['pagePrefix'] )$( $pageCfg['nameCompat'] )" - # in case multiple pages with the same name exist in a section, postfix the filename + # In case multiple pages with the same name exist in a section, postfix the filename. + # Compare the names as they will be written (truncated to fit '.md'), so two long names that only differ + # after the cut-off are also recognized as duplicates. The postfix is kept when truncating below, so it can't be cut off again. + $pageCfg['filePathRelKey'] = $filePathRel | Truncate-PathFileName -Length ($config['mdFileNameAndFolderNameMaxLength']['value'] - 3) $recurrence = 0 foreach ($p in $sectionCfg['pages']) { - if ($p['pagePrefix'] -eq $pageCfg['pagePrefix'] -and $p['pathFromRoot'] -eq $pageCfg['pathFromRoot']) { + if ($p['filePathRelKey'] -eq $pageCfg['filePathRelKey']) { $recurrence++ } } - if ($recurrence -gt 0) { - $filePathRel = "$filePathRel-$recurrence" - } - $filePathRel | Truncate-PathFileName -Length $config['mdFileNameAndFolderNameMaxLength']['value'] # Truncate to no more than 255 characters so we don't hit the folder name limit on most file systems on Windows / Linux + $pageCfg['filePathRelPostfix'] = if ($recurrence -gt 0) { "-$recurrence" } else { '' } + "$filePathRel$( $pageCfg['filePathRelPostfix'] )" | Truncate-PathFileName -Length $config['mdFileNameAndFolderNameMaxLength']['value'] -KeepSuffix $pageCfg['filePathRelPostfix'] # Truncate to no more than 255 characters so we don't hit the folder name limit on most file systems on Windows / Linux } $pageCfg['filePathRelUnderscore'] = $pageCfg['filePathRel'].Replace( [io.path]::DirectorySeparatorChar, '_' ) $pageCfg['filePathNormal'] = & { @@ -819,7 +1112,7 @@ Function New-SectionGroupConversionConfig { }else { [io.path]::combine( $cfg['notesDirectory'], $sectionCfg['nameCompat'], "$( $pageCfg['filePathRel'] )" ) } - "$( $pathWithoutExtension | Truncate-PathFileName -Length ($config['mdFileNameAndFolderNameMaxLength']['value'] - 3) ).md" # Truncate to no more than 255 characters so we don't hit the file name limit on Windows / Linux + "$( $pathWithoutExtension | Truncate-PathFileName -Length ($config['mdFileNameAndFolderNameMaxLength']['value'] - 3) -KeepSuffix $pageCfg['filePathRelPostfix'] ).md" # Truncate to no more than 255 characters so we don't hit the file name limit on Windows / Linux } $pageCfg['filePathLong'] = "\\?\$( $pageCfg['filePathNormal'] )" # A non-Win32 path. Prefixing with '\\?\' allows Windows Powershell <= 5 (based on Win32) to support long absolute paths. $pageCfg['filePath'] = if ($PSVersionTable.PSVersion.Major -le 5) { @@ -896,7 +1189,9 @@ Function New-SectionGroupConversionConfig { description = 'Change inserted attachment(s) filename references' replacements = @( @{ - searchRegex = [regex]::Escape( $attachmentCfg['object'].preferredName ) + # Only where the name is not part of a longer file name: with attachments '11-36-00002.pdf' and 'Datenblatt_11-36-00002.pdf' + # on one page, a plain search for the first one also matched inside the second, nesting one link inside the other + searchRegex = '(?` after unordered lists' replacements = @( - # Remove non-breaking spaces + # Non-breaking spaces: web-clipped content is full of them, often as the only space between two words + # (e.g. 'theWindowskey'). Deleting them all glued those words together ('Windowskey'), so: + # remove them at the end of a line (incl. lines that only contain them), since converting them there could + # create a trailing-double-space hard line break, @{ - searchRegex = [regex]::Escape([char]0x00A0) + searchRegex = '(?m)[ \t\u00A0]*\u00A0[ \t\u00A0]*$' replacement = '' } + # remove them at the start of a line, keeping its indentation, since converting them there could turn the line into an indented code block, + @{ + searchRegex = '(?m)^([ \t]*)\u00A0[ \t\u00A0]*' + replacement = '$1' + } + # and turn any other run of them (with adjacent spaces) into a single space + @{ + searchRegex = '[ \t]*\u00A0[ \t\u00A0]*' + replacement = ' ' + } # Remove an extra newline between each occurrence of '- some unordered list item' @{ searchRegex = '(\s*)- ([^\r\n]*)\r*\n\r*\n(?=\s*-)' @@ -1241,11 +1550,13 @@ Function Convert-OneNotePage { foreach ($image in $images) { # Rename Image try { - $newimageName = if ($config['medialocation']['value'] -eq 2) { + # The page-name prefix may contain characters that are valid in file names but break the markdown image link, + # e.g. '#' (read as a URL fragment) or "'" (e.g. from a page named "c't: Newsletter #10"). Strip those, like for inserted attachments. + $newimageName = $( if ($config['medialocation']['value'] -eq 2) { "$( $pageCfg['filePathRelUnderscore'] )-$($image.BaseName)$($image.Extension)" }else { "$( $pageCfg['pathFromRootCompat'] )-$($image.BaseName)$($image.Extension)" - } + } ) | Remove-InvalidFileNameCharsInsertedFiles $newimagePath = [io.path]::combine( $pageCfg['mediaPath'], $newimageName ) "Moving image: $( $image.FullName ) to $( $newimagePath )" | Write-Verbose if (!$config['dryRun']['value']) { @@ -1273,15 +1584,14 @@ Function Convert-OneNotePage { if (!$config['dryRun']['value']) { # Get markdown content $content = @( Get-Content -LiteralPath $pageCfg['filePath'] -ErrorAction Stop ) # Use -LiteralPath so that characters like '(', ')', '[', ']', '`', "'", '"' are supported. Or else we will get an error "Cannot find path 'xxx' because it does not exist" - $content = @( - if ($content.Count -gt 6) { - # Discard first 6 lines which contain a header, created date, and time. We are going to add our own header - $content[6..($content.Count - 1)] - }else { - # Empty page - '' - } - ) -join "`n" + # Discard the page's title, date and time lines (if present). We are going to add our own header + $content = @( Remove-PageHeader -Lines $content -Title $pageCfg['object'].name ) -join "`n" + if ($config['hardbreaks']['value'] -eq 2) { + $content = Add-MarkdownHardBreaks -Content $content + } + if ($config['keepescape']['value'] -eq 4) { + $content = Repair-MarkdownEscapes -Content $content + } } # Mutate @@ -1432,6 +1742,10 @@ Function Convert-OneNote2MarkDown { $pageConversionConfigsAll = @() foreach ($notebook in $notebooks) { "`nConverting notebook '$( $notebook.name )'... (Ignoring deleted notes)" | Write-Host -ForegroundColor Cyan + if ($config['overwrite']['value'] -eq 2) { + "Removing previous output of notebook '$( $notebook.name )' (keeping its docx folder)" | Write-Host -ForegroundColor Cyan + Remove-PreviousNotebookOutput -NotesDestination $config['notesdestpath']['value'] -NotebookName $notebook.name -DryRun:([bool]$config['dryRun']['value']) + } New-SectionGroupConversionConfig -OneNoteConnection $OneNote -NotesDestination $config['notesdestpath']['value'] -Config $config -SectionGroups $notebook -LevelsFromRoot 0 -ErrorVariable +totalerr | Tee-Object -Variable pageConversionConfigs | Convert-OneNotePage -OneNoteConnection $OneNote -Config $config -ErrorVariable +totalerr "`nDone converting notebook '$( $notebook.name )' with $( ($pageConversionConfigs | Measure-object).Count ) notes." | Write-Host -ForegroundColor Cyan $pageConversionConfigsAll += $pageConversionConfigs diff --git a/FORK-CHANGES.md b/FORK-CHANGES.md new file mode 100644 index 0000000..b8c7b7e --- /dev/null +++ b/FORK-CHANGES.md @@ -0,0 +1,180 @@ +# What this fork changes + +This is a fork of [theohbrothers/ConvertOneNote2MarkDown](https://github.com/theohbrothers/ConvertOneNote2MarkDown) (the PowerShell script that exports OneNote pages through Pandoc to Markdown). It is based on upstream `master` v2.23.1 (2026-09-24). Everything not listed here works as in upstream, and the defaults are unchanged: you only get a different result where a fix removes a bug, or when you switch on a new option. + +The fixes were found by converting two real notebooks (about 200 pages each, German and English, many web clippings) and reading the result. Five of them were offered upstream as pull requests, which are still open. + +## Do I need this? + +Look at your existing export. If you see one of these, this fork probably helps: + +| What you see in the exported notes | Fix | Where | +|---|---|---| +| Words run together: `Windowskey` instead of `Windows key` | [Non-breaking spaces](#words-were-glued-together) | PR [#191](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/191) | +| The first lines of some pages are missing (a table, a `Subject:` line, the first command) | [Page header](#page-content-was-deleted) | PR [#192](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/192) | +| Pages with the same title overwrite each other; attachment links like `[a.pdf]([b-a.pdf](...))`; images with `#` in the name show as broken | [File names](#file-names-and-links) | PR [#193](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/193) | +| Folder names like `AI ^M ML` or `DECT^J Bluetooth` | [Section names](#section-names-with-onenote-escape-codes) | PR [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | +| Old notes, folders or images stay after you renamed or deleted them in OneNote | [`overwrite = 2`](#new-option-overwrite) | PR [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | +| Lists show as `\- item` or `7\. item`, backslashes at line ends, loose lists, `--\>` | [`keepescape = 4`](#new-option-keepescape--4) | PR [#217](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/217) | +| Lines of a paragraph run together in some programs, or show a `\` at the line end | [`hardbreaks = 2`](#new-option-hardbreaks) | this fork only | +| `Created:` / `Modified:` has the wrong time zone offset (for example +05:30) | [Time zone](#page-content-was-deleted) | PR #192 | + +## How to use the fork + +1. Get the script: `git clone https://github.com/klangme1ster/ConvertOneNote2MarkDown` and use branch `main`. It contains all fixes below. Each pull-request branch (`fix/...`, `enhancement/...`) contains one fix on top of upstream, if you only want some. +2. Copy `config.example.ps1` to `config.ps1` next to the script and edit it. The rest (requirements, running, FAQ) is as in the [README](README.md). + +Recommended settings: + +```powershell +$keepescape = 4 # rebuild lists, drop escapes that don't matter (see below). 3 keeps every backslash +$conversion = 'markdown-simple_tables-multiline_tables-grid_tables+pipe_tables-smart' # '-smart' stops Pandoc escaping every quote as \" +$overwrite = 2 # when you re-export into an existing folder +``` + +Do not add `-link_attributes` or `-bracketed_spans` to `$conversion`: Pandoc then writes raw `` and `` HTML instead of Markdown. + +## The changes + +### Words were glued together + +*Symptom:* `the Windows key` became `Windowskey`. In one notebook 193 of 197 pages were affected. + +*Cause:* with `keepspaces = 1` (the default) every non-breaking space was deleted. Web-clipped content often has a non-breaking space as the only space between two words. + +*Fix:* they are now removed only at the start and end of a line (keeping the indentation, because a leading space could turn the line into a code block, a trailing double space into a hard line break). Anywhere else they become one normal space. + +### Page content was deleted + +*Symptom:* pages without a date line, such as emails sent to OneNote, lost real content (a Tags table, a `Subject:` line, the first command of the page). + +*Cause:* the script always dropped the first 6 lines of Pandoc's output, assuming they are title, date and time. + +*Fix:* the new function `Remove-PageHeader` removes the header only if a date and a time line are really there (any language, `9:19 AM` works too). Otherwise it removes a first line only if it repeats the page title. + +Also in this PR: `Created:` / `Modified:` used the format `zz00`, which printed only the hours of the offset (wrong for +05:30). The output format stays `+0200`, but the value is now right. + +### File names and links + +Three bugs with the same cause, names built from the page path: + +- **Duplicate page names overwrote each other.** The `-1`, `-2` postfix was added before the name was cut to `mdFileNameAndFolderNameMaxLength`, so the postfix could be cut off again and two pages were written to the same file. Duplicates are now detected on the cut name, and the postfix survives the cut. +- **Attachment links were nested.** With `11-36-00002.pdf` and `Datenblatt_11-36-00002.pdf` on one page, the first name also matched inside the second and the link was inserted into the other link. Now a name only matches if it is not part of a longer file name. +- **Image names broke links.** Names such as `c't-expert-community--Newsletter-#10-image1.jpg` contain `#`, which Markdown reads as a URL fragment. Image names now go through the same clean-up as attachments. + +### Section names with OneNote escape codes + +OneNote stores characters that are not allowed in a section file name as `^` codes: `AI / ML` is reported as `AI ^M ML`, `DECT & Bluetooth` as `DECT^J Bluetooth`. These ended up in folder names. `ConvertFrom-OneNoteEscapedName` decodes `^M` to `/` (then `-`, as for any `/`) and `^J` to `&` for section, section group and notebook names. Only these two codes have been seen; others would still show as `^`. + +*Note:* if you already exported before, these folders get new names (`AI ^M ML` becomes `AI-ML`), so links that point to the old folder names need updating. + +### New option: `overwrite` + +`overwrite = 2` removes a notebook's previous output before converting it, so notes, folders and media that were renamed or deleted in OneNote since the last run do not linger. The notebook's `docx` folder is always kept, so `usedocx = 2` still reuses it. Only the notebook's own folder under `notesdestpath` is touched. Read-only files and paths over 260 characters are handled. The default `1` keeps the previous output (old behaviour). + +### New option: `keepescape = 4` + +*Symptom:* in a typical export many lists are broken. Many people type `- item` or `7. item` in OneNote without using real lists. Pandoc is faithful and escapes this text so it does not turn into a list: + +```markdown +\- first point + +\- second point + +7\. numbered step\ +next line --\> result +``` + +That renders as plain paragraphs with a hyphen in front, a literal `\` at line ends, and loose spacing. Setting `keepescape = 1` removes every backslash, but also those in Windows paths (`C:\Program Files`) and in text you meant to escape. `2` and `3` do not repair the lists. + +*Fix:* `keepescape = 4` runs `Repair-MarkdownEscapes` on the Pandoc output: + +```markdown +- first point +- second point + +7. numbered step +next line --> result +``` + +- Rebuilds lists from `\- item` and `7\. item` (also inside block quotes, and when a non-breaking space follows the marker). Removes the blank lines between list items and lone `-` lines (empty bullets, or an accidental heading underline). +- A `\` at the end of a line (Pandoc's hard line break) becomes two trailing spaces, or is dropped at the end of a paragraph. +- Removes escapes that cannot change the rendering (`--\>`, `snake\_case`, `\.` in the middle of a line). +- Keeps escapes that matter: at the start of a line, `\[`, `\]`, `\*` pairs, `\\`, Windows and UNC paths. +- Code fences and inline code are never touched. +- Modes 1 to 3 and the default (`1`) are unchanged. + +*Result on two real notebooks (393 pages, same `.docx` files, mode 3 against mode 4):* + +| | mode 3 | mode 4 | +|---|---|---| +| escaped bullets `\-` | 601 | 0 | +| blank lines between list items | 67 | 0 | +| line-end backslashes | 1804 | 0 | +| lone `-` lines | 29 | 0 | +| escaped numbers `7\.` | 133 | 0 | + +The letters and digits of every page are identical in both modes, so no text is lost. The remaining escapes (`\>`, `\*`, `\_`) are the ones kept on purpose. Details and the decision: [docs/adr/0001-keepescape-mode-4.md](docs/adr/0001-keepescape-mode-4.md) and [docs/TRANSCRIPTION-QUALITY-REPORT.md](docs/TRANSCRIPTION-QUALITY-REPORT.md). + +*Caveat:* a literal `- ` at the start of a line is now treated as a list. If you meant it as plain text, keep `keepescape = 3`. + +### New option: `hardbreaks` + +*Symptom:* in some programs lines of one paragraph run together into a single line, in others they appear as separate lines. Markdown knows two kinds of line break: a single newline is a *soft* break (rendered as a space by strict CommonMark renderers, as a line break by others), and two spaces at the end of the line are a *hard* break that every program renders as a break. OneNote's line breaks (Shift+Enter) are written by Pandoc as a backslash at the line end, which is not rendered the same way everywhere either. `keepescape = 1` and `2` even delete that backslash, so the line break is lost. + +*Fix:* `hardbreaks = 2` runs `Add-MarkdownHardBreaks` before the `keepescape` replacements: + +```markdown +first line\ +second line +third line +``` + +becomes (the two trailing spaces are invisible) + +```markdown +first line +second line +third line +``` + +- A backslash at the end of a line (Pandoc's line break) becomes two spaces. A literal backslash is written by Pandoc as `\\` and stays as text. +- Every other line that is directly followed by another line of the same paragraph gets two spaces. +- Left alone: blank lines and paragraph ends, headings, tables, rules, HTML, code blocks and inline code, and a line before a list item. +- Default `1` changes nothing. The option works with every `keepescape` mode; with `keepescape = 4` the line-end backslash is converted anyway. +- Checked on one real notebook (196 pages, `keepescape = 1`): with `hardbreaks = 1` all 418 line breaks inside paragraphs are soft (the backslashes are gone); with `hardbreaks = 2` 415 are hard and 3 stay soft. The letters and digits of every page are identical. +- Limitation: a few lines that were separated only by a line of non-breaking spaces or a bare `>` end up next to each other after `keepspaces` removes that line; they stay soft. + +### Not changed on purpose + +- **`Modified` before `Created`:** on some pages the modified time is 1 to 2 hours before the created time. That is OneNote's own data (probably pages made by the web clipper, which store local time labelled as UTC); the script converts it correctly. +- **`>` removed at the start of lines** (`keepspaces = 1`): deliberate. Pandoc renders OneNote's indented paragraphs as block quotes, and this removes the indentation. + +## Status of the pull requests + +| PR | Branch | Contains | +|---|---|---| +| [#191](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/191) | `fix/keepspaces-keep-non-breaking-spaces-between-words-as-spaces` | Words glued together | +| [#192](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/192) | `fix/convert-remove-page-header-only-when-present` | Page content deleted, time zone offset | +| [#193](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/193) | `fix/file-names-keep-duplicate-postfix-and-avoid-broken-links` | Duplicate page names, nested attachment links, image names | +| [#194](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/194) | `fix/file-names-decode-onenote-escape-codes-in-section-names` | Section names with escape codes | +| [#195](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/195) | `enhancement/configuration-add-option-overwrite-to-remove-previous-output` | The `overwrite` option | + +The five PRs are independent and merge cleanly in any order. Two more PRs were opened on 2026-11-01: [#217](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/217) for `keepescape = 4` and [#216](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/216), which removes the retired `windows-2019` job and the unusable PowerShell 6.x/7.0 container jobs from the CI workflow (this fork's `main` has the same CI change). #216 should be merged first, so the CI of the other PRs can finish. The PR texts are in [docs/pr-descriptions.md](docs/pr-descriptions.md). If upstream merges the PRs, `main` can simply be replaced by upstream's version. + +`main` differs from the merged PR branches in one line: `Remove-PreviousNotebookOutput` also decodes the escape codes in the notebook name. + +## Development notes + +- **Keep both `.ps1` files pure ASCII.** Windows PowerShell 5.1 reads scripts without a BOM as ANSI, so a UTF-8 character (for example a literal non-breaking space in a regex) breaks parsing or silently changes the pattern. Write `\u00A0` in regexes and `[char]0x00A0` in strings. Check: `grep -c '[^ -~ ]' *.ps1` must print `0`. +- **Run the tests without installing anything:** `test/test.ps1` installs Pester 4 into the user profile. Instead: + + ```powershell + Save-PSResource -Name Pester -Version 4.10.1 -Path -TrustRepository # PowerShell 7 + # then, in Windows PowerShell 5.1: + Import-Module \Pester\4.10.1\Pester.psd1 -Force + Invoke-Pester -Script . -Tag Unit + ``` + + Do not set `$ErrorActionPreference = 'Stop'` for the run: several tests expect non-terminating errors. +- **Test status:** 130 unit tests. The only failure, "Throw if -Length is greater than 255", depends on the system language: it expects the English .NET error message and fails on a German Windows, with or without these changes (see [docs/known-issues.md](docs/known-issues.md)). diff --git a/README.md b/README.md index c23006f..c857e75 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![github-actions](https://github.com/theohbrothers/ConvertOneNote2MarkDown/actions/workflows/ci-master-pr.yml/badge.svg?branch=master)](https://github.com/theohbrothers/ConvertOneNote2MarkDown/actions/workflows/ci-master-pr.yml) [![github-release](https://img.shields.io/github/v/release/theohbrothers/ConvertOneNote2MarkDown?style=flat-square)](https://github.com/theohbrothers/ConvertOneNote2MarkDown/releases/) +> **This is a fork** of [theohbrothers/ConvertOneNote2MarkDown](https://github.com/theohbrothers/ConvertOneNote2MarkDown) with bug fixes (words glued together, missing page content, overwritten duplicate pages, broken links) and two new options: `overwrite` (remove a notebook's old output first) and `keepescape = 4` (repair lists and escapes that Pandoc writes as `\- item`, `7\. item`). See [FORK-CHANGES.md](FORK-CHANGES.md) for what changed, with examples and when you need it. + Ready to make the step to Markdown and saying farewell to your OneNote, EverNote or whatever proprietary note taking tool you are using? Nothing beats clear text, right? Read on! The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote Object Model on your workstation to convert all OneNote pages to Word documents and then utilizes Pandoc to convert the Word documents to Markdown (`.md`) format. @@ -29,10 +31,12 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote * `markdown_phpextra` (PHP Markdown Extra) * `markdown_strict` (original unextended Markdown) * Improved headers, with title now as a `#` heading, standardized `DateTime` format for created and modified dates, and horizontal line to separate from rest of document -* Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines, and `>` after unordered lists, which are created when converting with Pandoc -* Choose whether to remove `\` escape symbol that are created when converting with Pandoc +* Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines and line ends (other non-breaking spaces become normal spaces), and `>` after unordered lists, which are created when converting with Pandoc +* Choose whether to remove `\` escape symbol that are created when converting with Pandoc. Mode `4` (`$keepescape = 4`) keeps only the escapes that matter and rebuilds lists that Pandoc escaped (`\- item`, `7\. item`); see [FORK-CHANGES.md](FORK-CHANGES.md#new-option-keepescape--4) +* Choose whether to force hard line breaks (`$hardbreaks = 2`): lines that are followed by another line of the same paragraph end with two spaces, which every markdown program renders as a line break. Invisible, unlike a `\` at the line end, and it survives `$keepescape = 1` * Choose whether to use Line Feed (`LF`) or Carriage Return + Line Feed (`CRLF`) for new lines * Choose whether to include a `.pdf` export alongside the `.md` file. `.md` does not preserve `InkDrawing` (i.e. overlayed drawings, highlights, pen marks) absolute positions within a page, but a `.pdf` export is a complete page snapshot that preserves `InkDrawing` absolute positions within a page. +* Choose whether to **remove a notebook's previous output** before converting it (`$overwrite = 2`), so notes, folders and media that were renamed or deleted in OneNote since the last run don't linger. The notebook's `docx` folder is always kept, so existing `.docx` files can still be reused. * Detailed logs. Run the script with `-Verbose` to see detailed logs of each page's conversion. ## Known Issues diff --git a/config.example.ps1 b/config.example.ps1 index 6077bdf..30b8c03 100644 --- a/config.example.ps1 +++ b/config.example.ps1 @@ -77,8 +77,14 @@ $keepspaces = 1 # 1: Clear all '\' characters - Default # 2: Clear all '\' characters except those preceding alphanumeric characters # 3: Keep '\' symbol escape +# 4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter $keepescape = 1 +# Whether to end lines that are followed by another line of the same paragraph with two spaces, which is a hard line break in every markdown program. Without it a single newline is a 'soft' break, which some programs show as a line break and others join into one line; and keepescape 1 and 2 delete the backslash that pandoc writes for a line break. Headings, tables, rules, HTML, code and the lines before a list item are left alone +# 1: Leave line breaks as they are - Default +# 2: Force hard line breaks (two spaces at the end of the line) +$hardbreaks = 1 + # Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new lines # 1: LF (unix) - Default # 2: CRLF (windows) diff --git a/docs/TRANSCRIPTION-QUALITY-REPORT.md b/docs/TRANSCRIPTION-QUALITY-REPORT.md new file mode 100644 index 0000000..2a38778 --- /dev/null +++ b/docs/TRANSCRIPTION-QUALITY-REPORT.md @@ -0,0 +1,92 @@ +# Problem report: Markdown transcription quality + +Written 2026-11-01 from the Markdown cleanup side (`c:\Users\roman\SynologyDrive\Code\py\markdown_cleaner`). +This report is evidence plus a proposal. Nothing in this folder has been changed. + +## Setup the findings come from + +- `config.ps1`: `$keepescape = 3` (keep all Pandoc escapes), `$keepspaces = 1`, + `$conversion = 'markdown-simple_tables-multiline_tables-grid_tables+pipe_tables-smart'`. +- Output scanned: 392 `.md` notes in `c:\Users\roman\SynologyDrive\Dokumente\onenote2md` (read only). +- Example page: `Notizbuch-von-Roman\Teufel\JIRA-Prozess-Wiki.md` (list-heavy, badly affected). +- Not verified: whether the `.docx` of these pages still exists (`usedocx = 2`); I could not find one. + The Pandoc statements below are from reading the converter code and from Pandoc's known writer behaviour, + not from re-running a conversion. + +## Findings + +Counts are pages affected / occurrences, from a regex scan of the 392 notes. + +| # | Artifact in the output | Pages / occurrences | Effect when rendered | +|---|---|---|---| +| 1 | Escaped list marker at line start: `\- item` | 45 / 597 | Literal "- item" paragraphs, not a list | +| 2 | List items separated by a blank line (loose, mostly because of #1) | 65 / 369 | Wide, spaced list; in source one blank line per item | +| 3 | Line-end backslash (Pandoc `escaped_line_breaks`) | 89 / 1810 | `\` is a hard break only in CommonMark-style renderers; at the end of a paragraph it renders as a literal `\` | +| 4 | Lone `-` or `\-` line | 23 / 35 | Empty bullet; a bare `-` directly below a text line is a setext underline and turns that line into a heading | +| 5 | Escaped number: `7\.` | 22 / 133 | Not a numbered list; in `JIRA-Prozess-Wiki.md` the number even sits alone in a paragraph, with the item text in the next one | +| 6 | Needless escapes: `\>` (142 pages / 598), `\*` (33 / 489), `\_` (19 / 182) | | Noise in the source (`--\>` for `-->`, `snake\_case`) | + +## Cause (read from the code) + +1. **`keepspaces = 1` cannot see escaped markers.** + The list regexes in `ConvertOneNote2MarkDown-v2.ps1` (around line 932) are + `(\s*)- ([^\r\n]*)\r*\n\r*\n(?=\s*-)` and the numbered variant. With `keepescape = 3` the next item starts with `\-`, + so the look-ahead `(?=\s*-)` fails and the blank line stays. The same applies to `\d+\.` vs `\d+\\\.`. +2. **`\-` and `7\.` come from paragraphs that start with literal text "- " or "7."**, not from real OneNote lists + (a real list would come out of Pandoc as `- item`). Pandoc is faithful here and escapes the text so it does not + turn into a list. The converter could recognise this pattern and rebuild a real list. This is the main transcription gap. + To confirm, run Pandoc on the `.docx` of one of the affected pages and look at the paragraph styles. +3. **`keepescape` offers only coarse choices.** + `1` deletes every `\` in the file (`[regex]::Escape('\')` → empty). That also removes backslashes that carry + meaning: Windows paths, UNC paths (`\\server\share`), a deliberately escaped `\*`, and the `\` hard breaks. + `2` keeps escapes before alphanumerics only. `3` keeps everything, which is where the artifacts above come from. +4. **The `\` hard break** is Pandoc's `escaped_line_breaks` extension, enabled in the default `markdown` flavour. + Pandoc documents that disabling it (`markdown-escaped_line_breaks`) makes the writer emit two trailing spaces + instead. I did not test this against a real page. + +## Proposal + +Add a fourth `keepescape` mode (or make it the default) that removes only escapes which cannot change the rendering, +and rebuilds lists from escaped markers. A working Python reference implementation with 24 unit tests exists: + +- `c:\Users\roman\SynologyDrive\Code\py\markdown_cleaner\qon_helper.py` + - `escapes`: unescapes `_` `*` `.` `-` `+` `=` `#` `>` `!` and a few more only where that cannot change the + rendering (not at line start, not before `[`, `*` only when it cannot pair). + - `fix_lists`: `\- ` → `- `, `7\.` → `7.` (also when the number stands alone before its text), + drops lone `-` lines, drops blank lines between list items, turns line-end `\` into two trailing spaces + or removes it at paragraph end. Code fences, inline code and front matter are skipped. + - `repair` = `fix_lists` + escapes + collapse blank lines + hard breaks. +- Dry run of `repair` over the 392 notes: 245 pages changed, 0 errors; `repair` run twice differs in 4 pages + (whitespace only). +- Porting the rules to PowerShell (as post-processing in the existing replacement list) is the minimal change. + Alternative: run the Python filter as a post-step. That adds a Python dependency to this tool, so the + PowerShell port is probably the better fit. + +Also worth checking in the converter: + +- The first lines of many pages are `| **Tags:** #Teufel #Prozess |` followed by `|---|`, a one-cell table + with only a header row. I did not find where it is produced. If it is meant to be a tags line, a plain line + would render better and QOwnNotes could read the tags. +- `keepspaces` also strips `>` after lists. That is documented as deliberate; leave it. + +## Acceptance criteria + +- Re-export of the 392-page sample (or at least `JIRA-Prozess-Wiki.md`): + the scan counts for findings 1, 2, 4, 5 drop to zero; #3 and #6 drop to the genuinely needed cases. +- No intended backslash is lost (Windows paths, `\\server\share`, `\*` in math or code-like text). + Add Pester cases for these. +- Existing Pester tests stay unchanged; new behaviour gets new tests. +- Default behaviour stays the same unless the user picks the new mode (it changes rendering for pages that + start lines with "- "). +- Record the change in `FORK-CHANGES.md`. + +## Addendum 2026-11-01: verification and result + +Checked with Pandoc 3.11 on the `.docx` of `JIRA-Prozess-Wiki` (kept in `c:\temp\notes\\docx`) and on all 197 `.docx` of the notebook: + +- The `.docx` has no list numbering (`w:numPr` count 0, all paragraphs style `StandardWeb`). The `\-` and `7\.` are therefore literal text typed in OneNote; Pandoc is faithful and escapes it. This confirms cause 2. +- `-escaped_line_breaks` in `$conversion` does turn the line-end `\` into two trailing spaces (confirmed). Not needed any more with `keepescape = 4`. +- The one-cell `Tags` table is part of the OneNote content (it is in the Pandoc output of the `.docx`); the script does not produce it. +- Implemented as `keepescape = 4` (`Repair-MarkdownEscapes`, Pester tests added). First version was identical to `qon_helper.escapes(fix_lists(...))` on all 197 `.docx` pages. A real run against OneNote then showed three gaps, now fixed: lists inside block quotes (`> 1\. item`), non-breaking spaces after the marker, and several blank or nbsp-only lines between items. +- Real run (script + OneNote, same notebook, `keepescape` 3 vs 4, 197 pages, no errors), scan counts before/after: escaped bullets 503 to 0, blank lines between list items 45 to 0, line-end backslashes 1039 to 0, lone dash lines 25 to 0, escaped numbers 87 to 0, `\>` 316 to 25, `\*` 366 to 330, `\_` 9 to 9 (the rest is kept on purpose). Letters and digits are identical on every page. +- Not yet done: re-export into the real destination (`onenote2md`) and a look at the rendering in QOwnNotes; the `Tags` table is unchanged. diff --git a/docs/adr/0001-keepescape-mode-4.md b/docs/adr/0001-keepescape-mode-4.md new file mode 100644 index 0000000..9e691f0 --- /dev/null +++ b/docs/adr/0001-keepescape-mode-4.md @@ -0,0 +1,20 @@ +# 0001: keepescape mode 4 (repair pandoc escapes) + +## Status +Proposed (2026-11-01) + +## Context +With `keepescape = 3` the exported Markdown keeps every escape that Pandoc writes. OneNote pages often contain paragraphs that start with a literal `- ` or `7.`; Pandoc escapes them (`\- item`, `7\. item`), so the notes render as plain paragraphs instead of lists, with blank lines between the items. Pandoc's line-end `\` and many needless escapes (`\>`, `\_`) add noise. `keepescape = 1` removes every backslash, which also destroys Windows paths and intended escapes; `2` and `3` don't fix the list problem. See `docs/TRANSCRIPTION-QUALITY-REPORT.md`. + +## Decision +Add `keepescape = 4`, implemented as the function `Repair-MarkdownEscapes` that runs on the Pandoc output before the existing mutations. It rebuilds lists, converts line-end backslashes to two spaces, and removes only escapes that cannot change the rendering. Code is shielded. The default stays `1`; modes 1 to 3 are untouched. + +Rejected alternatives: +- Regex entries in the mutation list like modes 1 and 2: they cannot skip code blocks and inline code. +- Running the Python `markdown_cleaner` as a post-step: adds a Python dependency to this tool. +- Only `-escaped_line_breaks` in `$conversion`: fixes the line-end backslash, nothing else. + +## Consequences +- Opt-in, so no breaking change for existing users. Notes exported with mode 4 render lists as intended. +- The rules exist twice (here and in `qon_helper.py`); on all 197 pages of the test notebook both give identical output. Changes to one should be mirrored. +- Pages where a literal `- ` was meant as text, not as a list, become lists. diff --git a/docs/ideas.md b/docs/ideas.md new file mode 100644 index 0000000..c3b6f5d --- /dev/null +++ b/docs/ideas.md @@ -0,0 +1,5 @@ +# Ideas + +- [ ] 2026-11-01: One-cell `| **Tags:** ... |` table at page start - comes from the OneNote content itself (pandoc output of the .docx), not from this script; decide whether to turn it into a plain tags line +- [ ] 2026-11-01: Re-export the notebooks with `$keepescape = 4` and compare against the 392 notes in onenote2md +- [ ] 2026-11-01: Offer keepescape mode 4 upstream as a further PR (#196) once it has proven itself locally diff --git a/docs/known-issues.md b/docs/known-issues.md new file mode 100644 index 0000000..43fc313 --- /dev/null +++ b/docs/known-issues.md @@ -0,0 +1,3 @@ +# Known issues + +- 2026-09-24 | Tests: Pester test "Throw if -Length is greater than 255" expects the English .NET error message and fails on a German Windows, with or without local changes | Workaround: ignore this single failure (130 unit tests, 1 expected failure) diff --git a/docs/pr-descriptions.md b/docs/pr-descriptions.md new file mode 100644 index 0000000..2ea9efc --- /dev/null +++ b/docs/pr-descriptions.md @@ -0,0 +1,164 @@ +# Pull request descriptions + +Paste each title and description into GitHub's "Open a pull request" form for the matching branch. + +All five PRs are independent and merge cleanly in any order and combination. On a German Windows, one existing test ("Throw if -Length is greater than 255") fails with or without these changes: it expects the English .NET error message. + +--- + +## 1. Branch `fix/keepspaces-keep-non-breaking-spaces-between-words-as-spaces` + +**Title:** Fix (keepspaces): Keep non-breaking spaces between words as spaces + +**Description:** + +With `keepspaces = 1` (the default), every non-breaking space is removed. Content clipped from web pages often has a non-breaking space as the *only* space between two words, so words get glued together. For example, a page containing `press the Windowskey at the same timeto open` came out as `press the Windowskey at the same timeto open`. In my notebooks, 193 of 197 pages contained non-breaking spaces (4306 in total). + +This PR changes the rule: + +- **Start of a line:** removed, keeping the line's indentation. Turning them into spaces there could create an indented code block. +- **End of a line, and lines that contain only non-breaking spaces:** removed. Turning them into spaces there could create a trailing-double-space hard line break. +- **Anywhere else:** a run of them, together with adjacent spaces, becomes a single space. + +The option's description and the README are updated to match. The `keepspaces` test now also covers words separated by non-breaking spaces, a line that starts with them, and an indented line. + +--- + +## 2. Branch `fix/convert-remove-page-header-only-when-present` + +**Title:** Fix (convert): Remove the page's title, date and time lines only when present + +**Description:** + +`Convert-OneNotePage` always discards the first 6 lines of pandoc's output, assuming they are the page's title, date and time. Some pages don't have these, for example emails sent to OneNote from Outlook. On those pages, real content was lost. In my notebooks this happened to two pages: + +- one lost a `Tags` table and its `Subject:` line +- the other lost its first line, a `curl` command + +This PR adds `Remove-PageHeader`, which only removes the header when it's really there: + +- **Title, date, time:** removed when there is a date line (any text with a 4-digit year, so it works in any language, e.g. `Donnerstag, 17. Oktober 2024` or `Monday, September 8, 2014`) followed by a time line (`09:50`, `9:19 AM`). The title may be empty; an untitled page yields a line with only a non-breaking space. +- **No date/time:** only a first line that repeats the page's name is removed. The comparison uses letters and digits only, because pandoc escapes and autolinks the title. +- **Anything else:** kept as is. + +I checked this against pandoc's output for all 197 pages of one of my notebooks. The header was recognized on all pages that have one, and the two pages without one kept their content. + +**Also included**, since it's about the same header: the `Created:` / `Modified:` lines used the format `zz00`, which only prints the hours of the UTC offset. That is wrong for time zones like +05:30. The output format doesn't change (`+0200`). + +New unit tests cover `Remove-PageHeader`: German and English headers, a title containing a year, an untitled page, a page without a header, a title-only page, a header-only page, and empty input. + +--- + +## 3. Branch `fix/file-names-keep-duplicate-postfix-and-avoid-broken-links` + +**Title:** Fix (file names): Keep duplicate page postfixes and avoid broken attachment and image links + +**Description:** + +Three related problems with generated file names and links. + +**Duplicate page names overwrite each other.** The `-1`, `-2` postfix for identically named pages in a section is appended *before* the name is truncated to `mdFileNameAndFolderNameMaxLength`. For long page names, truncation cuts the postfix off again, and the pages are written to the same file. Now: + +- duplicates are detected on the truncated name, which also catches long names that only differ after the cut-off +- the postfix survives truncation, via a new `Truncate-PathFileName -KeepSuffix` parameter + +**Nested attachment links.** Inserted-file references are replaced with a plain text search. A page with the attachments `11-36-00002.pdf` and `Datenblatt_11-36-00002.pdf` ended up with a broken, nested link: `[Datenblatt_[11-36-00002.pdf](media/11-36-00002.pdf)](media/Datenblatt_[11-36-00002.pdf](media/11-36-00002.pdf))`. The search now only matches a name that isn't part of a longer file name. + +**Broken image links.** Image file names are prefixed with the page path, which can contain characters like `#` and `'`. For example, a page named `c't expert community: Newsletter #10` produces `c't-expert-community--Newsletter-#10-image1.jpg`. In a Markdown link, `#` starts a URL fragment, so the image doesn't load. Image names now go through `Remove-InvalidFileNameCharsInsertedFiles`, like inserted attachments already do. + +New unit tests: `-KeepSuffix` in both cases, long duplicate page names (including two names that only differ after the cut-off), and an attachment name contained in another attachment's name. + +--- + +## 4. Branch `fix/file-names-decode-onenote-escape-codes-in-section-names` + +**Title:** Fix (file names): Decode OneNote's escape codes in section names + +**Description:** + +OneNote reports characters that aren't allowed in section file names as `^` codes in the section's name, for example: + +- `AI ^M ML` for a section named `AI / ML` +- `DECT^J Bluetooth` for `DECT & Bluetooth` + +These codes ended up literally in folder names (`AI-^M--ML`, `DECT^J-Bluetooth`). + +This PR adds `ConvertFrom-OneNoteEscapedName`, which decodes `^M` to `/` and `^J` to `&`, the two codes I've seen. It runs before `Remove-InvalidFileNameChars` for notebook, section group and section names, so `/` then becomes `-` as usual. Other codes, if they exist, are left as they are. + +If the `overwrite` PR (#5 in this series) is merged too, `Remove-PreviousNotebookOutput` should use the same decoding for the notebook's directory name, so both agree for notebook names that contain these codes. + +New unit tests cover the decoding and the resulting directory name. + +--- + +## 5. Branch `enhancement/configuration-add-option-overwrite-to-remove-previous-output` + +**Title:** Enhancement (configuration): Add option `overwrite` to remove a notebook's previous output + +**Description:** + +When converting into an existing notes folder, files of the same name are overwritten. But notes, folders and media that were renamed or deleted in OneNote since the last run are left behind, and they're easy to mistake for current notes. + +This PR adds the option `overwrite`: + +- **`1` (default):** current behavior. +- **`2`:** before converting a notebook, its previous output is removed (`Remove-PreviousNotebookOutput`). + - The notebook's `docx` folder is kept, so `usedocx = 2` can still reuse it. + - Only the notebook's own folder under `notesdestpath` is touched, and nothing happens on a dry run. + - Read-only files (e.g. inserted attachments, which OneNote marks read-only) are removed too. + - On Windows PowerShell 5, paths longer than 260 characters are handled via the `\\?\` prefix. + +This overlaps in purpose with #190 (`skipIfExists`), which is about re-running conversions too. The two options don't conflict. + +New unit tests run against real files in `TestDrive`: removal while keeping `docx`, including a read-only file; a dry run; and a notebook folder that doesn't exist. I also used it on two notebooks (393 pages, with long paths): a planted stale note was removed, and both `docx` folders were kept. + +--- + +## 6. Branch `enhancement/configuration-add-keepescape-mode-4-to-repair-pandoc-escapes` + +Based on upstream `master`, independent of PRs 1 to 5. Opened upstream as [#217](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/217) on 2026-11-01. + +**Title:** Enhancement (configuration): Add keepescape mode 4 to repair Pandoc escapes + +**Description:** + +OneNote pages often start paragraphs with a literal `- ` or `7.` instead of using real lists. Pandoc escapes such text so it does not turn into a list (`\- item`, `7\. item`). In an export of two notebooks (393 pages) this left the notes with plain paragraphs instead of lists, a blank line between every item, a literal `\` at the end of many lines (Pandoc's hard line break) and many needless escapes (`--\>`, `snake\_case`). + +The existing `keepescape` modes cannot fix this: `1` deletes every backslash (also in Windows paths such as `C:\Program Files`), `2` and `3` keep the list escapes. + +This PR adds `keepescape = 4`, implemented as the function `Repair-MarkdownEscapes`, run on Pandoc's output before the existing replacements: + +- Rebuilds lists from `\- item` and `7\. item` (also inside block quotes and when a non-breaking space follows the marker), removes the blank lines between list items, and drops lone `-` lines (empty bullets, or an accidental heading underline). +- Turns a `\` at the end of a line into two trailing spaces, or drops it at the end of a paragraph. +- Removes escapes that cannot change the rendering (`\>` in `--\>`, `\_` in `snake\_case`, `\.` and `\-` in the middle of a line). +- Keeps escapes that matter: at the start of a line, `\[`, `\]`, `\*` pairs, `\`, Windows and UNC paths. +- Never touches code fences and inline code. + +The default (`1`) and modes 1 to 3 are unchanged. A literal `- ` at the start of a line is treated as a list in mode 4; use `3` if that is not wanted. + +**Tested** + +- 22 new unit tests: each rule, cases that must stay untouched (Windows and UNC paths, `\*`, code), block quotes, non-breaking spaces, running it twice gives the same result, and that mode 4 adds no replacement that touches backslashes. +- Real run against OneNote with two notebooks (393 pages, same `.docx` files, mode 3 against mode 4): escaped bullets 601 to 0, blank lines between list items 67 to 0, line-end backslashes 1804 to 0, lone `-` lines 29 to 0, escaped numbers 133 to 0. The letters and digits of every page are identical in both modes, so no text is lost. +- Unit tests: 96 passed in Windows PowerShell 5.1; the one failure ("Throw if -Length is greater than 255") expects the English .NET error message and also fails on `master` on a German Windows. +- Not tested here: PowerShell 6.x and 7.0 (see the CI PR below). + +--- + +## 7. Branch `fix/ci-remove-retired-windows-2019-and-unusable-container-jobs` + +Based on upstream `master`. Opened upstream as [#216](https://github.com/theohbrothers/ConvertOneNote2MarkDown/pull/216) on 2026-11-01. Best merged before any other open PR, so their CI can finish. + +**Title:** CI: Remove the retired windows-2019 job and the unusable PowerShell 6.x/7.0 container jobs + +**Description:** + +Running the workflow on this repository's pull requests (in a fork) shows three problems that have nothing to do with the code: + +- `test-powershell-5-1-windows-2019` stays queued forever because GitHub retired the `windows-2019` runner image (June 2025). The whole workflow run never completes. +- `test-powershell-6-0`, `6-1`, `6-2` and `7-0` run in Ubuntu 16.04/18.04 containers. `actions/checkout` fails there before any test runs: `/__e/node24/bin/node: ... GLIBC_2.28 not found`, because GitHub now forces a newer Node runtime on JavaScript actions. +- `actions/checkout@v3` is outdated. + +This PR removes those five jobs (and their entries in `needs:`) and updates `actions/checkout` to `v4`. Windows PowerShell 5.1 on `windows-2022` and PowerShell 7.1 to 7.4 remain and pass. The tests are not changed. + +The trade-off: PowerShell 6.x and 7.0 are no longer exercised by CI. They could be added back with a newer container image or a different way to install them, if wanted.