Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 5 additions & 78 deletions .github/workflows/ci-master-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,10 @@ on:
- master

jobs:
test-powershell-5-1-windows-2019:
runs-on: windows-2019
steps:
- uses: actions/checkout@v3
- name: Powershell version
run: |
powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
- name: Test
run: |
powershell -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1'

test-powershell-5-1-windows-2022:
runs-on: windows-2022
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Powershell version
run: |
powershell -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
Expand All @@ -36,64 +25,12 @@ jobs:
##########
# Docker #
##########
test-powershell-6-0:
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/powershell:6.0.2-ubuntu-16.04
steps:
- uses: actions/checkout@v3
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
- name: Test
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1'

test-powershell-6-1:
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/powershell:6.1.3-ubuntu-18.04
steps:
- uses: actions/checkout@v3
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
- name: Test
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1'

test-powershell-6-2:
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/powershell:6.2.4-ubuntu-18.04
steps:
- uses: actions/checkout@v3
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
- name: Test
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1'

test-powershell-7-0:
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/powershell:7.0.3-ubuntu-18.04
steps:
- uses: actions/checkout@v3
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
- name: Test
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command './test/test.ps1'

test-powershell-7-1:
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/powershell:7.1.5-ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
Expand All @@ -106,7 +43,7 @@ jobs:
container:
image: mcr.microsoft.com/powershell:7.2.2-ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
Expand All @@ -119,7 +56,7 @@ jobs:
container:
image: mcr.microsoft.com/powershell:7.3-ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
Expand All @@ -132,7 +69,7 @@ jobs:
container:
image: mcr.microsoft.com/powershell:7.4-ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Powershell version
run: |
pwsh -NoLogo -NonInteractive -NoProfile -Command '$PSVersionTable'
Expand All @@ -142,12 +79,7 @@ jobs:

update-draft-release:
needs:
- test-powershell-5-1-windows-2019
- test-powershell-5-1-windows-2022
- test-powershell-6-0
- test-powershell-6-1
- test-powershell-6-2
- test-powershell-7-0
- test-powershell-7-1
- test-powershell-7-2
- test-powershell-7-3
Expand All @@ -162,12 +94,7 @@ jobs:

publish-draft-release:
needs:
- test-powershell-5-1-windows-2019
- test-powershell-5-1-windows-2022
- test-powershell-6-0
- test-powershell-6-1
- test-powershell-6-2
- test-powershell-7-0
- test-powershell-7-1
- test-powershell-7-2
- test-powershell-7-3
Expand Down
108 changes: 108 additions & 0 deletions ConvertOneNote2MarkDown-v2.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,93 @@ Describe "Encode-Markdown" -Tag 'Unit' {

}

Describe "Repair-MarkdownEscapes" -Tag 'Unit' {

Context 'Behavior' {

It "Should turn escaped list markers into lists (<Name>)" -TestCases @(
@{ Name = 'bullet'; Content = '\- one'; Expected = '- one' }
@{ Name = 'indented bullet'; Content = ' \- one'; Expected = ' - one' }
@{ Name = 'number'; Content = '7\. one'; Expected = '7. one' }
@{ Name = 'number alone before its text'; Content = "7\.`n`nitem text"; Expected = '7. item text' }
) {
param ($Name, $Content, $Expected)

Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Expected
}

It "Should accept a non-breaking space after an escaped list marker" {
$nbsp = [char]0x00A0
Repair-MarkdownEscapes -Content "\-$nbsp one`n7\.$nbsp two" | Should -BeExactly "- one`n7. two"
}

It "Should repair lists inside block quotes (keepspaces removes the '>' later)" {
Repair-MarkdownEscapes -Content "> 1\. one`n>`n> 2\. two`n>`n> \- x" | Should -BeExactly "> 1. one`n> 2. two`n> - x"
}

It "Should drop a lone dash line that only had a line-end backslash" {
Repair-MarkdownEscapes -Content "- one`n-\`n`n`n" | Should -BeExactly "- one`n`n`n"
}

It "Should remove blank lines between list items" {
Repair-MarkdownEscapes -Content "\- a`n`n\- b`n`n\- c" | Should -BeExactly "- a`n- b`n- c"
Repair-MarkdownEscapes -Content "1\. a`n`n2\. b" | Should -BeExactly "1. a`n2. b"
}

It "Should remove several blank lines, also with non-breaking spaces, between list items" {
Repair-MarkdownEscapes -Content "- a`n`n`n$( [char]0x00A0 )`n`n- b" | Should -BeExactly "- a`n- b"
Repair-MarkdownEscapes -Content "1. a`n`n`n`n2. b`n`ntext" | Should -BeExactly "1. a`n2. b`n`ntext"
Repair-MarkdownEscapes -Content "- a`n`n> $( [char]0x00A0 ) `n`n- b" | Should -BeExactly "- a`n- b"
}

It "Should keep blank lines between a list and a paragraph" {
Repair-MarkdownEscapes -Content "- a`n`ntext" | Should -BeExactly "- a`n`ntext"
}

It "Should remove lone dash lines, also with non-breaking spaces" {
Repair-MarkdownEscapes -Content "text`n-`n\-`n-$( [char]0x00A0 )`nmore" | Should -BeExactly "text`nmore"
}

It "Should turn a backslash at the end of a line into two spaces, and drop it at the end of a paragraph" {
Repair-MarkdownEscapes -Content "one\`ntwo\`n`nnext" | Should -BeExactly "one `ntwo`n`nnext"
}

It "Should remove escapes that can't change the rendering" {
Repair-MarkdownEscapes -Content 'a --\> b, snake\_case, 1\. no, end\.' | Should -BeExactly 'a --> b, snake_case, 1. no, end.'
}

It "Should keep escapes that change the rendering (<Name>)" -TestCases @(
@{ Name = 'quote at line start'; Content = '\> not a quote' }
@{ Name = 'underscore at line start'; Content = '\_not emphasis_' }
@{ Name = 'image-like'; Content = 'a \![b](c)' }
@{ Name = 'pair of asterisks'; Content = 'a \*b\* c' }
@{ Name = 'brackets'; Content = '\[x\]' }
) {
param ($Name, $Content)

Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Content
}

It "Should keep Windows and UNC paths" {
$content = 'C:\Program Files\App and \\server\share\dir'
Repair-MarkdownEscapes -Content $content | Should -BeExactly $content
}

It "Should not touch code" {
$content = "``\- keep \_this`` and`n`n``````text`n\- keep`n7\. keep\`n``````"
Repair-MarkdownEscapes -Content $content | Should -BeExactly $content
}

It "Should give the same result when run twice" {
$content = "\- a`n`n\- b\`nc`n`n7\. x --\> y\_z\`n`nend\"
$once = Repair-MarkdownEscapes -Content $content
Repair-MarkdownEscapes -Content $once | Should -BeExactly $once
}

}

}

Describe "New-OneNoteConnection" -Tag 'Unit' {

Context 'Behavior' {
Expand Down Expand Up @@ -1380,6 +1467,27 @@ foo\bar\0
}
}

It "Should leave backslashes to Repair-MarkdownEscapes if config keepescape is 4" {
$params['Config']['keepescape']['value'] = 4

$result = @( New-SectionGroupConversionConfig @params 6>$null )
$result.Count | Should -Be 48

foreach ($pageCfg in $result) {
$fakeMarkdownContent = 'hello\$\^\\\*\_\[\]world'

# Mutate
foreach ($m in $pageCfg['mutations']) {
foreach ($r in $m['replacements']) {
$fakeMarkdownContent = $fakeMarkdownContent -replace $r['searchRegex'], $r['replacement']
}
}

# None of the mutations may touch backslashes
$fakeMarkdownContent | Should -Match ([regex]::Escape('hello\$\^\\\*\_\[\]world'))
}
}

It "Should honor config newlineCharacter" {
$params['Config']['newlineCharacter']['value'] = 1

Expand Down
107 changes: 106 additions & 1 deletion ConvertOneNote2MarkDown-v2.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,11 @@ Whether to clear escape symbols from md files. See: https://pandoc.org/MANUAL.ht
1: Clear all '\' characters - Default
2: Clear all '\' characters except those preceding alphanumeric characters
3: Keep '\' symbol escape
4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter
'@
default = 1
value = 1
validateRange = 1,3
validateRange = 1,4
}
newlineCharacter = @{
description = @'
Expand Down Expand Up @@ -497,6 +498,107 @@ Function Encode-Markdown {
$Name
}

Function Repair-MarkdownEscapes {
[CmdletBinding()]
param (
# The markdown content produced by pandoc (any newline style)
[Parameter(Mandatory,Position = 0,ValueFromPipeline = $true)]
[AllowEmptyString()]
[string]
$Content
)

process {
# Pandoc escapes text that merely looks like markdown. OneNote pages often start paragraphs with a literal '- ' or '7.' (no real list),
# which ends up as '\- item' or '7\. item'. Rebuild those lists, and remove only the escapes that can't change how the text is rendered.
# Fenced code blocks and inline code are never touched.
$saved = New-Object 'System.Collections.Generic.List[string]'
$shield = [System.Text.RegularExpressions.MatchEvaluator]{
param ($m)
$saved.Add($m.Value)
"MDSHIELD$( $saved.Count - 1 )END"
}
$text = $Content -replace '\r\n', "`n"
$text = [regex]::Replace($text, '(?ms)(?:^[ \t]*)?(`{3,})[^\n`]*\n.*?^[ \t]*\1`*[ \t]*$', $shield)
$text = [regex]::Replace($text, '(`+)(?!`)(.+?)(?<!`)\1(?!`)', $shield)

# A backslash at the end of a line is a hard break only in some renderers: use two spaces. At the end of a paragraph it has no effect.
$text = $text -replace '(?m)(?<!\\)\\$(?=\n[^\n])', ' '
$text = $text -replace '(?m)(?<!\\)\\[ \t]*$', ''
$text = $text -replace '(?m) +$(?=\n[ \t]*\n)', ''

# Lists
$text = $text -replace '(?m)^[ \t\u00A0>]*\\?-[ \t\u00A0]*$\n?', '' # A lone '-' or '\-' line is an empty bullet (below a text line, '-' would even make a heading)
$text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*)\\-[ \t\u00A0]+', '$1- '
$text = $text -replace '(?m)^(\d+)\\\.[ \t]*\n(?:[ \t]*\n)+(?=\S)', '$1. ' # A number alone on its line, with the item text in the next paragraph
$text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*\d+)\\\.[ \t\u00A0]+', '$1. '
$listStart = '^[ \t\u00A0>]*(?:[-*+]|\d+[.)])\s'
$lines = $text -split "`n"
$isBlank = '^[ \t\u00A0>]*$' # Lines with only non-breaking spaces or an empty block quote ('> ') become blank later on (keepspaces)
$text = @(
for ($i = 0; $i -lt $lines.Count; $i++) {
# Drop the blank lines between two list items
if ($lines[$i] -match $isBlank -and $i -gt 0) {
$prev = $i - 1
while ($prev -gt 0 -and $lines[$prev] -match $isBlank) { $prev-- }
$next = $i + 1
while ($next -lt $lines.Count - 1 -and $lines[$next] -match $isBlank) { $next++ }
if ($lines[$prev] -match $listStart -and $next -lt $lines.Count -and $lines[$next] -match $listStart) {
continue
}
}
$lines[$i]
}
) -join "`n"

# Escapes that can't change the rendering
$unescape = [System.Text.RegularExpressions.MatchEvaluator]{
param ($m)
if ($m.Value -eq '\\') { return $m.Value } # An escaped backslash is matched as a pair, so it can't hide the character after it
$c = $m.Value[1]
$start = $m.Index
$lineStart = if ($start -gt 0) { $text.LastIndexOf([char]10, $start - 1) + 1 } else { 0 }
$prefix = $text.Substring($lineStart, $start - $lineStart)
$before = if ($start -gt 0) { [string]$text[$start - 1] } else { '' }
$afterIndex = $start + 2
$after = if ($afterIndex -lt $text.Length) { [string]$text[$afterIndex] } else { '' }
$atLineStart = $prefix.Trim(" `t>").Length -eq 0
if (',."''%@/?!=+-#>'.Contains($c)) {
$keep = ('=+-#>'.Contains($c) -and $atLineStart) -or ($c -eq '.' -and $prefix -match '^[ \t>]*\d+$') -or ($c -eq '!' -and $after -eq '[')
if (!$keep -and $c -eq '#') {
$lineEnd = $text.IndexOf([char]10, $start)
$line = if ($lineEnd -lt 0) { $text.Substring($lineStart) } else { $text.Substring($lineStart, $lineEnd - $lineStart) }
$keep = $line -match '^\s*#{1,6}(?:\s|$)' # A closing '\#' in a heading stays
}
}elseif ($c -eq '*' -or $c -eq '_') {
$loneGap = ($before -eq '' -or $before -match '^\s$') -and ($after -eq '' -or $after -match '^\s$')
$intraword = $before -match '^[\p{L}\p{N}]$' -and $after -match '^[\p{L}\p{N}]$'
if ($c -eq '*') {
$pStart = if ($start -ge 2) { $text.LastIndexOf("`n`n", $start - 1, [System.StringComparison]::Ordinal) } else { -1 }
$pEnd = $text.IndexOf("`n`n", $start, [System.StringComparison]::Ordinal)
$from = if ($pStart -ge 0) { $pStart + 2 } else { 0 }
$to = if ($pEnd -ge 0) { $pEnd } else { $text.Length }
$paragraph = $text.Substring($from, $to - $from)
$keep = $atLineStart -or !($loneGap -or ($paragraph.Length - $paragraph.Replace('*', '').Length) -eq 1)
}else {
$keep = $atLineStart -or !($loneGap -or $intraword)
}
}else {
$keep = $true
}
if ($keep) { $m.Value } else { [string]$c }
}
$text = [regex]::Replace($text, '\\\\|\\.', $unescape)

# Restore code (fences can contain inline code and vice versa, so repeat until nothing is left)
$restore = [System.Text.RegularExpressions.MatchEvaluator]{ param ($m) $saved[[int]$m.Groups[1].Value] }
while ($text -match 'MDSHIELD\d+END') {
$text = [regex]::Replace($text, 'MDSHIELD(\d+)END', $restore)
}
$text
}
}

Function Set-ContentNoBom {
[CmdletBinding()]
param (
Expand Down Expand Up @@ -1282,6 +1384,9 @@ Function Convert-OneNotePage {
''
}
) -join "`n"
if ($config['keepescape']['value'] -eq 4) {
$content = Repair-MarkdownEscapes -Content $content
}
}

# Mutate
Expand Down
Loading
Loading