Skip to content

Commit 5f2ff39

Browse files
webstackdevwebstackdev
committed
Update 01-smoke E2E tests to output console errors when test cases fail
1 parent 040c7ba commit 5f2ff39

5 files changed

Lines changed: 149 additions & 135 deletions

File tree

‎_TODO.md‎

Lines changed: 7 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -98,142 +98,32 @@ if (window.matchMedia) {
9898
}
9999
```
100100

101-
```bash
102-
1) [mobile-chrome] › test/e2e/specs/01-smoke/critical-paths.spec.ts:38:3 › Critical Paths @smoke › @ready mobile navigation works across main pages
103-
104-
TimeoutError: page.waitForFunction: Timeout 15000ms exceeded.
105-
106-
at ../helpers/pageObjectModels/BasePage.ts:263
107-
108-
261 | }
109-
262 |
110-
> 263 | await this._page.waitForFunction(
111-
| ^
112-
264 | () => {
113-
265 | const header = document.getElementById('header')
114-
266 | const menu = document.querySelector('.main-nav-menu')
115-
at BasePage.openMobileMenu (/home/kevin/Repos/WebstackBuilders/CorporateWebsite/astro.webstackbuilders.com/test/e2e/helpers/pageObjectModels/BasePage.ts:263:22)
116-
at /home/kevin/Repos/WebstackBuilders/CorporateWebsite/astro.webstackbuilders.com/test/e2e/specs/01-smoke/critical-paths.spec.ts:53:7
117-
118-
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
119-
.cache/playwright/output/01-smoke-critical-paths-Cr-fda54-ion-works-across-main-pages-mobile-chrome/test-failed-1.png
120-
────────────────────────────────────────────────────────────────────────────────────────────────
121-
122-
Error Context: .cache/playwright/output/01-smoke-critical-paths-Cr-fda54-ion-works-across-main-pages-mobile-chrome/error-context.md
123-
124-
2) [mobile-safari] › test/e2e/specs/01-smoke/critical-paths.spec.ts:114:3 › Critical Paths @smoke › @ready main pages have no errors
125-
126-
Error: expect(received).toHaveLength(expected)
127-
128-
Expected length: 0
129-
Received length: 1
130-
Received array: [[Fetch API cannot load http: /localhost:4321/offline due to access control checks.]]
131-
132-
at ../helpers/pageObjectModels/BasePage.ts:665
133-
134-
663 | const errors = await this._page.pageErrors()
135-
664 | const filteredErrors = errors.filter((error) => !this.isIgnorablePageError(error))
136-
> 665 | expect(filteredErrors).toHaveLength(0)
137-
| ^
138-
666 | return filteredErrors
139-
667 | }
140-
668 |
141-
at BasePage.expectNoErrors (/home/kevin/Repos/WebstackBuilders/CorporateWebsite/astro.webstackbuilders.com/test/e2e/helpers/pageObjectModels/BasePage.ts:665:28)
142-
at /home/kevin/Repos/WebstackBuilders/CorporateWebsite/astro.webstackbuilders.com/test/e2e/specs/01-smoke/critical-paths.spec.ts:118:7
143-
144-
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
145-
.cache/playwright/output/01-smoke-critical-paths-Cr-4a53f-y-main-pages-have-no-errors-mobile-safari/test-failed-1.png
146-
────────────────────────────────────────────────────────────────────────────────────────────────
147-
148-
Error Context: .cache/playwright/output/01-smoke-critical-paths-Cr-4a53f-y-main-pages-have-no-errors-mobile-safari/error-context.md
149-
150-
2 failed
151-
[mobile-chrome] › test/e2e/specs/01-smoke/critical-paths.spec.ts:38:3 › Critical Paths @smoke › @ready mobile navigation works across main pages
152-
[mobile-safari] › test/e2e/specs/01-smoke/critical-paths.spec.ts:114:3 › Critical Paths @smoke › @ready main pages have no errors
153-
101+
## Playwright 01-smoke errors
154102

155-
156-
1) [webkit] › test/e2e/specs/01-smoke/dynamic-pages.spec.ts:171:3 › Dynamic Pages @smoke › @ready dynamic pages have no console errors
157-
158-
Error: expect(received).toHaveLength(expected)
159-
160-
Expected length: 0
161-
Received length: 1
162-
Received array: ["Failed to load resource: the server responded with a status of 403 (Forbidden)"]
163-
164-
192 |
165-
193 | // Fail if there are any unexpected 404s or errors
166-
> 194 | expect(errorChecker.getFilteredErrors()).toHaveLength(0)
167-
| ^
168-
195 | expect(errorChecker.getFiltered404s()).toHaveLength(0)
169-
196 | })
170-
197 | })
171-
at /home/kevin/Repos/WebstackBuilders/CorporateWebsite/astro.webstackbuilders.com/test/e2e/specs/01-smoke/dynamic-pages.spec.ts:194:46
172-
173-
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
174-
.cache/playwright/output/01-smoke-dynamic-pages-Dyn-8a015-ages-have-no-console-errors-webkit/test-failed-1.png
175-
────────────────────────────────────────────────────────────────────────────────────────────────
176-
177-
Error Context: .cache/playwright/output/01-smoke-dynamic-pages-Dyn-8a015-ages-have-no-console-errors-webkit/error-context.md
178-
179-
2) [mobile-safari] › test/e2e/specs/01-smoke/dynamic-pages.spec.ts:171:3 › Dynamic Pages @smoke › @ready dynamic pages have no console errors
180-
181-
Error: expect(received).toHaveLength(expected)
182-
183-
Expected length: 0
184-
Received length: 1
185-
Received array: ["Failed to load resource: the server responded with a status of 403 (Forbidden)"]
186-
187-
192 |
188-
193 | // Fail if there are any unexpected 404s or errors
189-
> 194 | expect(errorChecker.getFilteredErrors()).toHaveLength(0)
190-
| ^
191-
195 | expect(errorChecker.getFiltered404s()).toHaveLength(0)
192-
196 | })
193-
197 | })
194-
at /home/kevin/Repos/WebstackBuilders/CorporateWebsite/astro.webstackbuilders.com/test/e2e/specs/01-smoke/dynamic-pages.spec.ts:194:46
195-
196-
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
197-
.cache/playwright/output/01-smoke-dynamic-pages-Dyn-8a015-ages-have-no-console-errors-mobile-safari/test-failed-1.png
198-
────────────────────────────────────────────────────────────────────────────────────────────────
199-
200-
Error Context: .cache/playwright/output/01-smoke-dynamic-pages-Dyn-8a015-ages-have-no-console-errors-mobile-safari/error-context.md
201-
202-
2 failed
203-
[webkit] › test/e2e/specs/01-smoke/dynamic-pages.spec.ts:171:3 › Dynamic Pages @smoke › @ready dynamic pages have no console errors
204-
[mobile-safari] › test/e2e/specs/01-smoke/dynamic-pages.spec.ts:171:3 › Dynamic Pages @smoke › @ready dynamic pages have no console errors
103+
```bash
205104
```
206105

207-
--color-blue-200
208-
--color-blue-400
209-
--color-blue-50
210-
--color-blue-500
211-
--color-gray-100
212-
--color-gray-500
213-
--color-orange-100
214-
--color-purple-100
215-
--color-purple-600
216106

217107
Good Next Cascade Candidates (similar spirit to the text-* cleanup)
218108

219109
Global/link-default underline styling (biggest repetition)
220110

221111
Evidence: the exact pattern underline decoration-dotted decoration-content underline-offset-4 ... shows up 15 times across just 3 components (Footer/Calendar/BugReporter).
222-
What to cascade: give “normal text links” a default underline style (dotted + decoration-content + offset) via base CSS (or a shared wrapper scope), and then only override the exceptions.
112+
What to cascade: give "normal text links" a default underline style (dotted + decoration-content + offset) via base CSS (or a shared wrapper scope), and then only override the exceptions.
223113
Tradeoff: global a { … } is powerful but can surprise you (nav links, button-like links). A scoped approach (e.g., only inside markdown/content areas or only inside certain layout wrappers) is safer.
224114
Forms: default label + help text styling
225115

226116
Evidence: text-content-active in forms is still repeated ~25 times across 3 form components (Contact/Download/Privacy), often paired with text-sm font-medium.
227-
What to cascade: set label default to color: var(--color-content-active) and font-weight/size inside a form wrapper (e.g. .form root), so individual labels don’t need to specify it.
117+
What to cascade: set label default to color: var(--color-content-active) and font-weight/size inside a form wrapper (e.g. .form root), so individual labels don't need to specify it.
228118
Tradeoff: safest when scoped to the Forms components (not global label { … } site-wide).
229-
Inline style in JS validation errors (small but “smelly”)
119+
Inline style in JS validation errors (small but "smelly")
230120

231121
Evidence: validation.ts sets errorDiv.style.cssText = 'color: var(--color-danger); font-size: 0.85rem; ...'.
232122
What to cascade: assign a class and style it in CSS (then it inherits font family/line-height etc), instead of embedding presentation in JS.
233-
Tradeoff: not a huge volume win, but it’s a maintainability win and keeps styling consistent.
123+
Tradeoff: not a huge volume win, but it's a maintainability win and keeps styling consistent.
234124
Not worth it (yet)
235125

236-
Placeholder color: only 4 placeholder:text-content-active occurrences right now, so the cascade win is small unless you want a stronger “form system” anyway.
126+
Placeholder color: only 4 placeholder:text-content-active occurrences right now, so the cascade win is small unless you want a stronger "form system" anyway.
237127
Font families: very few font-sans/serif/mono usages in components; most appear intentional.
238128
If you want me to implement, pick one:
239129

‎test/e2e/helpers/consoleErrors.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,11 @@ export function setupConsoleErrorChecker(page: Page): ConsoleErrorChecker {
4242
page.on('console', (msg) => {
4343
if (msg.type() === 'error') {
4444
const text = msg.text()
45-
consoleErrors.push(text)
45+
const location = msg.location()
46+
const locationLabel = location.url
47+
? `${location.url}:${location.lineNumber}:${location.columnNumber}`
48+
: 'unknown-location'
49+
consoleErrors.push(`${locationLabel} ${text}`)
4650
}
4751
})
4852

‎test/e2e/helpers/pageObjectModels/BasePage.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -658,11 +658,27 @@ export class BasePage extends BuiltInsPage {
658658
/**
659659
* Returns up to (currently) 200 last uncaught exceptions from this page
660660
*/
661-
async expectNoErrors(): Promise<Array<Error>> {
661+
async getFilteredPageErrors(): Promise<Array<Error>> {
662662
await this.waitForPageComplete()
663663
const errors = await this._page.pageErrors()
664-
const filteredErrors = errors.filter((error) => !this.isIgnorablePageError(error))
665-
expect(filteredErrors).toHaveLength(0)
664+
return errors.filter((error) => !this.isIgnorablePageError(error))
665+
}
666+
667+
async expectNoErrors(): Promise<Array<Error>> {
668+
const filteredErrors = await this.getFilteredPageErrors()
669+
670+
const formattedErrors = filteredErrors
671+
.map((error, index) => {
672+
const header = `[${index + 1}] ${error.name}: ${error.message}`
673+
const stack = error.stack ? `\n${error.stack}` : ''
674+
return `${header}${stack}`
675+
})
676+
.join('\n\n')
677+
678+
expect(
679+
filteredErrors,
680+
formattedErrors.length > 0 ? `Unexpected page errors:\n\n${formattedErrors}` : undefined
681+
).toHaveLength(0)
666682
return filteredErrors
667683
}
668684

‎test/e2e/specs/01-smoke/critical-paths.spec.ts‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,21 +101,64 @@ test.describe('Critical Paths @smoke', () => {
101101
await page.expectCookiesContactForm()
102102
})
103103

104-
test('@ready main pages have no 404 errors', async ({ page: playwrightPage}) => {
104+
test('@ready main pages have no 404 errors', async ({ page: playwrightPage }, testInfo) => {
105105
const page = await BasePage.init(playwrightPage)
106106
await page.enable404Listener()
107+
108+
const failures: string[] = []
107109
for (const { url: path } of page.navigationItems) {
108110
page.reset404Errors()
109111
await page.goto(path)
110-
expect(page.errors404, `Received 404 errors for:\n${page.errors404.join("\n")}`).toHaveLength(0)
112+
113+
if (page.errors404.length > 0) {
114+
failures.push(`${path}\n${page.errors404.map((error) => ` - ${error}`).join('\n')}`)
115+
}
111116
}
117+
118+
if (failures.length > 0) {
119+
await testInfo.attach('404-errors', {
120+
body: failures.join('\n\n'),
121+
contentType: 'text/plain',
122+
})
123+
}
124+
125+
expect(
126+
failures,
127+
failures.length > 0 ? `Received 404 errors:\n\n${failures.join('\n\n')}` : undefined
128+
).toHaveLength(0)
112129
})
113130

114-
test('@ready main pages have no errors', async ({ page: playwrightPage }) => {
131+
test('@ready main pages have no errors', async ({ page: playwrightPage }, testInfo) => {
115132
const page = await BasePage.init(playwrightPage)
133+
134+
const failures: string[] = []
116135
for (const { url: path } of page.navigationItems) {
117136
await page.goto(path)
118-
await page.expectNoErrors()
137+
138+
const errors = await page.getFilteredPageErrors()
139+
if (errors.length > 0) {
140+
const formattedErrors = errors
141+
.map((error, index) => {
142+
const header = `[${index + 1}] ${error.name}: ${error.message}`
143+
const stack = error.stack ? `\n${error.stack}` : ''
144+
return `${header}${stack}`
145+
})
146+
.join('\n\n')
147+
148+
failures.push(`${path}\n${formattedErrors}`)
149+
}
150+
}
151+
152+
if (failures.length > 0) {
153+
await testInfo.attach('page-errors', {
154+
body: failures.join('\n\n'),
155+
contentType: 'text/plain',
156+
})
119157
}
158+
159+
expect(
160+
failures,
161+
failures.length > 0 ? `Unexpected page errors:\n\n${failures.join('\n\n')}` : undefined
162+
).toHaveLength(0)
120163
})
121164
})

‎test/e2e/specs/01-smoke/dynamic-pages.spec.ts‎

Lines changed: 71 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,17 @@ test.describe('Dynamic Pages @smoke', () => {
123123
expect(Array.isArray(manifest.icons)).toBe(true)
124124
})
125125

126-
test('@ready dynamic pages have no 404 errors', async ({ page: playwrightPage }) => {
126+
test('@ready dynamic pages have no 404 errors', async ({ page: playwrightPage }, testInfo) => {
127127
const page = await BasePage.init(playwrightPage)
128+
129+
const errorChecker = setupConsoleErrorChecker(page.page)
130+
const failures: string[] = []
131+
132+
const resetChecker = () => {
133+
errorChecker.consoleErrors.length = 0
134+
errorChecker.failed404s.length = 0
135+
}
136+
128137
// Test article page
129138
await page.goto('/articles')
130139
await page.waitForLoadState('networkidle')
@@ -133,10 +142,14 @@ test.describe('Dynamic Pages @smoke', () => {
133142
const articleUrl = await firstArticleLink.getAttribute('href')
134143

135144
if (articleUrl) {
136-
const articleChecker = setupConsoleErrorChecker(page.page)
145+
resetChecker()
137146
await page.goto(articleUrl)
138147
await page.waitForLoadState('networkidle')
139-
expect(articleChecker.failed404s).toHaveLength(0)
148+
149+
const filtered404s = errorChecker.getFiltered404s()
150+
if (filtered404s.length > 0) {
151+
failures.push(`article: ${articleUrl}\n${filtered404s.map((url) => ` - ${url}`).join('\n')}`)
152+
}
140153
}
141154

142155
// Test service page
@@ -147,10 +160,14 @@ test.describe('Dynamic Pages @smoke', () => {
147160
const serviceUrl = await firstServiceLink.getAttribute('href')
148161

149162
if (serviceUrl) {
150-
const serviceChecker = setupConsoleErrorChecker(page.page)
163+
resetChecker()
151164
await page.goto(serviceUrl)
152165
await page.waitForLoadState('networkidle')
153-
expect(serviceChecker.failed404s).toHaveLength(0)
166+
167+
const filtered404s = errorChecker.getFiltered404s()
168+
if (filtered404s.length > 0) {
169+
failures.push(`service: ${serviceUrl}\n${filtered404s.map((url) => ` - ${url}`).join('\n')}`)
170+
}
154171
}
155172

156173
// Test case study page
@@ -161,14 +178,30 @@ test.describe('Dynamic Pages @smoke', () => {
161178
const caseStudyUrl = await firstCaseStudyLink.getAttribute('href')
162179

163180
if (caseStudyUrl) {
164-
const caseStudyChecker = setupConsoleErrorChecker(page.page)
181+
resetChecker()
165182
await page.goto(caseStudyUrl)
166183
await page.waitForLoadState('networkidle')
167-
expect(caseStudyChecker.failed404s).toHaveLength(0)
184+
185+
const filtered404s = errorChecker.getFiltered404s()
186+
if (filtered404s.length > 0) {
187+
failures.push(`case-study: ${caseStudyUrl}\n${filtered404s.map((url) => ` - ${url}`).join('\n')}`)
188+
}
168189
}
190+
191+
if (failures.length > 0) {
192+
await testInfo.attach('404-errors', {
193+
body: failures.join('\n\n'),
194+
contentType: 'text/plain',
195+
})
196+
}
197+
198+
expect(
199+
failures,
200+
failures.length > 0 ? `Unexpected 404 resources:\n\n${failures.join('\n\n')}` : undefined
201+
).toHaveLength(0)
169202
})
170203

171-
test('@ready dynamic pages have no console errors', async ({ page: playwrightPage }) => {
204+
test('@ready dynamic pages have no console errors', async ({ page: playwrightPage }, testInfo) => {
172205
const page = await BasePage.init(playwrightPage)
173206
// First, get actual article URL
174207
await page.goto('/articles')
@@ -191,7 +224,35 @@ test.describe('Dynamic Pages @smoke', () => {
191224
logConsoleErrors(errorChecker)
192225

193226
// Fail if there are any unexpected 404s or errors
194-
expect(errorChecker.getFilteredErrors()).toHaveLength(0)
195-
expect(errorChecker.getFiltered404s()).toHaveLength(0)
227+
const filteredErrors = errorChecker.getFilteredErrors()
228+
const filtered404s = errorChecker.getFiltered404s()
229+
230+
if (filteredErrors.length > 0) {
231+
await testInfo.attach('console-errors', {
232+
body: filteredErrors.join('\n'),
233+
contentType: 'text/plain',
234+
})
235+
}
236+
237+
if (filtered404s.length > 0) {
238+
await testInfo.attach('404-errors', {
239+
body: filtered404s.join('\n'),
240+
contentType: 'text/plain',
241+
})
242+
}
243+
244+
expect(
245+
filteredErrors,
246+
filteredErrors.length > 0
247+
? `Unexpected console errors:\n\n${filteredErrors.map((error) => ` - ${error}`).join('\n')}`
248+
: undefined
249+
).toHaveLength(0)
250+
251+
expect(
252+
filtered404s,
253+
filtered404s.length > 0
254+
? `Unexpected 404 resources:\n\n${filtered404s.map((url) => ` - ${url}`).join('\n')}`
255+
: undefined
256+
).toHaveLength(0)
196257
})
197258
})

0 commit comments

Comments
 (0)