Skip to content

Commit 44ea8dd

Browse files
ramilamparoramilamparo
committed
Fix incorrect Playwright upload mapping to QA Sphere runs
Three bugs caused Playwright test results to map to wrong QA Sphere test cases, producing misleading run summaries: 1. Marker matching used substring includes, so QS1-104 incorrectly matched QS1-10427. Replace with word-boundary regex. 2. Playwright parser only used the first "test case" annotation per test, dropping additional ones. Now fans out one TestCaseResult per annotation. 3. Upload message said "Uploaded N test cases" conflating result count with target test case count. Now reports both: "Uploaded X results to Y test cases". Additionally, deduplicate file uploads when multiple results reference the same attachment. Previously each result uploaded its own copy. Now uploads each unique file once and reuses the URL. Progress message shows duplicates skipped count. Closes Hypersequent/tms-issues#2394
1 parent acf0d8c commit 44ea8dd

16 files changed

Lines changed: 411 additions & 57 deletions

‎src/tests/fixtures/allure/matching-tcases/005-result.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
"attachments": [
1111
{
1212
"name": "attachment",
13-
"source": "shared-attachment.txt",
13+
"source": "006-container.json",
1414
"type": "text/plain"
1515
}
1616
],

‎src/tests/fixtures/allure/missing-attachments/004-result.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
"attachments": [
1111
{
1212
"name": "attachment",
13-
"source": "existing-attachment.txt",
13+
"source": "001-result.json",
1414
"type": "text/plain"
1515
}
1616
]

‎src/tests/fixtures/junit-xml/matching-tcases.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@
173173
<testcase name="Menu page content TEST-007" classname="ui.contents.spec.ts" time="3.325">
174174
<system-out>
175175
<![CDATA[
176-
[[ATTACHMENT|./matching-tcases.xml]]
176+
[[ATTACHMENT|./empty-tsuite.xml]]
177177
]]>
178178
</system-out>
179179
</testcase>

‎src/tests/fixtures/junit-xml/missing-attachments.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@
173173
<testcase name="Menu page content TEST-007" classname="ui.contents.spec.ts" time="3.325">
174174
<system-out>
175175
<![CDATA[
176-
[[ATTACHMENT|./matching-tcases.xml]]
176+
[[ATTACHMENT|./empty-tsuite.xml]]
177177
]]>
178178
</system-out>
179179
</testcase>

‎src/tests/fixtures/playwright-json/matching-tcases.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@
149149
{
150150
"name": "attachment",
151151
"contentType": "application/json",
152-
"path": "./matching-tcases.json"
152+
"path": "./empty-tsuite.json"
153153
}
154154
]
155155
}

‎src/tests/fixtures/playwright-json/missing-attachments.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@
149149
{
150150
"name": "attachment",
151151
"contentType": "application/json",
152-
"path": "./matching-tcases.json"
152+
"path": "./empty-tsuite.json"
153153
}
154154
]
155155
}

‎src/tests/fixtures/playwright-json/multi-annotation-with-attachments.json‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{
2+
"suites": [
3+
{
4+
"title": "multi-annotation.spec.ts",
5+
"specs": [
6+
{
7+
"title": "Login flow covers multiple cases",
8+
"tags": [],
9+
"tests": [
10+
{
11+
"annotations": [
12+
{
13+
"type": "test case",
14+
"description": "https://qas.eu1.qasphere.com/project/TEST/tcase/10427"
15+
},
16+
{
17+
"type": "test case",
18+
"description": "https://qas.eu1.qasphere.com/project/TEST/tcase/10427"
19+
},
20+
{
21+
"type": "test case",
22+
"description": "https://qas.eu1.qasphere.com/project/TEST/tcase/10428"
23+
}
24+
],
25+
"expectedStatus": "passed",
26+
"projectName": "chromium",
27+
"results": [
28+
{
29+
"status": "passed",
30+
"errors": [],
31+
"stdout": [],
32+
"stderr": [],
33+
"retry": 0,
34+
"duration": 2500,
35+
"attachments": [
36+
{
37+
"name": "attachment",
38+
"contentType": "application/json",
39+
"path": "./multi-annotation-with-attachments.json"
40+
}
41+
]
42+
}
43+
],
44+
"status": "expected"
45+
}
46+
]
47+
},
48+
{
49+
"title": "Navigation bar items TEST-006",
50+
"tags": [],
51+
"tests": [
52+
{
53+
"annotations": [],
54+
"expectedStatus": "passed",
55+
"projectName": "chromium",
56+
"results": [
57+
{
58+
"status": "passed",
59+
"errors": [],
60+
"stdout": [],
61+
"stderr": [],
62+
"retry": 0,
63+
"duration": 1000,
64+
"attachments": [
65+
{
66+
"name": "attachment",
67+
"contentType": "application/json",
68+
"path": "./multi-annotation-with-attachments.json"
69+
}
70+
]
71+
}
72+
],
73+
"status": "expected"
74+
}
75+
]
76+
}
77+
],
78+
"suites": []
79+
}
80+
]
81+
}

‎src/tests/fixtures/testcases.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,4 +118,38 @@ export const runTestCases = [
118118
pos: 1,
119119
},
120120
},
121+
{
122+
id: '1CBd7Qsn1_abc10427xyz',
123+
version: 1,
124+
folderId: 13,
125+
pos: 4,
126+
seq: 10427,
127+
title: 'Login flow case 1',
128+
priority: 'medium',
129+
status: 'open',
130+
folder: {
131+
id: 13,
132+
parentId: 7,
133+
title: 'multi-annotation.spec.ts',
134+
projectId: '1CBd7PKmG_khB9xTm8gL2oe',
135+
pos: 1,
136+
},
137+
},
138+
{
139+
id: '1CBd7Qsn2_abc10428xyz',
140+
version: 1,
141+
folderId: 13,
142+
pos: 5,
143+
seq: 10428,
144+
title: 'Login flow case 2',
145+
priority: 'medium',
146+
status: 'open',
147+
folder: {
148+
id: 13,
149+
parentId: 7,
150+
title: 'multi-annotation.spec.ts',
151+
projectId: '1CBd7PKmG_khB9xTm8gL2oe',
152+
pos: 1,
153+
},
154+
},
121155
]

‎src/tests/marker-parser.spec.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,40 @@ describe('nameMatchesTCase', () => {
216216
test('no match for wrong seq', () => {
217217
expect(junit.nameMatchesTCase('TEST-002 Cart', 'TEST', 3)).toBe(false)
218218
})
219+
220+
describe('does not prefix-match longer sequences', () => {
221+
test('QS1-104 does not match name containing QS1-10427', () => {
222+
expect(playwright.nameMatchesTCase('QS1-10427: some test', 'QS1', 104)).toBe(false)
223+
})
224+
225+
test('QS1-107 does not match name containing QS1-10775', () => {
226+
expect(playwright.nameMatchesTCase('QS1-10775: some test', 'QS1', 107)).toBe(false)
227+
})
228+
229+
test('QS1-10427 still matches itself', () => {
230+
expect(playwright.nameMatchesTCase('QS1-10427: some test', 'QS1', 10427)).toBe(true)
231+
})
232+
233+
test('QS1-104 matches when it appears exactly', () => {
234+
expect(playwright.nameMatchesTCase('QS1-104: some test', 'QS1', 104)).toBe(true)
235+
})
236+
237+
test('QS1-104 matches at end of name', () => {
238+
expect(playwright.nameMatchesTCase('some test QS1-104', 'QS1', 104)).toBe(true)
239+
})
240+
241+
test('QS1-104 matches in middle of name with boundaries', () => {
242+
expect(playwright.nameMatchesTCase('some QS1-104 test', 'QS1', 104)).toBe(true)
243+
})
244+
245+
test('marker surrounded by parens still matches', () => {
246+
expect(playwright.nameMatchesTCase('some test (QS1-10427)', 'QS1', 10427)).toBe(true)
247+
})
248+
249+
test('marker surrounded by parens does not prefix-match', () => {
250+
expect(playwright.nameMatchesTCase('some test (QS1-10427)', 'QS1', 104)).toBe(false)
251+
})
252+
})
219253
})
220254

221255
describe('separator-bounded hyphenless (JUnit only)', () => {

‎src/tests/playwright-json-parsing.spec.ts‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,139 @@ describe('Playwright JSON parsing', () => {
379379
expect(testcases[2].name).toBe('PRJ-789: PRJ-456: Test with marker in name and annotation')
380380
})
381381

382+
test('Should fan out multiple results for test with multiple annotations', async () => {
383+
const jsonPath = `${playwrightJsonBasePath}/multi-annotation-with-attachments.json`
384+
const jsonContent = await readFile(jsonPath, 'utf8')
385+
386+
const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', {
387+
skipStdout: 'never',
388+
skipStderr: 'never',
389+
})
390+
391+
// Fixture has 1 test with 3 annotations (2 for 10427, 1 for 10428) + 1 test with no annotations = 4 results
392+
expect(testcases).toHaveLength(4)
393+
expect(testcases[0].name).toBe('TEST-10427: Login flow covers multiple cases')
394+
expect(testcases[1].name).toBe('TEST-10427: Login flow covers multiple cases')
395+
expect(testcases[2].name).toBe('TEST-10428: Login flow covers multiple cases')
396+
expect(testcases[3].name).toBe('Navigation bar items TEST-006')
397+
398+
// The three fan-out entries share the same status, duration, folder
399+
for (const tc of testcases.slice(0, 3)) {
400+
expect(tc.status).toBe('passed')
401+
expect(tc.timeTaken).toBe(2500)
402+
expect(tc.folder).toBe('multi-annotation.spec.ts')
403+
}
404+
405+
// All four entries have attachments from the fixture
406+
for (const tc of testcases) {
407+
expect(tc.attachments).toHaveLength(1)
408+
}
409+
})
410+
411+
test('Should still produce one result for single annotation', async () => {
412+
const jsonContent = JSON.stringify({
413+
suites: [
414+
{
415+
title: 'single.spec.ts',
416+
specs: [
417+
{
418+
title: 'Simple test',
419+
tags: [],
420+
tests: [
421+
{
422+
annotations: [
423+
{
424+
type: 'test case',
425+
description: 'https://qas.eu1.qasphere.com/project/PRJ/tcase/100',
426+
},
427+
],
428+
expectedStatus: 'passed',
429+
projectName: 'chromium',
430+
results: [
431+
{
432+
status: 'passed',
433+
errors: [],
434+
stdout: [],
435+
stderr: [],
436+
retry: 0,
437+
duration: 1000,
438+
attachments: [],
439+
},
440+
],
441+
status: 'expected',
442+
},
443+
],
444+
},
445+
],
446+
suites: [],
447+
},
448+
],
449+
})
450+
451+
const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', {
452+
skipStdout: 'never',
453+
skipStderr: 'never',
454+
})
455+
456+
expect(testcases).toHaveLength(1)
457+
expect(testcases[0].name).toBe('PRJ-100: Simple test')
458+
})
459+
460+
test('Should fan out by annotations even when name has a marker', async () => {
461+
const jsonContent = JSON.stringify({
462+
suites: [
463+
{
464+
title: 'precedence.spec.ts',
465+
specs: [
466+
{
467+
title: 'PRJ-999: Test with marker in name',
468+
tags: [],
469+
tests: [
470+
{
471+
annotations: [
472+
{
473+
type: 'test case',
474+
description: 'https://qas.eu1.qasphere.com/project/PRJ/tcase/100',
475+
},
476+
{
477+
type: 'test case',
478+
description: 'https://qas.eu1.qasphere.com/project/PRJ/tcase/200',
479+
},
480+
],
481+
expectedStatus: 'passed',
482+
projectName: 'chromium',
483+
results: [
484+
{
485+
status: 'passed',
486+
errors: [],
487+
stdout: [],
488+
stderr: [],
489+
retry: 0,
490+
duration: 1000,
491+
attachments: [],
492+
},
493+
],
494+
status: 'expected',
495+
},
496+
],
497+
},
498+
],
499+
suites: [],
500+
},
501+
],
502+
})
503+
504+
const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', {
505+
skipStdout: 'never',
506+
skipStderr: 'never',
507+
})
508+
509+
// Annotations take precedence — two results, not one from the name marker
510+
expect(testcases).toHaveLength(2)
511+
expect(testcases[0].name).toBe('PRJ-100: PRJ-999: Test with marker in name')
512+
expect(testcases[1].name).toBe('PRJ-200: PRJ-999: Test with marker in name')
513+
})
514+
382515
test('Should map test status correctly', async () => {
383516
const jsonContent = JSON.stringify({
384517
suites: [

0 commit comments

Comments
 (0)