diff --git a/lib/commits-webkit.js b/lib/commits-webkit.js new file mode 100644 index 0000000..f71f238 --- /dev/null +++ b/lib/commits-webkit.js @@ -0,0 +1,38 @@ +"use strict"; +const fetch = require('node-fetch'); + +const BASE_URL = 'https://commits.webkit.org'; + +// Only allow references to commits on WebKit's main branch, either as a +// canonical identifier (e.g. "324168@main") or as a legacy Subversion revision +// (e.g. "r266816"). References are parsed out of Bugzilla comments, so they +// are strictly validated before being used to build a URL. +const REF_PATTERN = /^(?:\d+@main|r\d+)$/; + +/** + * Retrieves information about a commit in the WebKit repository. + * + * @param {string} ref A commit reference, e.g. "324168@main" or "r266816". + * @return {Promise} The commit (with `hash`, `branch`, + * `identifier`, `revision` and `message` fields), or null if no such + * commit exists. + */ +async function get(ref) { + if (typeof ref !== 'string' || !REF_PATTERN.test(ref)) { + throw new Error(`Invalid WebKit commit reference: ${ref}`); + } + const response = await exports.fetch(`${BASE_URL}/${ref}/json`, { + headers: { "User-Agent": "wpt-pr-bot" }, + }); + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`Unexpected status ${response.status} from ${BASE_URL} for ${ref}`); + } + return response.json(); +} + +exports.get = get; +// Exposed so that tests can stub out network access. +exports.fetch = fetch; diff --git a/lib/metadata/webkit.js b/lib/metadata/webkit.js index f1f8235..679ae99 100644 --- a/lib/metadata/webkit.js +++ b/lib/metadata/webkit.js @@ -1,5 +1,38 @@ "use strict"; -var bugsWebkit = require('../bugs-webkit'); +var bugsWebkit = require('../bugs-webkit'), + commitsWebkit = require('../commits-webkit'), + logger = require('../logger'); + +// Accounts that resolve a bug as FIXED when a reviewed patch lands in WebKit. +const WEBKIT_LANDING_BOTS = ["ews-feeder", "commit-queue"]; + +// Matches the comment left by WebKit's landing bots, e.g. +// "Committed 324168@main (004e01c34435): " +// "Committed r266816: " +const LANDING_COMMENT_PATTERN = /Committed (\d+@main|r\d+)(?: \(([0-9a-f]{7,40})\))?/g; + +// bugs.webkit.org historically reported user identities (e.g. the `who` field +// of bug history) as full email addresses (e.g. "ews-feeder@webkit.org"), but +// now applies Bugzilla's email filtering for logged-out users, which reports +// only the local part of the address (e.g. "ews-feeder"). Accept either form. +// +// IMPORTANT: The local part alone is NOT a unique identity; anyone can register +// an account such as "ews-feeder@example.com", which is reported as +// "ews-feeder". This check is therefore only a cheap pre-filter, and a landing +// must additionally be verified against the WebKit repository (see `landed`). +function isWebKitLandingBot(who) { + if (typeof who !== "string") { + return false; + } + var at = who.indexOf("@"); + if (at > -1) { + if (who.slice(at + 1) !== "webkit.org") { + return false; + } + who = who.slice(0, at); + } + return WEBKIT_LANDING_BOTS.includes(who); +} exports.related = function(title) { return title.indexOf("WebKit export of") > -1; @@ -16,13 +49,81 @@ exports.verified = function(webkitIssue, wptPullRequestNumber) { }); }; +// Returns true if `commit` (as returned by commits-webkit) is a commit on +// WebKit's main branch whose message references the given WebKit bug. +function isLandedCommitForBug(commit, issue, abbreviatedHash) { + if (!commit || commit.branch !== "main" || typeof commit.message !== "string") { + return false; + } + if (abbreviatedHash && (typeof commit.hash !== "string" || !commit.hash.startsWith(abbreviatedHash))) { + return false; + } + // `issue` is always numeric (it is extracted with /\d+/ from the PR title), + // so it is safe to embed in the pattern. The negative lookahead prevents + // e.g. bug 12345 from matching a reference to bug 123456. + var bugURLPattern = new RegExp( + `(?:https?://bugs\\.webkit\\.org/show_bug\\.cgi\\?id=|https?://webkit\\.org/b/)${issue}(?!\\d)`); + return bugURLPattern.test(commit.message); +} + +// Determines whether a change for the given WebKit bug has actually landed on +// WebKit's main branch. +// +// Bugzilla identities cannot be trusted (see `isWebKitLandingBot`), and nor can +// comment text, so this looks for the "Committed ..." comment left by the +// landing bot and then confirms with the WebKit repository itself (via +// commits.webkit.org) that the referenced commit exists on main and that its +// message references this bug. Only WebKit committers can land on main, so +// this cannot be spoofed by an arbitrary Bugzilla user. +// +// Errors talking to commits.webkit.org are logged and treated as "not landed"; +// the pull request poller will try again later. +exports.landed = async function(issue) { + var body = await bugsWebkit.get("/rest/bug/:id/comment", { id: issue }); + var bug = body && body.bugs && body.bugs[issue]; + var comments = (bug && bug.comments) || []; + for (var comment of comments) { + if (!isWebKitLandingBot(comment.creator) || typeof comment.text !== "string") { + continue; + } + for (var match of comment.text.matchAll(LANDING_COMMENT_PATTERN)) { + var ref = match[1], abbreviatedHash = match[2]; + try { + var commit = await commitsWebkit.get(ref); + if (isLandedCommitForBug(commit, issue, abbreviatedHash)) { + return true; + } + } catch (e) { + logger.warn({ err: e, webkitIssue: issue, ref: ref }, + `Unable to verify WebKit commit ${ref} for bug ${issue}`); + } + } + } + return false; +}; + +function isLandingResolution(history_item, change) { + return change.field_name == "resolution" && + change.added.includes("FIXED") && + isWebKitLandingBot(history_item.who); +} + exports.flags = function(issue) { var flags = { reviewed: false, inCommit: false }; - return bugsWebkit.get("/rest/bug/:id/history", { id: issue }).then(function (body) { - body.bugs[0].history.forEach(function(history_item) { + return bugsWebkit.get("/rest/bug/:id/history", { id: issue }).then(async function (body) { + var history = body.bugs[0].history; + // Only verify the landing (which requires additional network + // requests) if the history claims that a landing bot fixed the bug. + var claimsLanding = history.some(function(history_item) { + return history_item.changes.some(function(change) { + return isLandingResolution(history_item, change); + }); + }); + var landed = claimsLanding && await exports.landed(issue); + history.forEach(function(history_item) { history_item.changes.forEach(function(change) { if (change.field_name == "flagtypes.name") { if (change.removed.includes("review+")) { @@ -40,11 +141,9 @@ exports.flags = function(issue) { flags.inCommit = true; } } - if (change.field_name == "resolution") { - if (change.added.includes("FIXED") && ["ews-feeder@webkit.org", "commit-queue@webkit.org"].includes(history_item.who)) { - flags.reviewed = true; - flags.inCommit = true; - } + if (landed && isLandingResolution(history_item, change)) { + flags.reviewed = true; + flags.inCommit = true; } }); }); diff --git a/test/commits-webkit.js b/test/commits-webkit.js new file mode 100644 index 0000000..6947932 --- /dev/null +++ b/test/commits-webkit.js @@ -0,0 +1,74 @@ +'use strict'; + +var assert = require('chai').assert, + sinon = require('sinon'), + commitsWebkit = require('../lib/commits-webkit'); + +suite('commits-webkit', function() { + var sandbox, fetch; + + setup(function() { + sandbox = sinon.createSandbox(); + fetch = sandbox.stub(commitsWebkit, 'fetch'); + }); + + teardown(function() { + sandbox.restore(); + }); + + function respond(status, body) { + fetch.resolves({ + status: status, + ok: status >= 200 && status < 300, + json: async () => body, + }); + } + + test('retrieves a commit by git identifier', async function() { + var commit = { hash: 'abc', branch: 'main', identifier: '324168@main' }; + respond(200, commit); + assert.deepEqual(await commitsWebkit.get('324168@main'), commit); + assert.equal(fetch.firstCall.args[0], 'https://commits.webkit.org/324168@main/json'); + }); + + test('retrieves a commit by Subversion revision', async function() { + var commit = { hash: 'def', branch: 'main', revision: 266816 }; + respond(200, commit); + assert.deepEqual(await commitsWebkit.get('r266816'), commit); + assert.equal(fetch.firstCall.args[0], 'https://commits.webkit.org/r266816/json'); + }); + + test('returns null for a commit that does not exist', async function() { + respond(404, { message: 'No commit with the specified reference could be found' }); + assert.isNull(await commitsWebkit.get('999999999@main')); + }); + + test('throws on other errors', async function() { + respond(503, {}); + try { + await commitsWebkit.get('324168@main'); + assert.fail('expected an error'); + } catch (e) { + assert.include(e.message, '503'); + } + }); + + [ + '324168@safari-7620-branch', + '../324168@main', + '324168@main/../../evil', + 'abcdef', + '', + undefined, + ].forEach(function(ref) { + test(`rejects invalid reference ${JSON.stringify(ref)}`, async function() { + try { + await commitsWebkit.get(ref); + assert.fail('expected an error'); + } catch (e) { + assert.include(e.message, 'Invalid WebKit commit reference'); + } + assert.isFalse(fetch.called); + }); + }); +}); diff --git a/test/get-metadata.js b/test/get-metadata.js index f0b9f80..b36478a 100644 --- a/test/get-metadata.js +++ b/test/get-metadata.js @@ -1,6 +1,8 @@ 'use strict'; var assert = require('chai').assert, sinon = require('sinon'), + bugsWebkit = require('../lib/bugs-webkit'), + commitsWebkit = require('../lib/commits-webkit'), getMetadata = require('../lib/metadata'); suite('getMetadata', function() { @@ -364,6 +366,24 @@ suite('getMetadata', function() { }); test('retrieval and formatting of metadata for a WebKit related pull request', async function() { + // The bug was resolved FIXED by commit-queue, so the landing is + // verified against the bug's comments and the WebKit repository. + sandbox.stub(bugsWebkit, 'get').callThrough() + .withArgs('/rest/bug/:id/comment', { id: '201401' }) + .resolves({ bugs: { '201401': { comments: [ + { + creator: 'commit-queue@webkit.org', + text: 'Comment on attachment 381810\nPatch\n\nClearing flags on attachment: 381810\n\n' + + 'Committed r251544: ', + }, + ] } } }); + sandbox.stub(commitsWebkit, 'get').withArgs('r251544').resolves({ + hash: '0000000000000000000000000000000000000000', + branch: 'main', + revision: 251544, + message: '[MSE][GStreamer] Fix video is not visible\n' + + 'https://bugs.webkit.org/show_bug.cgi?id=201401\n\nReviewed by Philippe Normand.\n', + }); var expected = { issue: 19538, title: 'WebKit export of https://bugs.webkit.org/show_bug.cgi?id=201401', diff --git a/test/webkit.js b/test/webkit.js new file mode 100644 index 0000000..b142358 --- /dev/null +++ b/test/webkit.js @@ -0,0 +1,246 @@ +'use strict'; + +var assert = require('chai').assert, + sinon = require('sinon'), + bugsWebkit = require('../lib/bugs-webkit'), + commitsWebkit = require('../lib/commits-webkit'), + webkit = require('../lib/metadata/webkit'); + +const BUG = '12345'; +const BUG_URL = `https://bugs.webkit.org/show_bug.cgi?id=${BUG}`; +const HASH = '004e01c34435249ef9bc54e0856cadb3ebc13856'; + +function resolvedFixedBy(who) { + return { + who: who, + changes: [ + { field_name: 'status', removed: 'NEW', added: 'RESOLVED' }, + { field_name: 'resolution', removed: '', added: 'FIXED' }, + ], + }; +} + +function landingComment(creator, text) { + return { + creator: creator, + text: text || 'Committed 324168@main (004e01c34435): \n\n' + + 'Reviewed commits have been landed. Closing PR #77500 and removing active labels.', + }; +} + +function mainCommit(overrides) { + return Object.assign({ + hash: HASH, + branch: 'main', + identifier: '324168@main', + revision: null, + message: `Do the thing\n${BUG_URL}\nrdar://190417737\n\nReviewed by Someone.\n`, + }, overrides); +} + +suite('webkit', function() { + var sandbox, bugsGet, commitsGet; + + setup(function() { + sandbox = sinon.createSandbox(); + bugsGet = sandbox.stub(bugsWebkit, 'get'); + bugsGet.rejects(new Error('unexpected bugs.webkit.org request')); + commitsGet = sandbox.stub(commitsWebkit, 'get'); + commitsGet.rejects(new Error('unexpected commits.webkit.org request')); + }); + + teardown(function() { + sandbox.restore(); + }); + + function stubHistory(history) { + bugsGet.withArgs('/rest/bug/:id/history', { id: BUG }) + .resolves({ bugs: [ { history: history } ] }); + } + + function stubComments(comments) { + bugsGet.withArgs('/rest/bug/:id/comment', { id: BUG }) + .resolves({ bugs: { [BUG]: { comments: comments } } }); + } + + function stubCommit(ref, commit) { + commitsGet.withArgs(ref).resolves(commit); + } + + suite('flags', function() { + [ + 'ews-feeder', + 'commit-queue', + 'ews-feeder@webkit.org', + 'commit-queue@webkit.org', + ].forEach(function(who) { + test(`treats a verified landing resolved FIXED by ${who} as reviewed`, async function() { + stubHistory([ resolvedFixedBy(who) ]); + stubComments([ landingComment(who) ]); + stubCommit('324168@main', mainCommit()); + var flags = await webkit.flags(BUG); + assert.deepEqual(flags, { reviewed: true, inCommit: true }); + }); + }); + + [ + ['another user', 'annevk'], + ['an email on another domain', 'ews-feeder@example.com'], + ['a lookalike domain', 'ews-feeder@webkit.org.example.com'], + ['a different case', 'EWS-Feeder'], + ['a missing user', undefined], + ].forEach(function([description, who]) { + test(`does not treat a bug resolved FIXED by ${description} as reviewed`, async function() { + stubHistory([ resolvedFixedBy(who) ]); + var flags = await webkit.flags(BUG); + assert.deepEqual(flags, { reviewed: false, inCommit: false }); + // No further verification should be attempted. + assert.isFalse(bugsGet.calledWith('/rest/bug/:id/comment')); + assert.isFalse(commitsGet.called); + }); + }); + + test('does not treat an unverified landing as reviewed', async function() { + // Simulates an attacker who registered "ews-feeder@example.com" + // (which Bugzilla reports as "ews-feeder") and resolved their own + // bug as FIXED, without anything landing in WebKit. + stubHistory([ resolvedFixedBy('ews-feeder') ]); + stubComments([ { creator: 'ews-feeder', text: 'Closing.' } ]); + var flags = await webkit.flags(BUG); + assert.deepEqual(flags, { reviewed: false, inCommit: false }); + }); + + test('treats review+ as reviewed without verifying a landing', async function() { + stubHistory([ { + who: 'reviewer', + changes: [ + { field_name: 'flagtypes.name', removed: 'review?', added: 'review+' }, + ], + } ]); + var flags = await webkit.flags(BUG); + assert.deepEqual(flags, { reviewed: true, inCommit: false }); + assert.isFalse(commitsGet.called); + }); + + test('a verified landing after flags were cleared is reviewed', async function() { + stubHistory([ + { + who: 'reviewer', + changes: [ + { field_name: 'flagtypes.name', removed: '', added: 'review+, commit-queue+' }, + ], + }, + { + who: 'commit-queue', + changes: [ + { field_name: 'flagtypes.name', removed: 'review+, commit-queue+', added: '' }, + ], + }, + resolvedFixedBy('commit-queue'), + ]); + stubComments([ landingComment('commit-queue', + 'Committed r251544: ') ]); + stubCommit('r251544', mainCommit({ revision: 251544 })); + var flags = await webkit.flags(BUG); + assert.deepEqual(flags, { reviewed: true, inCommit: true }); + }); + }); + + suite('landed', function() { + test('accepts a landing comment in the git identifier format', async function() { + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', mainCommit()); + assert.isTrue(await webkit.landed(BUG)); + }); + + test('accepts a landing comment in the Subversion format', async function() { + stubComments([ landingComment('commit-queue@webkit.org', + 'Comment on attachment 1\nPatch\n\nCommitted r266816: ') ]); + stubCommit('r266816', mainCommit({ revision: 266816 })); + assert.isTrue(await webkit.landed(BUG)); + }); + + test('accepts a commit message using the webkit.org/b/ short form', async function() { + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', mainCommit({ message: `Fix\nhttps://webkit.org/b/${BUG}\n` })); + assert.isTrue(await webkit.landed(BUG)); + }); + + test('checks every referenced commit', async function() { + stubComments([ + landingComment('ews-feeder', + 'Committed 1@main (aaaaaaaaaaaa): \nCommitted 324168@main (004e01c34435): '), + ]); + stubCommit('1@main', null); + stubCommit('324168@main', mainCommit()); + assert.isTrue(await webkit.landed(BUG)); + }); + + test('ignores landing comments from other users', async function() { + stubComments([ landingComment('attacker') ]); + stubCommit('324168@main', mainCommit()); + assert.isFalse(await webkit.landed(BUG)); + assert.isFalse(commitsGet.called); + }); + + test('rejects a commit that does not exist', async function() { + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', null); + assert.isFalse(await webkit.landed(BUG)); + }); + + test('rejects a commit that is not on main', async function() { + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', mainCommit({ branch: 'safari-7620-branch' })); + assert.isFalse(await webkit.landed(BUG)); + }); + + test('rejects a commit whose hash does not match the comment', async function() { + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', mainCommit({ hash: 'ffffffffffff249ef9bc54e0856cadb3ebc13856' })); + assert.isFalse(await webkit.landed(BUG)); + }); + + test('rejects a commit for a different bug', async function() { + // Pointing at a real, unrelated WebKit commit must not work. + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', mainCommit({ + message: 'Unrelated\nhttps://bugs.webkit.org/show_bug.cgi?id=99999\n', + })); + assert.isFalse(await webkit.landed(BUG)); + }); + + test('rejects a commit for a bug whose number has this bug as a prefix', async function() { + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', mainCommit({ message: `Unrelated\n${BUG_URL}6\n` })); + assert.isFalse(await webkit.landed(BUG)); + }); + + test('rejects a commit without a message', async function() { + stubComments([ landingComment('ews-feeder') ]); + stubCommit('324168@main', mainCommit({ message: undefined })); + assert.isFalse(await webkit.landed(BUG)); + }); + + test('ignores comments without a landing reference', async function() { + stubComments([ + { creator: 'ews-feeder', text: 'Committed nothing in particular' }, + { creator: 'ews-feeder', text: 'Committed 123@safari-7620-branch (abcdef1): ' }, + { creator: 'ews-feeder' }, + ]); + assert.isFalse(await webkit.landed(BUG)); + assert.isFalse(commitsGet.called); + }); + + test('treats errors from commits.webkit.org as not landed', async function() { + stubComments([ landingComment('ews-feeder') ]); + commitsGet.withArgs('324168@main').rejects(new Error('Unexpected status 503')); + assert.isFalse(await webkit.landed(BUG)); + }); + + test('handles a response without comments', async function() { + bugsGet.withArgs('/rest/bug/:id/comment', { id: BUG }).resolves({ bugs: {} }); + assert.isFalse(await webkit.landed(BUG)); + }); + }); +});