diff --git a/bin/helpers/capabilityHelper.js b/bin/helpers/capabilityHelper.js index ad344639..97266e5e 100644 --- a/bin/helpers/capabilityHelper.js +++ b/bin/helpers/capabilityHelper.js @@ -10,9 +10,44 @@ const logger = require("./logger").winstonLogger, const isAccessibilityOn = (bsConfig) => process.env.BROWSERSTACK_TEST_ACCESSIBILITY === 'true' || String(bsConfig.run_settings.accessibility) === 'true'; -const isAboveAccessibilityCypressMajor = (cypressVersion) => { +const cypressMajor = (cypressVersion) => { const major = String(cypressVersion).match(/^(\d+)/); - return !!major && Number(major[1]) > Constants.MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY; + return major ? Number(major[1]) : undefined; +}; + +const isAboveAccessibilityCypressMajor = (cypressVersion) => { + const major = cypressMajor(cypressVersion); + return major !== undefined && major > Constants.MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY; +}; + +const versionParts = (version) => String(version).split('.').map((n) => Number(n)); + +const isBefore = (version, other) => { + const a = versionParts(version), b = versionParts(other); + for (let i = 0; i < 3; i++) { + if ((a[i] || 0) !== (b[i] || 0)) return (a[i] || 0) < (b[i] || 0); + } + return false; +}; + +// True for an accessibility build whose Cypress has the extension service-worker bug. +// "16" and "16.latest" name no minor, so Rails picks it: they count while 16.2.0 is the only +// Cypress 16 Rails offers. Once Rails offers 16.2.1+, drop that branch. +const needsForceHttp1ForAccessibility = (cypressVersion) => { + const [firstAffected, firstFixed] = Constants.ACCESSIBILITY_FORCE_HTTP1_CYPRESS_RANGE; + const value = String(cypressVersion).trim().toLowerCase(); + const exact = value.match(/^(\d+)\.(\d+)\.(\d+)$/); + if (exact) return !isBefore(value, firstAffected) && isBefore(value, firstFixed); + const majorOnly = value.match(/^(\d+)(\.latest)?$/); + return !!majorOnly && Number(majorOnly[1]) === versionParts(firstAffected)[0]; +}; + +// Adds forceHttp1=true to the run's Cypress config unless the user already set forceHttp1 +// (either way), keeping every other key they passed. +const withForceHttp1 = (config) => { + const current = Utils.isUndefined(config) ? '' : String(config).trim(); + if (/(^|,)\s*forceHttp1\s*=/.test(current)) return config; + return current ? `${current},forceHttp1=true` : 'forceHttp1=true'; }; const caps = (bsConfig, zip) => { @@ -144,9 +179,18 @@ const caps = (bsConfig, zip) => { // Rails defaults to its newest Cypress, which accessibility does not support yet, so an // accessibility build without a version asks for the newest major accessibility supports. if (isAccessibilityOn(bsConfig) && Utils.isUndefined(runSettings.cypress_version)) { - runSettings = Object.assign({}, runSettings, { cypress_version: String(Constants.MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY) }); + runSettings = Object.assign({}, runSettings, { cypress_version: String(Constants.DEFAULT_CYPRESS_MAJOR_FOR_ACCESSIBILITY) }); logger.info(Constants.userMessages.ACCESSIBILITY_CYPRESS_VERSION_PINNED); } + // Accessibility scans on the affected Cypress 16 versions only complete with forceHttp1. + if (isAccessibilityOn(bsConfig) && !Utils.isUndefined(runSettings.cypress_version) + && needsForceHttp1ForAccessibility(runSettings.cypress_version)) { + const config = withForceHttp1(runSettings.config); + if (config !== runSettings.config) { + runSettings = Object.assign({}, runSettings, { config }); + logger.info(Constants.userMessages.ACCESSIBILITY_FORCE_HTTP1_SET); + } + } obj.run_settings = JSON.stringify(runSettings); } diff --git a/bin/helpers/constants.js b/bin/helpers/constants.js index 0d858178..bc4362e1 100644 --- a/bin/helpers/constants.js +++ b/bin/helpers/constants.js @@ -87,7 +87,9 @@ const userMessages = { RETRY_LIMIT_EXCEEDED: `Max retries exceeded trying to connect to the host (retries: ${config.retries})`, CHECK_DASHBOARD_AT: "Please check the build status at: ", ACCESSIBILITY_CYPRESS_VERSION_PINNED: - "Accessibility Testing supports Cypress up to version 15, so this build runs on the latest Cypress 15.", + "Accessibility builds without a cypress_version run on the latest Cypress 15.", + ACCESSIBILITY_FORCE_HTTP1_SET: + "Accessibility on Cypress 16.1.0-16.2.0 needs Cypress's HTTP/1.1 proxy, so this build runs with forceHttp1=true.", CYPRESS_VERSION_CHANGED: "Your build will run using Cypress instead of Cypress . Read more about supported versions here: http://browserstack.com/docs/automate/cypress/supported-versions", LOCAL_START_FAILED: "Local Testing setup failed.", @@ -201,7 +203,7 @@ const validationMessages = { NODE_VERSION_PARSING_ERROR: "We weren't able to successfully parse the specified nodeVersion. We will be using the default nodeVersion to run your tests.", ACCESSIBILITY_CYPRESS_VERSION_UNSUPPORTED: - "Accessibility Testing supports Cypress up to version 15. Set cypress_version to 15 in browserstack.json, or turn off accessibility to run Cypress 16 or later.", + "Accessibility Testing supports Cypress up to version 16. Set cypress_version to 16 or lower in browserstack.json, or turn off accessibility to run a later Cypress.", AUTO_IMPORT_CONFLICT_ERROR: "Cannot use both 'auto_import_dev_dependencies' and manual npm dependency configuration. Please either set 'auto_import_dev_dependencies' to false or remove manual 'npm_dependencies', 'win_npm_dependencies', and 'mac_npm_dependencies' configurations.", AUTO_IMPORT_INVALID_TYPE: @@ -447,8 +449,16 @@ const REDACTED_AUTH = `auth: { "username": ${REDACTED}, "access_key": ${REDACTED const SPEC_TIMEOUT_LIMIT = 120; // IN MINS -// Accessibility does not support Cypress 16 yet (APS-22894). Raising this needs no Rails change. -const MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY = 15; +// Newest Cypress major accessibility runs on (APS-22894). Raising this needs no Rails change. +const MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY = 16; +// Major an accessibility build asks for when it names no cypress_version. Stays on 15 while +// Rails' own default does, so an unpinned build keeps today's Cypress. +const DEFAULT_CYPRESS_MAJOR_FOR_ACCESSIBILITY = 15; +// Cypress versions where the a11y extension's service worker never starts, so every scan times +// out unless forceHttp1 is on: [first affected, first fixed). Regression from cypress-io/cypress#34762 +// (16.1.0), reported as cypress-io/cypress#35127, fixed by #34958 in 16.2.1. +// SDK-8048 preprod A/B on 16.2.0: scan step 25.6s without forceHttp1, 4.2s with it. +const ACCESSIBILITY_FORCE_HTTP1_CYPRESS_RANGE = ['16.1.0', '16.2.1']; const CYPRESS_CUSTOM_ERRORS_TO_PRINT_KEY = "custom_errors_to_print"; @@ -516,6 +526,8 @@ module.exports = Object.freeze({ BUILD_FAILED_EXIT_CODE, SPEC_TIMEOUT_LIMIT, MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY, + DEFAULT_CYPRESS_MAJOR_FOR_ACCESSIBILITY, + ACCESSIBILITY_FORCE_HTTP1_CYPRESS_RANGE, CYPRESS_CUSTOM_ERRORS_TO_PRINT_KEY, CYPRESS_V9_AND_OLDER_TYPE, CYPRESS_V10_AND_ABOVE_TYPE, diff --git a/test/unit/bin/helpers/capabilityHelper.js b/test/unit/bin/helpers/capabilityHelper.js index 15ec65be..5eb3cb22 100644 --- a/test/unit/bin/helpers/capabilityHelper.js +++ b/test/unit/bin/helpers/capabilityHelper.js @@ -140,6 +140,45 @@ describe("capabilityHelper.js", () => { chai.assert.isUndefined(sent.cypress_version); }); }); + + ["16", "16.latest", "16.1.0", "16.1.1", "16.2.0"].forEach((version) => { + it(`adds forceHttp1=true for accessibility on cypress_version ${version}`, () => { + return sentRunSettings({ accessibility: true, cypress_version: version }).then(({ sent, bsConfig }) => { + chai.assert.equal(sent.config, "forceHttp1=true"); + chai.assert.isUndefined(bsConfig.run_settings.config); + }); + }); + }); + + it("keeps the user's other config keys when adding forceHttp1", () => { + return sentRunSettings({ accessibility: true, cypress_version: "16.2.0", config: "video=false,baseUrl=https://x.test" }).then(({ sent }) => { + chai.assert.equal(sent.config, "video=false,baseUrl=https://x.test,forceHttp1=true"); + }); + }); + + ["forceHttp1=false", "video=true, forceHttp1 = false"].forEach((config) => { + it(`does not override a forceHttp1 the user set (${config})`, () => { + return sentRunSettings({ accessibility: true, cypress_version: "16", config }).then(({ sent }) => { + chai.assert.equal(sent.config, config); + }); + }); + }); + + [ + ["accessibility on, Cypress 15", { accessibility: true, cypress_version: "15.21.1" }], + ["accessibility on, no version (pinned to 15)", { accessibility: true }], + ["accessibility off, Cypress 16", { accessibility: false, cypress_version: "16.2.0" }], + ["accessibility on, cypress_version latest", { accessibility: true, cypress_version: "latest" }], + ["accessibility on, Cypress 16.2.1 (fixed)", { accessibility: true, cypress_version: "16.2.1" }], + ["accessibility on, Cypress 16.3.0 (fixed)", { accessibility: true, cypress_version: "16.3.0" }], + ["accessibility on, Cypress 16.0.0 (before the regression)", { accessibility: true, cypress_version: "16.0.0" }], + ].forEach(([name, runSettings]) => { + it(`does not add forceHttp1 for ${name}`, () => { + return sentRunSettings(runSettings).then(({ sent }) => { + chai.assert.isUndefined(sent.config); + }); + }); + }); }); it("handle empty test_suite", () => { @@ -808,7 +847,7 @@ describe("capabilityHelper.js", () => { else process.env.BROWSERSTACK_TEST_ACCESSIBILITY = savedA11yEnv; }); - ["16", "16.2.0", "16.latest", 17].forEach((version) => { + ["17", "17.0.0", "17.latest", 18].forEach((version) => { it(`rejects cypress_version ${version} with accessibility on`, () => { return validateError(a11yConfig({ accessibility: true, cypress_version: version })).then((error) => { chai.assert.equal(error, Constants.validationMessages.ACCESSIBILITY_CYPRESS_VERSION_UNSUPPORTED); @@ -816,18 +855,20 @@ describe("capabilityHelper.js", () => { }); }); - it("rejects cypress_version 16 when accessibility is on through BROWSERSTACK_TEST_ACCESSIBILITY", () => { + it("rejects cypress_version 17 when accessibility is on through BROWSERSTACK_TEST_ACCESSIBILITY", () => { process.env.BROWSERSTACK_TEST_ACCESSIBILITY = 'true'; - return validateError(a11yConfig({ cypress_version: "16" })).then((error) => { + return validateError(a11yConfig({ cypress_version: "17" })).then((error) => { chai.assert.equal(error, Constants.validationMessages.ACCESSIBILITY_CYPRESS_VERSION_UNSUPPORTED); }); }); [ ["cypress_version 15 with accessibility on", { accessibility: true, cypress_version: "15" }], + ["cypress_version 16 with accessibility on", { accessibility: true, cypress_version: "16" }], + ["cypress_version 16.2.0 with accessibility on", { accessibility: true, cypress_version: "16.2.0" }], ["no cypress_version with accessibility on", { accessibility: "true" }], - ["cypress_version 16 with accessibility off", { accessibility: false, cypress_version: "16" }], - ["cypress_version 16 without accessibility", { cypress_version: "16" }], + ["cypress_version 17 with accessibility off", { accessibility: false, cypress_version: "17" }], + ["cypress_version 17 without accessibility", { cypress_version: "17" }], ].forEach(([name, runSettings]) => { it(`does not reject ${name} for accessibility`, () => { return validateError(a11yConfig(runSettings)).then((error) => {