Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 47 additions & 3 deletions bin/helpers/capabilityHelper.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,44 @@ const logger = require("./logger").winstonLogger,
const isAccessibilityOn = (bsConfig) =>
process.env.BROWSERSTACK_TEST_ACCESSIBILITY === 'true' || String(bsConfig.run_settings.accessibility) === 'true';

const isAboveAccessibilityCypressMajor = (cypressVersion) => {
const cypressMajor = (cypressVersion) => {
const major = String(cypressVersion).match(/^(\d+)/);
return !!major && Number(major[1]) > Constants.MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY;
return major ? Number(major[1]) : undefined;
};

const isAboveAccessibilityCypressMajor = (cypressVersion) => {
const major = cypressMajor(cypressVersion);
return major !== undefined && major > Constants.MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY;
};

const versionParts = (version) => String(version).split('.').map((n) => Number(n));

const isBefore = (version, other) => {
const a = versionParts(version), b = versionParts(other);
for (let i = 0; i < 3; i++) {
if ((a[i] || 0) !== (b[i] || 0)) return (a[i] || 0) < (b[i] || 0);
}
return false;
};

// True for an accessibility build whose Cypress has the extension service-worker bug.
// "16" and "16.latest" name no minor, so Rails picks it: they count while 16.2.0 is the only
// Cypress 16 Rails offers. Once Rails offers 16.2.1+, drop that branch.
const needsForceHttp1ForAccessibility = (cypressVersion) => {
const [firstAffected, firstFixed] = Constants.ACCESSIBILITY_FORCE_HTTP1_CYPRESS_RANGE;
const value = String(cypressVersion).trim().toLowerCase();
const exact = value.match(/^(\d+)\.(\d+)\.(\d+)$/);
if (exact) return !isBefore(value, firstAffected) && isBefore(value, firstFixed);
const majorOnly = value.match(/^(\d+)(\.latest)?$/);
return !!majorOnly && Number(majorOnly[1]) === versionParts(firstAffected)[0];
};

// Adds forceHttp1=true to the run's Cypress config unless the user already set forceHttp1
// (either way), keeping every other key they passed.
const withForceHttp1 = (config) => {
const current = Utils.isUndefined(config) ? '' : String(config).trim();
if (/(^|,)\s*forceHttp1\s*=/.test(current)) return config;
return current ? `${current},forceHttp1=true` : 'forceHttp1=true';
};

const caps = (bsConfig, zip) => {
Expand Down Expand Up @@ -144,9 +179,18 @@ const caps = (bsConfig, zip) => {
// Rails defaults to its newest Cypress, which accessibility does not support yet, so an
// accessibility build without a version asks for the newest major accessibility supports.
if (isAccessibilityOn(bsConfig) && Utils.isUndefined(runSettings.cypress_version)) {
runSettings = Object.assign({}, runSettings, { cypress_version: String(Constants.MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY) });
runSettings = Object.assign({}, runSettings, { cypress_version: String(Constants.DEFAULT_CYPRESS_MAJOR_FOR_ACCESSIBILITY) });
logger.info(Constants.userMessages.ACCESSIBILITY_CYPRESS_VERSION_PINNED);
}
// Accessibility scans on the affected Cypress 16 versions only complete with forceHttp1.
if (isAccessibilityOn(bsConfig) && !Utils.isUndefined(runSettings.cypress_version)
&& needsForceHttp1ForAccessibility(runSettings.cypress_version)) {
const config = withForceHttp1(runSettings.config);
if (config !== runSettings.config) {
runSettings = Object.assign({}, runSettings, { config });
logger.info(Constants.userMessages.ACCESSIBILITY_FORCE_HTTP1_SET);
}
}
obj.run_settings = JSON.stringify(runSettings);
}

Expand Down
20 changes: 16 additions & 4 deletions bin/helpers/constants.js
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ const userMessages = {
RETRY_LIMIT_EXCEEDED: `Max retries exceeded trying to connect to the host (retries: ${config.retries})`,
CHECK_DASHBOARD_AT: "Please check the build status at: ",
ACCESSIBILITY_CYPRESS_VERSION_PINNED:
"Accessibility Testing supports Cypress up to version 15, so this build runs on the latest Cypress 15.",
"Accessibility builds without a cypress_version run on the latest Cypress 15.",
ACCESSIBILITY_FORCE_HTTP1_SET:
"Accessibility on Cypress 16.1.0-16.2.0 needs Cypress's HTTP/1.1 proxy, so this build runs with forceHttp1=true.",
CYPRESS_VERSION_CHANGED:
"Your build will run using Cypress <actualVersion> instead of Cypress <preferredVersion>.<frameworkUpgradeMessage> Read more about supported versions here: http://browserstack.com/docs/automate/cypress/supported-versions",
LOCAL_START_FAILED: "Local Testing setup failed.",
Expand Down Expand Up @@ -201,7 +203,7 @@ const validationMessages = {
NODE_VERSION_PARSING_ERROR:
"We weren't able to successfully parse the specified nodeVersion. We will be using the default nodeVersion to run your tests.",
ACCESSIBILITY_CYPRESS_VERSION_UNSUPPORTED:
"Accessibility Testing supports Cypress up to version 15. Set cypress_version to 15 in browserstack.json, or turn off accessibility to run Cypress 16 or later.",
"Accessibility Testing supports Cypress up to version 16. Set cypress_version to 16 or lower in browserstack.json, or turn off accessibility to run a later Cypress.",
AUTO_IMPORT_CONFLICT_ERROR:
"Cannot use both 'auto_import_dev_dependencies' and manual npm dependency configuration. Please either set 'auto_import_dev_dependencies' to false or remove manual 'npm_dependencies', 'win_npm_dependencies', and 'mac_npm_dependencies' configurations.",
AUTO_IMPORT_INVALID_TYPE:
Expand Down Expand Up @@ -447,8 +449,16 @@ const REDACTED_AUTH = `auth: { "username": ${REDACTED}, "access_key": ${REDACTED

const SPEC_TIMEOUT_LIMIT = 120; // IN MINS

// Accessibility does not support Cypress 16 yet (APS-22894). Raising this needs no Rails change.
const MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY = 15;
// Newest Cypress major accessibility runs on (APS-22894). Raising this needs no Rails change.
const MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY = 16;
// Major an accessibility build asks for when it names no cypress_version. Stays on 15 while
// Rails' own default does, so an unpinned build keeps today's Cypress.
const DEFAULT_CYPRESS_MAJOR_FOR_ACCESSIBILITY = 15;
// Cypress versions where the a11y extension's service worker never starts, so every scan times
// out unless forceHttp1 is on: [first affected, first fixed). Regression from cypress-io/cypress#34762
// (16.1.0), reported as cypress-io/cypress#35127, fixed by #34958 in 16.2.1.
// SDK-8048 preprod A/B on 16.2.0: scan step 25.6s without forceHttp1, 4.2s with it.
const ACCESSIBILITY_FORCE_HTTP1_CYPRESS_RANGE = ['16.1.0', '16.2.1'];

const CYPRESS_CUSTOM_ERRORS_TO_PRINT_KEY = "custom_errors_to_print";

Expand Down Expand Up @@ -516,6 +526,8 @@ module.exports = Object.freeze({
BUILD_FAILED_EXIT_CODE,
SPEC_TIMEOUT_LIMIT,
MAX_CYPRESS_MAJOR_FOR_ACCESSIBILITY,
DEFAULT_CYPRESS_MAJOR_FOR_ACCESSIBILITY,
ACCESSIBILITY_FORCE_HTTP1_CYPRESS_RANGE,
CYPRESS_CUSTOM_ERRORS_TO_PRINT_KEY,
CYPRESS_V9_AND_OLDER_TYPE,
CYPRESS_V10_AND_ABOVE_TYPE,
Expand Down
51 changes: 46 additions & 5 deletions test/unit/bin/helpers/capabilityHelper.js
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,45 @@ describe("capabilityHelper.js", () => {
chai.assert.isUndefined(sent.cypress_version);
});
});

["16", "16.latest", "16.1.0", "16.1.1", "16.2.0"].forEach((version) => {
it(`adds forceHttp1=true for accessibility on cypress_version ${version}`, () => {
return sentRunSettings({ accessibility: true, cypress_version: version }).then(({ sent, bsConfig }) => {
chai.assert.equal(sent.config, "forceHttp1=true");
chai.assert.isUndefined(bsConfig.run_settings.config);
});
});
});

it("keeps the user's other config keys when adding forceHttp1", () => {
return sentRunSettings({ accessibility: true, cypress_version: "16.2.0", config: "video=false,baseUrl=https://x.test" }).then(({ sent }) => {
chai.assert.equal(sent.config, "video=false,baseUrl=https://x.test,forceHttp1=true");
});
});

["forceHttp1=false", "video=true, forceHttp1 = false"].forEach((config) => {
it(`does not override a forceHttp1 the user set (${config})`, () => {
return sentRunSettings({ accessibility: true, cypress_version: "16", config }).then(({ sent }) => {
chai.assert.equal(sent.config, config);
});
});
});

[
["accessibility on, Cypress 15", { accessibility: true, cypress_version: "15.21.1" }],
["accessibility on, no version (pinned to 15)", { accessibility: true }],
["accessibility off, Cypress 16", { accessibility: false, cypress_version: "16.2.0" }],
["accessibility on, cypress_version latest", { accessibility: true, cypress_version: "latest" }],
["accessibility on, Cypress 16.2.1 (fixed)", { accessibility: true, cypress_version: "16.2.1" }],
["accessibility on, Cypress 16.3.0 (fixed)", { accessibility: true, cypress_version: "16.3.0" }],
["accessibility on, Cypress 16.0.0 (before the regression)", { accessibility: true, cypress_version: "16.0.0" }],
].forEach(([name, runSettings]) => {
it(`does not add forceHttp1 for ${name}`, () => {
return sentRunSettings(runSettings).then(({ sent }) => {
chai.assert.isUndefined(sent.config);
});
});
});
});

it("handle empty test_suite", () => {
Expand Down Expand Up @@ -808,26 +847,28 @@ describe("capabilityHelper.js", () => {
else process.env.BROWSERSTACK_TEST_ACCESSIBILITY = savedA11yEnv;
});

["16", "16.2.0", "16.latest", 17].forEach((version) => {
["17", "17.0.0", "17.latest", 18].forEach((version) => {
it(`rejects cypress_version ${version} with accessibility on`, () => {
return validateError(a11yConfig({ accessibility: true, cypress_version: version })).then((error) => {
chai.assert.equal(error, Constants.validationMessages.ACCESSIBILITY_CYPRESS_VERSION_UNSUPPORTED);
});
});
});

it("rejects cypress_version 16 when accessibility is on through BROWSERSTACK_TEST_ACCESSIBILITY", () => {
it("rejects cypress_version 17 when accessibility is on through BROWSERSTACK_TEST_ACCESSIBILITY", () => {
process.env.BROWSERSTACK_TEST_ACCESSIBILITY = 'true';
return validateError(a11yConfig({ cypress_version: "16" })).then((error) => {
return validateError(a11yConfig({ cypress_version: "17" })).then((error) => {
chai.assert.equal(error, Constants.validationMessages.ACCESSIBILITY_CYPRESS_VERSION_UNSUPPORTED);
});
});

[
["cypress_version 15 with accessibility on", { accessibility: true, cypress_version: "15" }],
["cypress_version 16 with accessibility on", { accessibility: true, cypress_version: "16" }],
["cypress_version 16.2.0 with accessibility on", { accessibility: true, cypress_version: "16.2.0" }],
["no cypress_version with accessibility on", { accessibility: "true" }],
["cypress_version 16 with accessibility off", { accessibility: false, cypress_version: "16" }],
["cypress_version 16 without accessibility", { cypress_version: "16" }],
["cypress_version 17 with accessibility off", { accessibility: false, cypress_version: "17" }],
["cypress_version 17 without accessibility", { cypress_version: "17" }],
].forEach(([name, runSettings]) => {
it(`does not reject ${name} for accessibility`, () => {
return validateError(a11yConfig(runSettings)).then((error) => {
Expand Down