diff options
Diffstat (limited to 'toolkit/components/backgroundtasks/tests/xpcshell/test_backgroundtask_experiments.js')
-rw-r--r-- | toolkit/components/backgroundtasks/tests/xpcshell/test_backgroundtask_experiments.js | 416 |
1 files changed, 416 insertions, 0 deletions
diff --git a/toolkit/components/backgroundtasks/tests/xpcshell/test_backgroundtask_experiments.js b/toolkit/components/backgroundtasks/tests/xpcshell/test_backgroundtask_experiments.js new file mode 100644 index 0000000000..087c557acf --- /dev/null +++ b/toolkit/components/backgroundtasks/tests/xpcshell/test_backgroundtask_experiments.js @@ -0,0 +1,416 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim: sw=4 ts=4 sts=4 et + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This file tests several things. +// +// 1. We verify that we can forcefully opt-in to (particular branches of) +// experiments, that the resulting Firefox Messaging Experiment applies, and +// that the Firefox Messaging System respects lifetime frequency caps. +// 2. We verify that Nimbus randomization works with specific Normandy +// randomization IDs. +// 3. We verify that relevant opt-out prefs disable the Nimbus and Firefox +// Messaging System experience. + +const { ASRouterTargeting } = ChromeUtils.import( + "resource://activity-stream/lib/ASRouterTargeting.jsm" +); + +// These randomization IDs were extracted by hand from Firefox instances. +// Randomization is sufficiently stable to hard-code these IDs rather than +// generating new ones at test time. +const BRANCH_MAP = { + "treatment-a": { + randomizationId: "d0e95fc3-fb15-4bc4-8151-a89582a56e29", + title: "Treatment A", + text: "Body A", + }, + "treatment-b": { + randomizationId: "90a60347-66cc-4716-9fef-cf49dd992d51", + title: "Treatment B", + text: "Body B", + }, +}; + +setupProfileService(); + +let taskProfile; + +// Arrange a dummy Remote Settings server so that no non-local network +// connections are opened. +// And arrange dummy task profile. +add_setup(() => { + info("Setting up profile service"); + let profileService = Cc["@mozilla.org/toolkit/profile-service;1"].getService( + Ci.nsIToolkitProfileService + ); + + let taskProfD = do_get_profile(); + taskProfD.append("test_backgroundtask_experiments_task"); + taskProfile = profileService.createUniqueProfile( + taskProfD, + "test_backgroundtask_experiments_task" + ); + + registerCleanupFunction(() => { + taskProfile.remove(true); + }); +}); + +function resetProfile(profile) { + profile.rootDir.remove(true); + profile.rootDir.create(Ci.nsIFile.DIRECTORY_TYPE, 0o700); + info(`Reset profile '${profile.rootDir.path}'`); +} + +// Run the "message" background task with some default configuration. Return +// "info" items output from the task as an array and as a map. +async function doMessage({ extraArgs = [], extraEnv = {} } = {}) { + let sentinel = Services.uuid.generateUUID().toString(); + sentinel = sentinel.substring(1, sentinel.length - 1); + + let infoArray = []; + let exitCode = await do_backgroundtask("message", { + extraArgs: [ + "--sentinel", + sentinel, + // Use a test-specific non-ephemeral profile, not the system-wide shared + // task-specific profile. + "--profile", + taskProfile.rootDir.path, + // Don't actually show a toast notification. + "--disable-alerts-service", + // Don't contact Remote Settings server. Can be overridden by subsequent + // `--experiments ...`. + "--no-experiments", + ...extraArgs, + ], + extraEnv: { + MOZ_LOG: "Dump:5,BackgroundTasks:5", + ...extraEnv, + }, + onStdoutLine: line => { + if (line.includes(sentinel)) { + let info = JSON.parse(line.split(sentinel)[1]); + infoArray.push(info); + } + }, + }); + + Assert.equal( + 0, + exitCode, + "The message background task exited with exit code 0" + ); + + // Turn [{x:...}, {y:...}] into {x:..., y:...}. + let infoMap = Object.assign({}, ...infoArray); + + return { infoArray, infoMap }; +} + +// Opt-in to an experiment. Verify that the experiment state is non-ephemeral, +// i.e., persisted. Verify that messages are shown until we hit the lifetime +// frequency caps. +// +// It's awkward to inspect the `ASRouter.jsm` internal state directly in this +// manner, but this is the pattern for testing such things at the time of +// writing. +add_task(async function test_backgroundtask_caps() { + let experimentFile = do_get_file("experiment.json"); + let experimentFileURI = Services.io.newFileURI(experimentFile); + + let { infoMap } = await doMessage({ + extraArgs: [ + // Opt-in to an experiment from a file. + "--url", + `${experimentFileURI.spec}?optin_branch=treatment-a`, + ], + }); + + // Verify that the correct experiment and branch generated an impression. + let impressions = infoMap.ASRouterState.messageImpressions; + Assert.deepEqual(Object.keys(impressions), ["test-experiment:treatment-a"]); + Assert.equal(impressions["test-experiment:treatment-a"].length, 1); + + // Verify that the correct toast notification was shown. + let alert = infoMap.showAlert.args[0]; + Assert.equal(alert.title, "Treatment A"); + Assert.equal(alert.text, "Body A"); + Assert.equal(alert.name, "optin-test-experiment:treatment-a"); + + // Now, do it again. No need to opt-in to the experiment this time. + ({ infoMap } = await doMessage({})); + + // Verify that only the correct experiment and branch generated an impression. + impressions = infoMap.ASRouterState.messageImpressions; + Assert.deepEqual(Object.keys(impressions), ["test-experiment:treatment-a"]); + Assert.equal(impressions["test-experiment:treatment-a"].length, 2); + + // Verify that the correct toast notification was shown. + alert = infoMap.showAlert.args[0]; + Assert.equal(alert.title, "Treatment A"); + Assert.equal(alert.text, "Body A"); + Assert.equal(alert.name, "optin-test-experiment:treatment-a"); + + // A third time. We'll hit the lifetime frequency cap (which is 2). + ({ infoMap } = await doMessage({})); + + // Verify that the correct experiment and branch impressions are untouched. + impressions = infoMap.ASRouterState.messageImpressions; + Assert.deepEqual(Object.keys(impressions), ["test-experiment:treatment-a"]); + Assert.equal(impressions["test-experiment:treatment-a"].length, 2); + + // Verify that no toast notication was shown. + Assert.ok(!("showAlert" in infoMap), "No alert shown"); +}); + +// Test that background tasks are enrolled into branches based on the Normandy +// randomization ID as expected. Run the message task with a hard-coded list of +// a single experiment and known randomization IDs, and verify that the enrolled +// branches are as expected. +add_task(async function test_backgroundtask_randomization() { + let experimentFile = do_get_file("experiment.json"); + + for (let [branchSlug, branchDetails] of Object.entries(BRANCH_MAP)) { + // Start fresh each time. + resetProfile(taskProfile); + + // Invoke twice; verify the branch is consistent each time. + for (let count = 1; count <= 2; count++) { + let { infoMap } = await doMessage({ + extraArgs: [ + // Read experiments from a file. + "--experiments", + experimentFile.path, + // Fixed randomization ID yields a deterministic enrollment branch assignment. + "--randomizationId", + branchDetails.randomizationId, + ], + }); + + // Verify that only the correct experiment and branch generated an impression. + let impressions = infoMap.ASRouterState.messageImpressions; + Assert.deepEqual(Object.keys(impressions), [ + `test-experiment:${branchSlug}`, + ]); + Assert.equal(impressions[`test-experiment:${branchSlug}`].length, count); + + // Verify that the correct toast notification was shown. + let alert = infoMap.showAlert.args[0]; + Assert.equal(alert.title, branchDetails.title, "Title is correct"); + Assert.equal(alert.text, branchDetails.text, "Text is correct"); + Assert.equal( + alert.name, + `test-experiment:${branchSlug}`, + "Name (tag) is correct" + ); + } + } +}); + +// Test that background tasks respect the datareporting and studies opt-out +// preferences. +add_task(async function test_backgroundtask_optout_preferences() { + let experimentFile = do_get_file("experiment.json"); + + let OPTION_MAP = { + "--no-datareporting": { + "datareporting.healthreport.uploadEnabled": false, + "app.shield.optoutstudies.enabled": true, + }, + "--no-optoutstudies": { + "datareporting.healthreport.uploadEnabled": true, + "app.shield.optoutstudies.enabled": false, + }, + }; + + for (let [option, expectedPrefs] of Object.entries(OPTION_MAP)) { + // Start fresh each time. + resetProfile(taskProfile); + + let { infoMap } = await doMessage({ + extraArgs: [ + option, + // Read experiments from a file. Opting in to an experiment with + // `--url` does not consult relevant preferences. + "--experiments", + experimentFile.path, + ], + }); + + Assert.deepEqual(infoMap.taskProfilePrefs, expectedPrefs); + + // Verify that no experiment generated an impression. + let impressions = infoMap.ASRouterState.messageImpressions; + Assert.deepEqual( + impressions, + [], + `No impressions generated with ${option}` + ); + + // Verify that no toast notication was shown. + Assert.ok(!("showAlert" in infoMap), `No alert shown with ${option}`); + } +}); + +const TARGETING_LIST = [ + // Target based on background task details. + ["isBackgroundTaskMode", 1], + ["backgroundTaskName == 'message'", 1], + ["backgroundTaskName == 'unrecognized'", 0], + // Filter based on `defaultProfile` targeting snapshot. + ["(currentDate|date - defaultProfile.currentDate|date) > 0", 1], + ["(currentDate|date - defaultProfile.currentDate|date) > 999999", 0], +]; + +// Test that background tasks targeting works for Nimbus experiments. +add_task(async function test_backgroundtask_Nimbus_targeting() { + let experimentFile = do_get_file("experiment.json"); + let experimentData = await IOUtils.readJSON(experimentFile.path); + + // We can't take a full environment snapshot under `xpcshell`. Select a few + // items that do work. + let target = { + currentDate: ASRouterTargeting.Environment.currentDate, + firefoxVersion: ASRouterTargeting.Environment.firefoxVersion, + }; + let targetSnapshot = await ASRouterTargeting.getEnvironmentSnapshot(target); + + for (let [targeting, expectedLength] of TARGETING_LIST) { + // Start fresh each time. + resetProfile(taskProfile); + + let snapshotFile = taskProfile.rootDir.clone(); + snapshotFile.append("targeting.snapshot.json"); + await IOUtils.writeJSON(snapshotFile.path, targetSnapshot); + + // Write updated experiment data. + experimentData.data.targeting = targeting; + let targetingExperimentFile = taskProfile.rootDir.clone(); + targetingExperimentFile.append("targeting.experiment.json"); + await IOUtils.writeJSON(targetingExperimentFile.path, experimentData); + + let { infoMap } = await doMessage({ + extraArgs: [ + "--experiments", + targetingExperimentFile.path, + "--targeting-snapshot", + snapshotFile.path, + ], + }); + + // Verify that the given targeting generated the expected number of impressions. + let impressions = infoMap.ASRouterState.messageImpressions; + Assert.equal( + Object.keys(impressions).length, + expectedLength, + `${expectedLength} impressions generated with targeting '${targeting}'` + ); + } +}); + +// Test that background tasks targeting works for Firefox Messaging System branches. +add_task(async function test_backgroundtask_Messaging_targeting() { + // Don't target the Nimbus experiment at all. Use a consistent + // randomization ID to always enroll in the first branch. Target + // the first branch of the Firefox Messaging Experiment to the given + // targeting. Therefore, we either get the first branch if the + // targeting matches, or nothing at all. + + let treatmentARandomizationId = BRANCH_MAP["treatment-a"].randomizationId; + + let experimentFile = do_get_file("experiment.json"); + let experimentData = await IOUtils.readJSON(experimentFile.path); + + // We can't take a full environment snapshot under `xpcshell`. Select a few + // items that do work. + let target = { + currentDate: ASRouterTargeting.Environment.currentDate, + firefoxVersion: ASRouterTargeting.Environment.firefoxVersion, + }; + let targetSnapshot = await ASRouterTargeting.getEnvironmentSnapshot(target); + + for (let [targeting, expectedLength] of TARGETING_LIST) { + // Start fresh each time. + resetProfile(taskProfile); + + let snapshotFile = taskProfile.rootDir.clone(); + snapshotFile.append("targeting.snapshot.json"); + await IOUtils.writeJSON(snapshotFile.path, targetSnapshot); + + // Write updated experiment data. + experimentData.data.targeting = "true"; + experimentData.data.branches[0].features[0].value.targeting = targeting; + + let targetingExperimentFile = taskProfile.rootDir.clone(); + targetingExperimentFile.append("targeting.experiment.json"); + await IOUtils.writeJSON(targetingExperimentFile.path, experimentData); + + let { infoMap } = await doMessage({ + extraArgs: [ + "--experiments", + targetingExperimentFile.path, + "--targeting-snapshot", + snapshotFile.path, + "--randomizationId", + treatmentARandomizationId, + ], + }); + + // Verify that the given targeting generated the expected number of impressions. + let impressions = infoMap.ASRouterState.messageImpressions; + Assert.equal( + Object.keys(impressions).length, + expectedLength, + `${expectedLength} impressions generated with targeting '${targeting}'` + ); + + if (expectedLength > 0) { + // Verify that the correct toast notification was shown. + let alert = infoMap.showAlert.args[0]; + Assert.equal( + alert.title, + BRANCH_MAP["treatment-a"].title, + "Title is correct" + ); + Assert.equal( + alert.text, + BRANCH_MAP["treatment-a"].text, + "Text is correct" + ); + Assert.equal( + alert.name, + `test-experiment:treatment-a`, + "Name (tag) is correct" + ); + } + } +}); + +// Verify that `RemoteSettingsClient.sync` is invoked before any +// `RemoteSettingsClient.get` invocations. This ensures the Remote Settings +// recipe collection is not allowed to go stale. +add_task( + async function test_backgroundtask_RemoteSettingsClient_invokes_sync() { + let { infoArray, infoMap } = await doMessage({}); + + Assert.ok( + "RemoteSettingsClient.get" in infoMap, + "RemoteSettingsClient.get was invoked" + ); + + for (let info of infoArray) { + if ("RemoteSettingsClient.get" in info) { + const { options: calledOptions } = info["RemoteSettingsClient.get"]; + Assert.ok( + calledOptions.forceSync, + "RemoteSettingsClient.get was first called with `forceSync`" + ); + return; + } + } + } +); |