1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
import mozpack.path as mozpath
from mozbuild.base import MozbuildObject
from mozbuild.util import memoize
from taskgraph import files_changed
from taskgraph.optimize import register_strategy, OptimizationStrategy
from taskgraph.util.taskcluster import find_task_id, status_task
logger = logging.getLogger(__name__)
@register_strategy("index-search")
class IndexSearch(OptimizationStrategy):
# A task with no dependencies remaining after optimization will be replaced
# if artifacts exist for the corresponding index_paths.
# Otherwise, we're in one of the following cases:
# - the task has un-optimized dependencies
# - the artifacts have expired
# - some changes altered the index_paths and new artifacts need to be
# created.
# In every of those cases, we need to run the task to create or refresh
# artifacts.
def should_replace_task(self, task, params, index_paths):
"Look for a task with one of the given index paths"
for index_path in index_paths:
try:
task_id = find_task_id(index_path)
status = status_task(task_id)
if status not in ("exception", "failed"):
return task_id
except KeyError:
# 404 will end up here and go on to the next index path
pass
return False
@register_strategy("skip-unless-changed")
class SkipUnlessChanged(OptimizationStrategy):
def should_remove_task(self, task, params, file_patterns):
# pushlog_id == -1 - this is the case when run from a cron.yml job
if params.get("pushlog_id") == -1:
return False
changed = files_changed.check(params, file_patterns)
if not changed:
logger.debug(
"no files found matching a pattern in `skip-unless-changed` for "
+ task.label
)
return True
return False
@register_strategy("skip-unless-schedules")
class SkipUnlessSchedules(OptimizationStrategy):
@memoize
def scheduled_by_push(self, repository, revision):
changed_files = files_changed.get_changed_files(repository, revision)
mbo = MozbuildObject.from_environment()
# the decision task has a sparse checkout, so, mozbuild_reader will use
# a MercurialRevisionFinder with revision '.', which should be the same
# as `revision`; in other circumstances, it will use a default reader
rdr = mbo.mozbuild_reader(config_mode="empty")
components = set()
for p, m in rdr.files_info(changed_files).items():
components |= set(m["SCHEDULES"].components)
return components
def should_remove_task(self, task, params, conditions):
if params.get("pushlog_id") == -1:
return False
scheduled = self.scheduled_by_push(
params["head_repository"], params["head_rev"]
)
conditions = set(conditions)
# if *any* of the condition components are scheduled, do not optimize
if conditions & scheduled:
return False
return True
@register_strategy("skip-unless-has-relevant-tests")
class SkipUnlessHasRelevantTests(OptimizationStrategy):
"""Optimizes tasks that don't run any tests that were
in child directories of a modified file.
"""
@memoize
def get_changed_dirs(self, repo, rev):
changed = map(mozpath.dirname, files_changed.get_changed_files(repo, rev))
# Filter out empty directories (from files modified in the root).
# Otherwise all tasks would be scheduled.
return {d for d in changed if d}
def should_remove_task(self, task, params, _):
if not task.attributes.get("test_manifests"):
return True
for d in self.get_changed_dirs(params["head_repository"], params["head_rev"]):
for t in task.attributes["test_manifests"]:
if t.startswith(d):
logger.debug(
"{} runs a test path ({}) contained by a modified file ({})".format(
task.label, t, d
)
)
return False
return True
|
