DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (c68fe15a81fc)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
# -*- coding: utf-8 -*-

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import json
import re
from types import FunctionType
from collections import namedtuple

from six import text_type

from taskgraph import create
from taskgraph.config import load_graph_config
from taskgraph.util import taskcluster, yaml, hash
from taskgraph.util.python_path import import_sibling_modules
from taskgraph.parameters import Parameters
from mozbuild.util import memoize


actions = []
callbacks = {}

Action = namedtuple('Action', ['order', 'cb_name', 'generic', 'action_builder'])


def is_json(data):
    """ Return ``True``, if ``data`` is a JSON serializable data structure. """
    try:
        json.dumps(data)
    except ValueError:
        return False
    return True


@memoize
def read_taskcluster_yml(filename):
    '''Load and parse .taskcluster.yml, memoized to save some time'''
    return yaml.load_yaml(filename)


@memoize
def hash_taskcluster_yml(filename):
    '''
    Generate a hash of the given .taskcluster.yml.  This is the first 10 digits
    of the sha256 of the file's content, and is used by administrative scripts
    to create a hook based on this content.
    '''
    return hash.hash_path(filename)[:10]


def register_callback_action(name, title, symbol, description, order=10000,
                             context=[], available=lambda parameters: True,
                             schema=None, generic=True, cb_name=None):
    """
    Register an action callback that can be triggered from supporting
    user interfaces, such as Treeherder.

    This function is to be used as a decorator for a callback that takes
    parameters as follows:

    ``parameters``:
        Decision task parameters, see ``taskgraph.parameters.Parameters``.
    ``input``:
        Input matching specified JSON schema, ``None`` if no ``schema``
        parameter is given to ``register_callback_action``.
    ``task_group_id``:
        The id of the task-group this was triggered for.
    ``task_id`` and `task``:
        task identifier and task definition for task the action was triggered
        for, ``None`` if no ``context`` parameters was given to
        ``register_callback_action``.

    Parameters
    ----------
    name : str
        An identifier for this action, used by UIs to find the action.
    title : str
        A human readable title for the action to be used as label on a button
        or text on a link for triggering the action.
    symbol : str
        Treeherder symbol for the action callback, this is the symbol that the
        task calling your callback will be displayed as. This is usually 1-3
        letters abbreviating the action title.
    description : str
        A human readable description of the action in **markdown**.
        This will be display as tooltip and in dialog window when the action
        is triggered. This is a good place to describe how to use the action.
    order : int
        Order of the action in menus, this is relative to the ``order`` of
        other actions declared.
    context : list of dict
        List of tag-sets specifying which tasks the action is can take as input.
        If no tag-sets is specified as input the action is related to the
        entire task-group, and won't be triggered with a given task.

        Otherwise, if ``context = [{'k': 'b', 'p': 'l'}, {'k': 't'}]`` will only
        be displayed in the context menu for tasks that has
        ``task.tags.k == 'b' && task.tags.p = 'l'`` or ``task.tags.k = 't'``.
        Esentially, this allows filtering on ``task.tags``.

        If this is a function, it is given the decision parameters and must return
        a value of the form described above.
    available : function
        An optional function that given decision parameters decides if the
        action is available. Defaults to a function that always returns ``True``.
    schema : dict
        JSON schema specifying input accepted by the action.
        This is optional and can be left ``null`` if no input is taken.
    generic : boolean
        Whether this is a generic action or has its own permissions.
    cb_name : string
        The name under which this function should be registered, defaulting to
        `name`.  This is used to generation actionPerm for non-generic hook
        actions, and thus appears in ci-configuration and various role and hook
        names.  Unlike `name`, which can appear multiple times, cb_name must be
        unique among all registered callbacks.

    Returns
    -------
    function
        To be used as decorator for the callback function.
    """
    mem = {"registered": False}  # workaround nonlocal missing in 2.x

    assert isinstance(title, text_type), 'title must be a string'
    assert isinstance(description, text_type), 'description must be a string'
    title = title.strip()
    description = description.strip()

    # ensure that context is callable
    if not callable(context):
        context_value = context
        context = lambda params: context_value  # noqa

    def register_callback(cb, cb_name=cb_name):
        assert isinstance(name, text_type), 'name must be a string'
        assert isinstance(order, int), 'order must be an integer'
        assert callable(schema) or is_json(schema), 'schema must be a JSON compatible object'
        assert isinstance(cb, FunctionType), 'callback must be a function'
        # Allow for json-e > 25 chars in the symbol.
        if '$' not in symbol:
            assert 1 <= len(symbol) <= 25, 'symbol must be between 1 and 25 characters'
        assert isinstance(symbol, text_type), 'symbol must be a string'

        assert not mem['registered'], 'register_callback_action must be used as decorator'
        if not cb_name:
            cb_name = name
        assert cb_name not in callbacks, 'callback name {} is not unique'.format(cb_name)

        def action_builder(parameters, graph_config, decision_task_id):
            if not available(parameters):
                return None

            actionPerm = 'generic' if generic else cb_name

            # gather up the common decision-task-supplied data for this action
            repo_param = '{}head_repository'.format(graph_config['project-repo-param-prefix'])
            repository = {
                'url': parameters[repo_param],
                'project': parameters['project'],
                'level': parameters['level'],
            }

            revision = parameters['{}head_rev'.format(graph_config['project-repo-param-prefix'])]
            push = {
                'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
                'pushlog_id': parameters['pushlog_id'],
                'revision': revision,
            }

            match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters[repo_param])
            if not match:
                raise Exception('Unrecognized {}'.format(repo_param))
            action = {
                'name': name,
                'title': title,
                'description': description,
                # target taskGroupId (the task group this decision task is creating)
                'taskGroupId': decision_task_id,
                'cb_name': cb_name,
                'symbol': symbol,
            }

            rv = {
                'name': name,
                'title': title,
                'description': description,
                'context': context(parameters),
            }
            if schema:
                rv['schema'] = schema(graph_config=graph_config) if callable(schema) else schema

            trustDomain = graph_config['trust-domain']
            level = parameters['level']
            tcyml_hash = hash_taskcluster_yml(graph_config.taskcluster_yml)

            # the tcyml_hash is prefixed with `/` in the hookId, so users will be granted
            # hooks:trigger-hook:project-gecko/in-tree-action-3-myaction/*; if another
            # action was named `myaction/release`, then the `*` in the scope would also
            # match that action.  To prevent such an accident, we prohibit `/` in hook
            # names.
            if '/' in actionPerm:
                raise Exception('`/` is not allowed in action names; use `-`')

            rv.update({
                'kind': 'hook',
                'hookGroupId': 'project-{}'.format(trustDomain),
                'hookId': 'in-tree-action-{}-{}/{}'.format(level, actionPerm, tcyml_hash),
                'hookPayload': {
                    # provide the decision-task parameters as context for triggerHook
                    "decision": {
                        'action': action,
                        'repository': repository,
                        'push': push,
                    },

                    # and pass everything else through from our own context
                    "user": {
                        'input': {'$eval': 'input'},
                        'taskId': {'$eval': 'taskId'},  # target taskId (or null)
                        'taskGroupId': {'$eval': 'taskGroupId'},  # target task group
                    }
                },
                'extra': {
                    'actionPerm': actionPerm,
                },
            })

            return rv

        actions.append(Action(order, cb_name, generic, action_builder))

        mem['registered'] = True
        callbacks[cb_name] = cb
        return cb
    return register_callback


def render_actions_json(parameters, graph_config, decision_task_id):
    """
    Render JSON object for the ``public/actions.json`` artifact.

    Parameters
    ----------
    parameters : taskgraph.parameters.Parameters
        Decision task parameters.

    Returns
    -------
    dict
        JSON object representation of the ``public/actions.json`` artifact.
    """
    assert isinstance(parameters, Parameters), 'requires instance of Parameters'
    actions = []
    for action in sorted(_get_actions(graph_config), key=lambda action: action.order):
        action = action.action_builder(parameters, graph_config, decision_task_id)
        if action:
            assert is_json(action), 'action must be a JSON compatible object'
            actions.append(action)
    return {
        'version': 1,
        'variables': {},
        'actions': actions,
    }


def sanity_check_task_scope(callback, parameters, graph_config):
    """
    If this action is not generic, then verify that this task has the necessary
    scope to run the action. This serves as a backstop preventing abuse by
    running non-generic actions using generic hooks. While scopes should
    prevent serious damage from such abuse, it's never a valid thing to do.
    """
    for action in _get_actions(graph_config):
        if action.cb_name == callback:
            break
    else:
        raise Exception('No action with cb_name {}'.format(callback))

    actionPerm = 'generic' if action.generic else action.cb_name

    repo_param = '{}head_repository'.format(graph_config['project-repo-param-prefix'])
    head_repository = parameters[repo_param]
    assert head_repository.startswith('https://hg.mozilla.org/')
    expected_scope = 'assume:repo:{}:action:{}'.format(head_repository[8:], actionPerm)

    # the scope should appear literally; no need for a satisfaction check. The use of
    # get_current_scopes here calls the auth service through the Taskcluster Proxy, giving
    # the precise scopes available to this task.
    if expected_scope not in taskcluster.get_current_scopes():
        raise Exception('Expected task scope {} for this action'.format(expected_scope))


def trigger_action_callback(task_group_id, task_id, input, callback, parameters, root,
                            test=False):
    """
    Trigger action callback with the given inputs. If `test` is true, then run
    the action callback in testing mode, without actually creating tasks.
    """
    graph_config = load_graph_config(root)
    graph_config.register()
    callbacks = _get_callbacks(graph_config)
    cb = callbacks.get(callback, None)
    if not cb:
        raise Exception('Unknown callback: {}. Known callbacks: {}'.format(
            callback, ', '.join(callbacks)))

    if test:
        create.testing = True
        taskcluster.testing = True

    if not test:
        sanity_check_task_scope(callback, parameters, graph_config)

    cb(Parameters(**parameters), graph_config, input, task_group_id, task_id)


def _load(graph_config):
    # Load all modules from this folder, relying on the side-effects of register_
    # functions to populate the action registry.
    import_sibling_modules(exceptions=('util.py',))
    return callbacks, actions


def _get_callbacks(graph_config):
    return _load(graph_config)[0]


def _get_actions(graph_config):
    return _load(graph_config)[1]