DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (c2593a3058af)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****

import argparse
import os
import re
from collections import defaultdict

from mozharness.base.script import PostScriptAction
from mozharness.base.transfer import TransferMixin

try_config_options = [
    [["--try-message"],
     {"action": "store",
      "dest": "try_message",
      "default": None,
      "help": "try syntax string to select tests to run",
      }],
]

test_flavors = {
    'browser-chrome': {},
    'chrome': {},
    'devtools-chrome': {},
    'mochitest': {},
    'xpcshell': {},
    'reftest': {
        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
    },
    'crashtest': {
        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
    },
    'web-platform-tests': {
        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
    },
    'web-platform-tests-reftests': {
        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
    },
    'web-platform-tests-wdspec': {
        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
    },
}


class TryToolsMixin(TransferMixin):
    """Utility functions for an interface between try syntax and out test harnesses.
    Requires log and script mixins."""

    harness_extra_args = None
    try_test_paths = {}
    known_try_arguments = {
        '--tag': ({
            'action': 'append',
            'dest': 'tags',
            'default': None,
        }, (
            'browser-chrome',
            'chrome',
            'devtools-chrome',
            'marionette',
            'mochitest',
            'web-plaftform-tests',
            'xpcshell',
        )),
        '--setenv': ({
            'action': 'append',
            'dest': 'setenv',
            'default': [],
            'metavar': 'NAME=VALUE',
        }, (
            'browser-chrome',
            'chrome',
            'crashtest',
            'devtools-chrome',
            'mochitest',
            'reftest',
        )),
    }

    def _extract_try_message(self):
        msg = None
        if "try_message" in self.config and self.config["try_message"]:
            msg = self.config["try_message"]
        elif 'TRY_COMMIT_MSG' in os.environ:
            msg = os.environ['TRY_COMMIT_MSG']

        if not msg:
            self.warning('Try message not found.')
        return msg

    def _extract_try_args(self, msg):
        """ Returns a list of args from a try message, for parsing """
        if not msg:
            return None
        all_try_args = None
        for line in msg.splitlines():
            if 'try: ' in line:
                # Autoland adds quotes to try strings that will confuse our
                # args later on.
                if line.startswith('"') and line.endswith('"'):
                    line = line[1:-1]
                # Allow spaces inside of [filter expressions]
                try_message = line.strip().split('try: ', 1)
                all_try_args = re.findall(r'(?:\[.*?\]|\S)+', try_message[1])
                break
        if not all_try_args:
            self.warning('Try syntax not found in: %s.' % msg)
        return all_try_args

    def try_message_has_flag(self, flag, message=None):
        """
        Returns True if --`flag` is present in message.
        """
        parser = argparse.ArgumentParser()
        parser.add_argument('--' + flag, action='store_true')
        message = message or self._extract_try_message()
        if not message:
            return False
        msg_list = self._extract_try_args(message)
        args, _ = parser.parse_known_args(msg_list)
        return getattr(args, flag, False)

    def _is_try(self):
        repo_path = None
        get_branch = self.config.get('branch', repo_path)
        if get_branch is not None:
            on_try = ('try' in get_branch or 'Try' in get_branch)
        elif os.environ is not None:
            on_try = ('TRY_COMMIT_MSG' in os.environ)
        else:
            on_try = False
        return on_try

    @PostScriptAction('download-and-extract')
    def set_extra_try_arguments(self, action, success=None):
        """Finds a commit message and parses it for extra arguments to pass to the test
        harness command line and test paths used to filter manifests.

        Extracting arguments from a commit message taken directly from the try_parser.
        """
        if not self._is_try():
            return

        msg = self._extract_try_message()
        if not msg:
            return

        all_try_args = self._extract_try_args(msg)
        if not all_try_args:
            return

        parser = argparse.ArgumentParser(
            description=('Parse an additional subset of arguments passed to try syntax'
                         ' and forward them to the underlying test harness command.'))

        label_dict = {}

        def label_from_val(val):
            if val in label_dict:
                return label_dict[val]
            return '--%s' % val.replace('_', '-')

        for label, (opts, _) in self.known_try_arguments.iteritems():
            if 'action' in opts and opts['action'] not in ('append', 'store',
                                                           'store_true', 'store_false'):
                self.fatal('Try syntax does not support passing custom or store_const '
                           'arguments to the harness process.')
            if 'dest' in opts:
                label_dict[opts['dest']] = label

            parser.add_argument(label, **opts)

        parser.add_argument('--try-test-paths', nargs='*')
        (args, _) = parser.parse_known_args(all_try_args)
        self.try_test_paths = self._group_test_paths(args.try_test_paths)
        del args.try_test_paths

        out_args = defaultdict(list)
        # This is a pretty hacky way to echo arguments down to the harness.
        # Hopefully this can be improved once we have a configuration system
        # in tree for harnesses that relies less on a command line.
        for arg, value in vars(args).iteritems():
            if value:
                label = label_from_val(arg)
                _, flavors = self.known_try_arguments[label]

                for f in flavors:
                    if isinstance(value, bool):
                        # A store_true or store_false argument.
                        out_args[f].append(label)
                    elif isinstance(value, list):
                        out_args[f].extend(['%s=%s' % (label, el) for el in value])
                    else:
                        out_args[f].append('%s=%s' % (label, value))

        self.harness_extra_args = dict(out_args)

    def _group_test_paths(self, args):
        rv = defaultdict(list)

        if args is None:
            return rv

        for item in args:
            suite, path = item.split(":", 1)
            rv[suite].append(path)
        return rv

    def try_args(self, flavor):
        """Get arguments, test_list derived from try syntax to apply to a command"""
        args = []
        if self.harness_extra_args:
            args = self.harness_extra_args.get(flavor, [])[:]

        if self.try_test_paths.get(flavor):
            self.info('TinderboxPrint: Tests will be run from the following '
                      'files: %s.' % ','.join(self.try_test_paths[flavor]))
            args.extend(['--this-chunk=1', '--total-chunks=1'])

            path_func = test_flavors[flavor].get("path", lambda x: x)
            tests = [path_func(os.path.normpath(item)) for item in self.try_test_paths[flavor]]
        else:
            tests = []

        if args or tests:
            self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
                      'to the test command:\nTinderboxPrint: \t%s -- %s' %
                      (" ".join(args), " ".join(tests)))

        return args, tests