DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (8aa8bbbf0bee)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import os
import sys

try:
    import cPickle as pickle
except ImportError:
    import pickle


import manifestparser
import mozpack.path as mozpath
from mozpack.copier import FileCopier
from mozpack.manifests import InstallManifest

# These definitions provide a single source of truth for modules attempting
# to get a view of all tests for a build. Used by the emitter to figure out
# how to read/install manifests and by test dependency annotations in Files()
# entries to enumerate test flavors.

# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
#    (flavor, install_root, install_subdir, package_tests)
#
# flavor identifies the flavor of this test.
# install_root is the path prefix to install the files starting from the root
#     directory and not as specified by the manifest location. (bug 972168)
# install_subdir is the path of where to install the files in
#     the tests directory.
# package_tests indicates whether to package test files into the test
#     package; suites that compile the test files should not install
#     them into the test package.
#
TEST_MANIFESTS = dict(
    A11Y=('a11y', 'testing/mochitest', 'a11y', True),
    BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
    ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
    FIREFOX_UI_FUNCTIONAL=('firefox-ui-functional', 'firefox-ui', '.', False),
    FIREFOX_UI_UPDATE=('firefox-ui-update', 'firefox-ui', '.', False),
    PUPPETEER_FIREFOX=('firefox-ui-functional', 'firefox-ui', '.', False),
    PYTHON_UNITTEST=('python', 'python', '.', False),
    CRAMTEST=('cram', 'cram', '.', False),
    TELEMETRY_TESTS_CLIENT=(
        'telemetry-tests-client',
        'toolkit/components/telemetry/tests/marionette', '.', False
    ),

    # marionette tests are run from the srcdir
    # TODO(ato): make packaging work as for other test suites
    MARIONETTE=('marionette', 'marionette', '.', False),
    MARIONETTE_UNIT=('marionette', 'marionette', '.', False),
    MARIONETTE_WEBAPI=('marionette', 'marionette', '.', False),

    MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
    MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
    WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
    XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
)

# reftests, wpt, and puppeteer all have their own manifest formats
# and are processed separately
REFTEST_FLAVORS = ('crashtest', 'reftest')
PUPPETEER_FLAVORS = ('puppeteer',)
WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',)


def all_test_flavors():
    return ([v[0] for v in TEST_MANIFESTS.values()] +
            list(REFTEST_FLAVORS) +
            list(PUPPETEER_FLAVORS) +
            list(WEB_PLATFORM_TESTS_FLAVORS))


class TestInstallInfo(object):
    def __init__(self):
        self.seen = set()
        self.pattern_installs = []
        self.installs = []
        self.external_installs = set()
        self.deferred_installs = set()

    def __ior__(self, other):
        self.pattern_installs.extend(other.pattern_installs)
        self.installs.extend(other.installs)
        self.external_installs |= other.external_installs
        self.deferred_installs |= other.deferred_installs
        return self


class SupportFilesConverter(object):
    """Processes a "support-files" entry from a test object, either from
    a parsed object from a test manifests or its representation in
    moz.build and returns the installs to perform for this test object.

    Processing the same support files multiple times will not have any further
    effect, and the structure of the parsed objects from manifests will have a
    lot of repeated entries, so this class takes care of memoizing.
    """

    def __init__(self):
        self._fields = (('head', set()),
                        ('support-files', set()),
                        ('generated-files', set()))

    def convert_support_files(self, test, install_root, manifest_dir, out_dir):
        # Arguments:
        #  test - The test object to process.
        #  install_root - The directory under $objdir/_tests that will contain
        #                 the tests for this harness (examples are "testing/mochitest",
        #                 "xpcshell").
        #  manifest_dir - Absoulute path to the (srcdir) directory containing the
        #                 manifest that included this test
        #  out_dir - The path relative to $objdir/_tests used as the destination for the
        #            test, based on the relative path to the manifest in the srcdir,
        #            the install_root, and 'install-to-subdir', if present in the manifest.
        info = TestInstallInfo()
        for field, seen in self._fields:
            value = test.get(field, '')
            for pattern in value.split():

                # We track uniqueness locally (per test) where duplicates are forbidden,
                # and globally, where they are permitted. If a support file appears multiple
                # times for a single test, there are unnecessary entries in the manifest. But
                # many entries will be shared across tests that share defaults.
                # We need to memoize on the basis of both the path and the output
                # directory for the benefit of tests specifying 'install-to-subdir'.
                key = field, pattern, out_dir
                if key in info.seen:
                    raise ValueError(
                        "%s appears multiple times in a test manifest under a %s field,"
                        " please omit the duplicate entry." % (pattern, field))
                info.seen.add(key)
                if key in seen:
                    continue
                seen.add(key)

                if field == 'generated-files':
                    info.external_installs.add(mozpath.normpath(mozpath.join(out_dir, pattern)))
                # '!' indicates our syntax for inter-directory support file
                # dependencies. These receive special handling in the backend.
                elif pattern[0] == '!':
                    info.deferred_installs.add(pattern)
                # We only support globbing on support-files because
                # the harness doesn't support * for head.
                elif '*' in pattern and field == 'support-files':
                    info.pattern_installs.append((manifest_dir, pattern, out_dir))
                # "absolute" paths identify files that are to be
                # placed in the install_root directory (no globs)
                elif pattern[0] == '/':
                    full = mozpath.normpath(mozpath.join(manifest_dir,
                                                         mozpath.basename(pattern)))
                    info.installs.append((full, mozpath.join(install_root, pattern[1:])))
                else:
                    full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
                    dest_path = mozpath.join(out_dir, pattern)

                    # If the path resolves to a different directory
                    # tree, we take special behavior depending on the
                    # entry type.
                    if not full.startswith(manifest_dir):
                        # If it's a support file, we install the file
                        # into the current destination directory.
                        # This implementation makes installing things
                        # with custom prefixes impossible. If this is
                        # needed, we can add support for that via a
                        # special syntax later.
                        if field == 'support-files':
                            dest_path = mozpath.join(out_dir,
                                                     os.path.basename(pattern))
                        # If it's not a support file, we ignore it.
                        # This preserves old behavior so things like
                        # head files doesn't get installed multiple
                        # times.
                        else:
                            continue
                    info.installs.append((full, mozpath.normpath(dest_path)))
        return info


def _resolve_installs(paths, topobjdir, manifest):
    """Using the given paths as keys, find any unresolved installs noted
    by the build backend corresponding to those keys, and add them
    to the given manifest.
    """
    filename = os.path.join(topobjdir, 'test-installs.pkl')
    with open(filename, 'rb') as fh:
        resolved_installs = pickle.load(fh)

    for path in paths:
        path = path[2:]
        if path not in resolved_installs:
            raise Exception('A cross-directory support file path noted in a '
                            'test manifest does not appear in any other manifest.\n "%s" '
                            'must appear in another test manifest to specify an install '
                            'for "!/%s".' % (path, path))
        installs = resolved_installs[path]
        for install_info in installs:
            try:
                if len(install_info) == 3:
                    manifest.add_pattern_link(*install_info)
                if len(install_info) == 2:
                    manifest.add_link(*install_info)
            except ValueError:
                # A duplicate value here is pretty likely when running
                # multiple directories at once, and harmless.
                pass


def _make_install_manifest(topsrcdir, topobjdir, test_objs):

    flavor_info = {flavor: (root, prefix, install)
                   for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}

    converter = SupportFilesConverter()
    install_info = TestInstallInfo()

    for o in test_objs:
        flavor = o['flavor']
        if flavor not in flavor_info:
            # This is a test flavor that isn't installed by the build system.
            continue
        root, prefix, install = flavor_info[flavor]
        if not install:
            # This flavor isn't installed to the objdir.
            continue

        manifest_path = o['manifest']
        manifest_dir = mozpath.dirname(manifest_path)

        out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
        file_relpath = o['file_relpath']
        source = mozpath.join(topsrcdir, file_relpath)
        dest = mozpath.join(root, prefix, file_relpath)
        if 'install-to-subdir' in o:
            out_dir = mozpath.join(out_dir, o['install-to-subdir'])
            manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
            dest = mozpath.join(out_dir, manifest_relpath)

        install_info.installs.append((source, dest))
        install_info |= converter.convert_support_files(o, root,
                                                        manifest_dir,
                                                        out_dir)

    manifest = InstallManifest()

    for source, dest in set(install_info.installs):
        if dest in install_info.external_installs:
            continue
        manifest.add_link(source, dest)
    for base, pattern, dest in install_info.pattern_installs:
        manifest.add_pattern_link(base, pattern, dest)

    _resolve_installs(install_info.deferred_installs, topobjdir, manifest)

    return manifest


def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
    """Installs the requested test files to the objdir. This is invoked by
    test runners to avoid installing tens of thousands of test files when
    only a few tests need to be run.
    """

    if test_objs:
        manifest = _make_install_manifest(topsrcdir, topobjdir, test_objs)
    else:
        # If we don't actually have a list of tests to install we install
        # test and support files wholesale.
        manifest = InstallManifest(mozpath.join(topobjdir, '_build_manifests',
                                                'install', '_test_files'))

    harness_files_manifest = mozpath.join(topobjdir, '_build_manifests',
                                          'install', tests_root)

    if os.path.isfile(harness_files_manifest):
        # If the backend has generated an install manifest for test harness
        # files they are treated as a monolith and installed each time we
        # run tests. Fortunately there are not very many.
        manifest |= InstallManifest(harness_files_manifest)

    copier = FileCopier()
    manifest.populate_registry(copier)
    copier.copy(mozpath.join(topobjdir, tests_root),
                remove_unaccounted=False)


# Convenience methods for test manifest reading.
def read_manifestparser_manifest(context, manifest_path):
    path = manifest_path.full_path
    return manifestparser.TestManifest(manifests=[path], strict=True,
                                       rootdir=context.config.topsrcdir,
                                       finder=context._finder,
                                       handle_defaults=False)


def read_reftest_manifest(context, manifest_path):
    import reftest
    path = manifest_path.full_path
    manifest = reftest.ReftestManifest(finder=context._finder)
    manifest.load(path)
    return manifest


def read_wpt_manifest(context, paths):
    manifest_path, tests_root = paths
    full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
    old_path = sys.path[:]
    try:
        # Setup sys.path to include all the dependencies required to import
        # the web-platform-tests manifest parser. web-platform-tests provides
        # a the localpaths.py to do the path manipulation, which we load,
        # providing the __file__ variable so it can resolve the relative
        # paths correctly.
        paths_file = os.path.join(context.config.topsrcdir, "testing",
                                  "web-platform", "tests", "tools", "localpaths.py")
        _globals = {"__file__": paths_file}
        execfile(paths_file, _globals)
        import manifest as wptmanifest
    finally:
        sys.path = old_path
        f = context._finder.get(full_path)
        try:
            rv = wptmanifest.manifest.load(tests_root, f)
        except wptmanifest.manifest.ManifestVersionMismatch:
            # If we accidentially end up with a committed manifest that's the wrong
            # version, then return an empty manifest here just to not break the build
            rv = wptmanifest.manifest.Manifest()
        return rv