DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (1aeaa33a64f9)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
import argparse
import imp
import os
import sys
from collections import defaultdict

from mozlog.structured import commandline
from wptrunner.wptcommandline import get_test_paths, set_from_config

manifest = None

def do_delayed_imports(wpt_dir):
    global manifest
    sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
    import manifest

def create_parser():
    p = argparse.ArgumentParser()
    p.add_argument("--check-clean", action="store_true",
                   help="Check that updating the manifest doesn't lead to any changes")
    p.add_argument("--rebuild", action="store_true",
                   help="Rebuild the manifest from scratch")
    commandline.add_logging_group(p)

    return p


def update(logger, wpt_dir, check_clean=True, rebuild=False):
    localpaths = imp.load_source("localpaths",
                                 os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
    kwargs = {"config": os.path.join(wpt_dir, "wptrunner.ini"),
              "tests_root": None,
              "metadata_root": None}

    set_from_config(kwargs)
    config = kwargs["config"]
    test_paths = get_test_paths(config)

    do_delayed_imports(wpt_dir)

    if check_clean:
        return _check_clean(logger, test_paths)

    return _update(logger, test_paths, rebuild)


def _update(logger, test_paths, rebuild):
    for url_base, paths in test_paths.iteritems():
        manifest_path = os.path.join(paths["metadata_path"], "MANIFEST.json")
        if rebuild:
            m = manifest.manifest.Manifest(url_base)
        else:
            m = manifest.manifest.load(paths["tests_path"], manifest_path)
        manifest.update.update(paths["tests_path"], m, working_copy=True)
        manifest.manifest.write(m, manifest_path)
    return 0


def _check_clean(logger, test_paths):
    manifests_by_path = {}
    rv = 0
    for url_base, paths in test_paths.iteritems():
        tests_path = paths["tests_path"]
        manifest_path = os.path.join(paths["metadata_path"], "MANIFEST.json")
        old_manifest = manifest.manifest.load(tests_path, manifest_path)
        new_manifest = manifest.manifest.Manifest.from_json(tests_path,
                                                            old_manifest.to_json())
        manifest.update.update(tests_path, new_manifest, working_copy=True)
        manifests_by_path[manifest_path] = (old_manifest, new_manifest)

    for manifest_path, (old_manifest, new_manifest) in manifests_by_path.iteritems():
        if not diff_manifests(logger, manifest_path, old_manifest, new_manifest):
            rv = 1
    if rv:
        logger.error("Manifest %s is outdated, use |mach wpt-manifest-update| to fix." % manifest_path)

    return rv


def diff_manifests(logger, manifest_path, old_manifest, new_manifest):
    """Lint the differences between old and new versions of a
    manifest. Differences are considered significant (and so produce
    lint errors) if they produce a meaningful difference in the actual
    tests run.

    :param logger: mozlog logger to use for output
    :param manifest_path: Path to the manifest being linted
    :param old_manifest: Manifest object representing the initial manifest
    :param new_manifest: Manifest object representing the updated manifest
    """
    logger.info("Diffing old and new manifests %s" % manifest_path)
    old_items, new_items = defaultdict(set), defaultdict(set)
    for manifest, items in [(old_manifest, old_items),
                            (new_manifest, new_items)]:
        for test_type, path, tests in manifest:
            for test in tests:
                test_id = [test.id]
                test_id.extend(tuple(item) if isinstance(item, list) else item
                               for item in test.meta_key())
                if hasattr(test, "references"):
                    test_id.extend(tuple(item) for item in test.references)
                test_id = tuple(test_id)
                items[path].add((test_type, test_id))

    old_paths = set(old_items.iterkeys())
    new_paths = set(new_items.iterkeys())

    added_paths = new_paths - old_paths
    deleted_paths = old_paths - new_paths

    common_paths = new_paths & old_paths

    clean = True

    for path in added_paths:
        clean = False
        log_error(logger, manifest_path, "%s in source but not in manifest." % path)
    for path in deleted_paths:
        clean = False
        log_error(logger, manifest_path, "%s in manifest but removed from source." % path)

    for path in common_paths:
        old_tests = old_items[path]
        new_tests = new_items[path]
        added_tests = new_tests - old_tests
        removed_tests = old_tests - new_tests
        if added_tests or removed_tests:
            clean = False
            log_error(logger, manifest_path, "%s changed test types or metadata" % path)

    if clean:
        # Manifest currently has some list vs tuple inconsistencies that break
        # a simple equality comparison.
        new_paths = {(key, value[0], value[1])
                     for (key, value) in new_manifest.to_json()["paths"].iteritems()}
        old_paths = {(key, value[0], value[1])
                     for (key, value) in old_manifest.to_json()["paths"].iteritems()}
        if old_paths != new_paths:
            logger.warning("Manifest %s contains correct tests but file hashes changed; please update" % manifest_path)

    return clean

def log_error(logger, manifest_path, msg):
    logger.lint_error(path=manifest_path,
                      message=msg,
                      lineno=0,
                      source="",
                      linter="wpt-manifest")