DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (d38398e5144e)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

"""Utility functions for mozrunner"""

import mozinfo
import os
import sys

__all__ = ['findInPath', 'get_metadata_from_egg']


# python package method metadata by introspection
try:
    import pkg_resources

    def get_metadata_from_egg(module):
        ret = {}
        try:
            dist = pkg_resources.get_distribution(module)
        except pkg_resources.DistributionNotFound:
            return {}
        if dist.has_metadata("PKG-INFO"):
            key = None
            value = ""
            for line in dist.get_metadata("PKG-INFO").splitlines():
                # see http://www.python.org/dev/peps/pep-0314/
                if key == 'Description':
                    # descriptions can be long
                    if not line or line[0].isspace():
                        value += '\n' + line
                        continue
                    else:
                        key = key.strip()
                        value = value.strip()
                        ret[key] = value

                key, value = line.split(':', 1)
                key = key.strip()
                value = value.strip()
                ret[key] = value
        if dist.has_metadata("requires.txt"):
            ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt")
        return ret
except ImportError:
    # package resources not avaialable
    def get_metadata_from_egg(module):
        return {}


def findInPath(fileName, path=os.environ['PATH']):
    """python equivalent of which; should really be in the stdlib"""
    dirs = path.split(os.pathsep)
    for dir in dirs:
        if os.path.isfile(os.path.join(dir, fileName)):
            return os.path.join(dir, fileName)
        if mozinfo.isWin:
            if os.path.isfile(os.path.join(dir, fileName + ".exe")):
                return os.path.join(dir, fileName + ".exe")

if __name__ == '__main__':
    for i in sys.argv[1:]:
        print findInPath(i)


def _find_marionette_in_args(*args, **kwargs):
    try:
        m = [a for a in args + tuple(kwargs.values()) if hasattr(a, 'session')][0]
    except IndexError:
        print("Can only apply decorator to function using a marionette object")
        raise
    return m


def _raw_log():
    import logging
    return logging.getLogger(__name__)


def test_environment(xrePath, env=None, crashreporter=True, debugger=False,
                     dmdPath=None, lsanPath=None, log=None):
    """
    populate OS environment variables for mochitest and reftests.

    Originally comes from automationutils.py. Don't use that for new code.
    """

    env = os.environ.copy() if env is None else env
    log = log or _raw_log()

    assert os.path.isabs(xrePath)

    if mozinfo.isMac:
        ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS")
    else:
        ldLibraryPath = xrePath

    envVar = None
    dmdLibrary = None
    preloadEnvVar = None
    if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk":
        # Skip all of this, it's only valid for the host.
        pass
    elif mozinfo.isUnix:
        envVar = "LD_LIBRARY_PATH"
        env['MOZILLA_FIVE_HOME'] = xrePath
        dmdLibrary = "libdmd.so"
        preloadEnvVar = "LD_PRELOAD"
    elif mozinfo.isMac:
        envVar = "DYLD_LIBRARY_PATH"
        dmdLibrary = "libdmd.dylib"
        preloadEnvVar = "DYLD_INSERT_LIBRARIES"
    elif mozinfo.isWin:
        envVar = "PATH"
        dmdLibrary = "dmd.dll"
        preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
    if envVar:
        envValue = ((env.get(envVar), str(ldLibraryPath))
                    if mozinfo.isWin
                    else (ldLibraryPath, dmdPath, env.get(envVar)))
        env[envVar] = os.path.pathsep.join([path for path in envValue if path])

    if dmdPath and dmdLibrary and preloadEnvVar:
        env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)

    # crashreporter
    env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
    env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'

    if crashreporter and not debugger:
        env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
        env['MOZ_CRASHREPORTER'] = '1'
    else:
        env['MOZ_CRASHREPORTER_DISABLE'] = '1'

    # Crash on non-local network connections by default.
    # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
    # enable non-local connections for the purposes of local testing.  Don't
    # override the user's choice here.  See bug 1049688.
    env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')

    # Set WebRTC logging in case it is not set yet
    env.setdefault(
        'MOZ_LOG',
        'signaling:3,mtransport:4,DataChannel:4,jsep:4,MediaPipelineFactory:4'
    )
    env.setdefault('R_LOG_LEVEL', '6')
    env.setdefault('R_LOG_DESTINATION', 'stderr')
    env.setdefault('R_LOG_VERBOSE', '1')

    # ASan specific environment stuff
    asan = bool(mozinfo.info.get("asan"))
    if asan and (mozinfo.isLinux or mozinfo.isMac):
        try:
            # Symbolizer support
            llvmsym = os.path.join(xrePath, "llvm-symbolizer")
            if os.path.isfile(llvmsym):
                env["ASAN_SYMBOLIZER_PATH"] = llvmsym
                log.info("INFO | runtests.py | ASan using symbolizer at %s"
                         % llvmsym)
            else:
                log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find"
                         " ASan symbolizer at %s" % llvmsym)

            # Returns total system memory in kilobytes.
            # Works only on unix-like platforms where `free` is in the path.
            totalMemory = int(os.popen("free").readlines()[1].split()[1])

            # Only 4 GB RAM or less available? Use custom ASan options to reduce
            # the amount of resources required to do the tests. Standard options
            # will otherwise lead to OOM conditions on the current test slaves.
            message = "INFO | runtests.py | ASan running in %s configuration"
            asanOptions = []
            if totalMemory <= 1024 * 1024 * 4:
                message = message % 'low-memory'
                asanOptions = [
                    'quarantine_size=50331648', 'malloc_context_size=5']
            else:
                message = message % 'default memory'

            if lsanPath:
                log.info("LSan enabled.")
                asanOptions.append('detect_leaks=1')
                lsanOptions = ["exitcode=0"]
                # Uncomment out the next line to report the addresses of leaked objects.
                # lsanOptions.append("report_objects=1")
                suppressionsFile = os.path.join(
                    lsanPath, 'lsan_suppressions.txt')
                if os.path.exists(suppressionsFile):
                    log.info("LSan using suppression file " + suppressionsFile)
                    lsanOptions.append("suppressions=" + suppressionsFile)
                else:
                    log.info("WARNING | runtests.py | LSan suppressions file"
                             " does not exist! " + suppressionsFile)
                env["LSAN_OPTIONS"] = ':'.join(lsanOptions)

            if len(asanOptions):
                env['ASAN_OPTIONS'] = ':'.join(asanOptions)

        except OSError as err:
            log.info("Failed determine available memory, disabling ASan"
                     " low-memory configuration: %s" % err.strerror)
        except:
            log.info("Failed determine available memory, disabling ASan"
                     " low-memory configuration")
        else:
            log.info(message)

    tsan = bool(mozinfo.info.get("tsan"))
    if tsan and mozinfo.isLinux:
        # Symbolizer support.
        llvmsym = os.path.join(xrePath, "llvm-symbolizer")
        if os.path.isfile(llvmsym):
            env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym
            log.info("INFO | runtests.py | TSan using symbolizer at %s"
                     % llvmsym)
        else:
            log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan"
                     " symbolizer at %s" % llvmsym)

    return env


def get_stack_fixer_function(utilityPath, symbolsPath):
    """
    Return a stack fixing function, if possible, to use on output lines.

    A stack fixing function checks if a line conforms to the output from
    MozFormatCodeAddressDetails.  If the line does not, the line is returned
    unchanged.  If the line does, an attempt is made to convert the
    file+offset into something human-readable (e.g. a function name).
    """
    if not mozinfo.info.get('debug'):
        return None

    def import_stack_fixer_module(module_name):
        sys.path.insert(0, utilityPath)
        module = __import__(module_name, globals(), locals(), [])
        sys.path.pop(0)
        return module

    if symbolsPath and os.path.exists(symbolsPath):
        # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad
        # symbol files).
        # This method is preferred for Tinderbox builds, since native
        # symbols may have been stripped.
        stack_fixer_module = import_stack_fixer_module(
            'fix_stack_using_bpsyms')

        def stack_fixer_function(line):
            return stack_fixer_module.fixSymbols(line, symbolsPath)

    elif mozinfo.isMac:
        # Run each line through fix_macosx_stack.py (uses atos).
        # This method is preferred for developer machines, so we don't
        # have to run "make buildsymbols".
        stack_fixer_module = import_stack_fixer_module(
            'fix_macosx_stack')

        def stack_fixer_function(line):
            return stack_fixer_module.fixSymbols(line)

    elif mozinfo.isLinux:
        # Run each line through fix_linux_stack.py (uses addr2line).
        # This method is preferred for developer machines, so we don't
        # have to run "make buildsymbols".
        stack_fixer_module = import_stack_fixer_module(
            'fix_linux_stack')

        def stack_fixer_function(line):
            return stack_fixer_module.fixSymbols(line)

    else:
        return None

    return stack_fixer_function