DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (b855de816bbf)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import

import os
import sys
import json

from collections import Iterable, OrderedDict
from types import StringTypes, ModuleType

import mozpack.path as mozpath

from mozbuild.util import (
    FileAvoidWrite,
    memoized_property,
    ReadOnlyDict,
)
from mozbuild.shellutil import quote as shell_quote


if sys.version_info.major == 2:
    text_type = unicode
else:
    text_type = str


class BuildConfig(object):
    """Represents the output of configure."""

    _CODE_CACHE = {}

    def __init__(self):
        self.topsrcdir = None
        self.topobjdir = None
        self.defines = {}
        self.non_global_defines = []
        self.substs = {}
        self.files = []
        self.mozconfig = None

    @classmethod
    def from_config_status(cls, path):
        """Create an instance from a config.status file."""
        code_cache = cls._CODE_CACHE
        mtime = os.path.getmtime(path)

        # cache the compiled code as it can be reused
        # we cache it the first time, or if the file changed
        if not path in code_cache or code_cache[path][0] != mtime:
            # Add config.status manually to sys.modules so it gets picked up by
            # iter_modules_in_path() for automatic dependencies.
            mod = ModuleType('config.status')
            mod.__file__ = path
            sys.modules['config.status'] = mod

            with open(path, 'rt') as fh:
                source = fh.read()
                code_cache[path] = (
                    mtime,
                    compile(source, path, 'exec', dont_inherit=1)
                )

        g = {
            '__builtins__': __builtins__,
            '__file__': path,
        }
        l = {}
        exec(code_cache[path][1], g, l)

        config = BuildConfig()

        for name in l['__all__']:
            setattr(config, name, l[name])

        return config


class ConfigEnvironment(object):
    """Perform actions associated with a configured but bare objdir.

    The purpose of this class is to preprocess files from the source directory
    and output results in the object directory.

    There are two types of files: config files and config headers,
    each treated through a different member function.

    Creating a ConfigEnvironment requires a few arguments:
      - topsrcdir and topobjdir are, respectively, the top source and
        the top object directory.
      - defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in
        autoconf.
      - non_global_defines are a list of names appearing in defines above
        that are not meant to be exported in ACDEFINES (see below)
      - substs is a dict filled from AC_SUBST in autoconf.

    ConfigEnvironment automatically defines one additional substs variable
    from all the defines not appearing in non_global_defines:
      - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
        preprocessor command lines. The order in which defines were given
        when creating the ConfigEnvironment is preserved.
    and two other additional subst variables from all the other substs:
      - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted
        order, for use in autoconf.mk. It includes ACDEFINES
        Only substs with a VALUE are included, such that the resulting file
        doesn't change when new empty substs are added.
        This results in less invalidation of build dependencies in the case
        of autoconf.mk..
      - ALLEMPTYSUBSTS contains the substs with an empty value, in the form
        NAME =.

    ConfigEnvironment expects a "top_srcdir" subst to be set with the top
    source directory, in msys format on windows. It is used to derive a
    "srcdir" subst when treating config files. It can either be an absolute
    path or a path relative to the topobjdir.
    """

    def __init__(self, topsrcdir, topobjdir, defines=None,
        non_global_defines=None, substs=None, source=None, mozconfig=None):

        if not source:
            source = mozpath.join(topobjdir, 'config.status')
        self.source = source
        self.defines = ReadOnlyDict(defines or {})
        self.non_global_defines = non_global_defines or []
        self.substs = dict(substs or {})
        self.topsrcdir = mozpath.abspath(topsrcdir)
        self.topobjdir = mozpath.abspath(topobjdir)
        self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
        self.lib_prefix = self.substs.get('LIB_PREFIX', '')
        self.rust_lib_prefix = self.substs.get('RUST_LIB_PREFIX', '')
        if 'LIB_SUFFIX' in self.substs:
            self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX']
        if 'RUST_LIB_SUFFIX' in self.substs:
            self.rust_lib_suffix = '.%s' % self.substs['RUST_LIB_SUFFIX']
        self.dll_prefix = self.substs.get('DLL_PREFIX', '')
        self.dll_suffix = self.substs.get('DLL_SUFFIX', '')
        self.host_dll_prefix = self.substs.get('HOST_DLL_PREFIX', '')
        self.host_dll_suffix = self.substs.get('HOST_DLL_SUFFIX', '')
        if self.substs.get('IMPORT_LIB_SUFFIX'):
            self.import_prefix = self.lib_prefix
            self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX']
        else:
            self.import_prefix = self.dll_prefix
            self.import_suffix = self.dll_suffix
        self.bin_suffix = self.substs.get('BIN_SUFFIX', '')

        global_defines = [name for name in self.defines
            if not name in self.non_global_defines]
        self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name,
            shell_quote(self.defines[name]).replace('$', '$$'))
            for name in sorted(global_defines)])
        def serialize(name, obj):
            if isinstance(obj, StringTypes):
                return obj
            if isinstance(obj, Iterable):
                return ' '.join(obj)
            raise Exception('Unhandled type %s for %s', type(obj), str(name))
        self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name,
            serialize(name, self.substs[name])) for name in self.substs if self.substs[name]]))
        self.substs['ALLEMPTYSUBSTS'] = '\n'.join(sorted(['%s =' % name
            for name in self.substs if not self.substs[name]]))

        self.substs = ReadOnlyDict(self.substs)

        self.external_source_dir = None
        external = self.substs.get('EXTERNAL_SOURCE_DIR', '')
        if external:
            external = mozpath.normpath(external)
            if not os.path.isabs(external):
                external = mozpath.join(self.topsrcdir, external)
            self.external_source_dir = mozpath.normpath(external)

        # Populate a Unicode version of substs. This is an optimization to make
        # moz.build reading faster, since each sandbox needs a Unicode version
        # of these variables and doing it over a thousand times is a hotspot
        # during sandbox execution!
        # Bug 844509 tracks moving everything to Unicode.
        self.substs_unicode = {}

        def decode(v):
            if not isinstance(v, text_type):
                try:
                    return v.decode('utf-8')
                except UnicodeDecodeError:
                    return v.decode('utf-8', 'replace')

        for k, v in self.substs.items():
            if not isinstance(v, StringTypes):
                if isinstance(v, Iterable):
                    type(v)(decode(i) for i in v)
            elif not isinstance(v, text_type):
                v = decode(v)

            self.substs_unicode[k] = v

        self.substs_unicode = ReadOnlyDict(self.substs_unicode)

    @property
    def is_artifact_build(self):
        return self.substs.get('MOZ_ARTIFACT_BUILDS', False)

    @memoized_property
    def acdefines(self):
        acdefines = dict((name, self.defines[name])
                         for name in self.defines
                         if name not in self.non_global_defines)
        return ReadOnlyDict(acdefines)

    @staticmethod
    def from_config_status(path):
        config = BuildConfig.from_config_status(path)

        return ConfigEnvironment(config.topsrcdir, config.topobjdir,
            config.defines, config.non_global_defines, config.substs, path)


class PartialConfigDict(object):
    """Facilitates mapping the config.statusd defines & substs with dict-like access.

    This allows a buildconfig client to use buildconfig.defines['FOO'] (and
    similar for substs), where the value of FOO is delay-loaded until it is
    needed.
    """
    def __init__(self, config_statusd, typ, environ_override=False):
        self._dict = {}
        self._datadir = mozpath.join(config_statusd, typ)
        self._config_track = mozpath.join(self._datadir, 'config.track')
        self._files = set()
        self._environ_override = environ_override

    def _load_config_track(self):
        existing_files = set()
        try:
            with open(self._config_track) as fh:
                existing_files.update(fh.read().splitlines())
        except IOError:
            pass
        return existing_files

    def _write_file(self, key, value):
        encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
        filename = mozpath.join(self._datadir, key)
        with FileAvoidWrite(filename) as fh:
            json.dump(value, fh, indent=4, encoding=encoding)
        return filename

    def _fill_group(self, values):
        # Clear out any cached values. This is mostly for tests that will check
        # the environment, write out a new set of variables, and then check the
        # environment again. Normally only configure ends up calling this
        # function, and other consumers create their own
        # PartialConfigEnvironments in new python processes.
        self._dict = {}

        existing_files = self._load_config_track()

        new_files = set()
        for k, v in values.iteritems():
            new_files.add(self._write_file(k, v))

        for filename in existing_files - new_files:
            # We can't actually os.remove() here, since make would not see that the
            # file has been removed and that the target needs to be updated. Instead
            # we just overwrite the file with a value of None, which is equivalent
            # to a non-existing file.
            with FileAvoidWrite(filename) as fh:
                json.dump(None, fh)

        with FileAvoidWrite(self._config_track) as fh:
            for f in sorted(new_files):
                fh.write('%s\n' % f)

    def __getitem__(self, key):
        if self._environ_override:
            if (key not in ('CPP', 'CXXCPP', 'SHELL')) and (key in os.environ):
                return os.environ[key]

        if key not in self._dict:
            data = None
            try:
                filename = mozpath.join(self._datadir, key)
                self._files.add(filename)
                with open(filename) as f:
                    data = json.load(f)
            except IOError:
                pass
            self._dict[key] = data

        if self._dict[key] is None:
            raise KeyError("'%s'" % key)
        return self._dict[key]

    def __setitem__(self, key, value):
        self._dict[key] = value

    def get(self, key, default=None):
        return self[key] if key in self else default

    def __contains__(self, key):
        try:
            return self[key] is not None
        except KeyError:
            return False

    def iteritems(self):
        existing_files = self._load_config_track()
        for f in existing_files:
            # The track file contains filenames, and the basename is the
            # variable name.
            var = mozpath.basename(f)
            yield var, self[var]


class PartialConfigEnvironment(object):
    """Allows access to individual config.status items via config.statusd/* files.

    This class is similar to the full ConfigEnvironment, which uses
    config.status, except this allows access and tracks dependencies to
    individual configure values. It is intended to be used during the build
    process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
    anything else that may need to access specific substs or defines.

    Creating a PartialConfigEnvironment requires only the topobjdir, which is
    needed to distinguish between the top-level environment and the js/src
    environment.

    The PartialConfigEnvironment automatically defines one additional subst variable
    from all the defines not appearing in non_global_defines:
      - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
        preprocessor command lines. The order in which defines were given
        when creating the ConfigEnvironment is preserved.

    and one additional define from all the defines as a dictionary:
      - ALLDEFINES contains all of the global defines as a dictionary. This is
      intended to be used instead of the defines structure from config.status so
      that scripts can depend directly on its value.
    """
    def __init__(self, topobjdir):
        config_statusd = mozpath.join(topobjdir, 'config.statusd')
        self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True)
        self.defines = PartialConfigDict(config_statusd, 'defines')
        self.topobjdir = topobjdir

    def write_vars(self, config):
        substs = config['substs'].copy()
        defines = config['defines'].copy()

        global_defines = [
            name for name in config['defines']
            if name not in config['non_global_defines']
        ]
        acdefines = ' '.join(['-D%s=%s' % (name,
            shell_quote(config['defines'][name]).replace('$', '$$'))
            for name in sorted(global_defines)])
        substs['ACDEFINES'] = acdefines

        all_defines = OrderedDict()
        for k in global_defines:
            all_defines[k] = config['defines'][k]
        defines['ALLDEFINES'] = all_defines

        self.substs._fill_group(substs)
        self.defines._fill_group(defines)

    def get_dependencies(self):
        return ['$(wildcard %s)' % f for f in self.substs._files | self.defines._files]