DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (8d30b15ca8f9)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import

import argparse
import optparse
import os
import sys
from collections import defaultdict

from . import handlers
from . import formatters
from .structuredlog import StructuredLogger, set_default_logger
import six

log_formatters = {
    'raw': (formatters.JSONFormatter, "Raw structured log messages "
                                      "(provided by mozlog)"),
    'unittest': (formatters.UnittestFormatter, "Unittest style output "
                                               "(provided by mozlog)"),
    'xunit': (formatters.XUnitFormatter, "xUnit compatible XML "
                                         "(povided by mozlog)"),
    'html': (formatters.HTMLFormatter, "HTML report "
                                       "(provided by mozlog)"),
    'mach': (formatters.MachFormatter, "Human-readable output "
                                       "(provided by mozlog)"),
    'tbpl': (formatters.TbplFormatter, "TBPL style log format "
                                       "(provided by mozlog)"),
    'grouped': (formatters.GroupingFormatter, "Grouped summary of test results "
                                              "(provided by mozlog)"),
    'errorsummary': (formatters.ErrorSummaryFormatter, argparse.SUPPRESS),
}

TEXT_FORMATTERS = ('raw', 'mach')
"""a subset of formatters for non test harnesses related applications"""


DOCS_URL = "https://firefox-source-docs.mozilla.org/mozbase/mozlog.html"


def level_filter_wrapper(formatter, level):
    return handlers.LogLevelFilter(formatter, level)


def verbose_wrapper(formatter, verbose):
    formatter.verbose = verbose
    return formatter


def compact_wrapper(formatter, compact):
    formatter.compact = compact
    return formatter


def buffer_handler_wrapper(handler, buffer_limit):
    if buffer_limit == "UNLIMITED":
        buffer_limit = None
    else:
        buffer_limit = int(buffer_limit)
    return handlers.BufferHandler(handler, buffer_limit)


def screenshot_wrapper(formatter, enable_screenshot):
    formatter.enable_screenshot = enable_screenshot
    return formatter


def valgrind_handler_wrapper(handler):
    return handlers.ValgrindHandler(handler)


def default_formatter_options(log_type, overrides):
    formatter_option_defaults = {
        "raw": {
            "level": "debug"
        }
    }
    rv = {"verbose": False,
          "level": "info"}
    rv.update(formatter_option_defaults.get(log_type, {}))

    if overrides is not None:
        rv.update(overrides)

    return rv


fmt_options = {
    # <option name>: (<wrapper function>, description, <applicable formatters>, action)
    # "action" is used by the commandline parser in use.
    'verbose': (verbose_wrapper,
                "Enables verbose mode for the given formatter.",
                {"mach"}, "store_true"),
    'compact': (compact_wrapper,
                "Enables compact mode for the given formatter.",
                {"tbpl"}, "store_true"),
    'level': (level_filter_wrapper,
              "A least log level to subscribe to for the given formatter "
              "(debug, info, error, etc.)",
              {"mach", "raw", "tbpl"}, "store"),
    'buffer': (buffer_handler_wrapper,
               "If specified, enables message buffering at the given buffer size limit.",
               ["mach", "tbpl"], "store"),
    'screenshot': (screenshot_wrapper,
                   "Enable logging reftest-analyzer compatible screenshot data.",
                   {"mach"}, "store_true"),
    'no-screenshot': (screenshot_wrapper,
                      "Disable logging reftest-analyzer compatible screenshot data.",
                      {"mach"}, "store_false"),
}


def log_file(name):
    if name == "-":
        return sys.stdout
    # ensure we have a correct dirpath by using realpath
    dirpath = os.path.dirname(os.path.realpath(name))
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)
    return open(name, "w")


def add_logging_group(parser, include_formatters=None):
    """
    Add logging options to an argparse ArgumentParser or
    optparse OptionParser.

    Each formatter has a corresponding option of the form --log-{name}
    where {name} is the name of the formatter. The option takes a value
    which is either a filename or "-" to indicate stdout.

    :param parser: The ArgumentParser or OptionParser object that should have
                   logging options added.
    :param include_formatters: List of formatter names that should be included
                               in the option group. Default to None, meaning
                               all the formatters are included. A common use
                               of this option is to specify
                               :data:`TEXT_FORMATTERS` to include only the
                               most useful formatters for a command line tool
                               that is not related to test harnesses.
    """
    group_name = "Output Logging"
    group_description = ("Each option represents a possible logging format "
                         "and takes a filename to write that format to, "
                         "or '-' to write to stdout. Some options are "
                         "provided by the mozlog utility; see %s "
                         "for extended documentation." % DOCS_URL)

    if include_formatters is None:
        include_formatters = list(log_formatters.keys())

    if isinstance(parser, optparse.OptionParser):
        group = optparse.OptionGroup(parser,
                                     group_name,
                                     group_description)
        parser.add_option_group(group)
        opt_log_type = 'str'
        group_add = group.add_option
    else:
        group = parser.add_argument_group(group_name,
                                          group_description)
        opt_log_type = log_file
        group_add = group.add_argument

    for name, (cls, help_str) in six.iteritems(log_formatters):
        if name in include_formatters:
            group_add("--log-" + name, action="append", type=opt_log_type,
                      help=help_str)

    for fmt in include_formatters:
        for optname, (cls, help_str, formatters_, action) in six.iteritems(fmt_options):
            if fmt not in formatters_:
                continue
            if optname.startswith("no-") and action == "store_false":
                dest = optname.split("-", 1)[1]
            else:
                dest = optname
            dest = dest.replace("-", "_")
            group_add("--log-%s-%s" % (fmt, optname), action=action,
                      help=help_str, default=None, dest="log_%s_%s" % (fmt, dest))


def setup_handlers(logger, formatters, formatter_options, allow_unused_options=False):
    """
    Add handlers to the given logger according to the formatters and
    options provided.

    :param logger: The logger configured by this function.
    :param formatters: A dict of {formatter, [streams]} to use in handlers.
    :param formatter_options: a dict of {formatter: {option: value}} to
                              to use when configuring formatters.
    """
    unused_options = set(formatter_options.keys()) - set(formatters.keys())
    if unused_options and not allow_unused_options:
        msg = ("Options specified for unused formatter(s) (%s) have no effect" %
               list(unused_options))
        raise ValueError(msg)

    for fmt, streams in six.iteritems(formatters):
        formatter_cls = log_formatters[fmt][0]
        formatter = formatter_cls()
        handler_wrappers_and_options = []

        for option, value in six.iteritems(formatter_options[fmt]):
            wrapper, wrapper_args = None, ()
            if option == "valgrind":
                wrapper = valgrind_handler_wrapper
            elif option == "buffer":
                wrapper, wrapper_args = fmt_options[option][0], (value,)
            else:
                formatter = fmt_options[option][0](formatter, value)

            if wrapper is not None:
                handler_wrappers_and_options.append((wrapper, wrapper_args))

        for value in streams:
            handler = handlers.StreamHandler(stream=value, formatter=formatter)
            for wrapper, wrapper_args in handler_wrappers_and_options:
                handler = wrapper(handler, *wrapper_args)
            logger.add_handler(handler)


def setup_logging(logger, args, defaults=None, formatter_defaults=None,
                  allow_unused_options=False):
    """
    Configure a structuredlogger based on command line arguments.

    The created structuredlogger will also be set as the default logger, and
    can be retrieved with :py:func:`~mozlog.get_default_logger`.

    :param logger: A StructuredLogger instance or string name. If a string, a
                   new StructuredLogger instance will be created using
                   `logger` as the name.
    :param args: A dictionary of {argument_name:value} produced from
                 parsing the command line arguments for the application
    :param defaults: A dictionary of {formatter name: output stream} to apply
                     when there is no logging supplied on the command line. If
                     this isn't supplied, reasonable defaults are chosen
                     (coloured mach formatting if stdout is a terminal, or raw
                     logs otherwise).
    :param formatter_defaults: A dictionary of {option_name: default_value} to provide
                               to the formatters in the absence of command line overrides.
    :rtype: StructuredLogger
    """

    if not isinstance(logger, StructuredLogger):
        logger = StructuredLogger(logger)
        # The likely intent when using this function is to get a brand new
        # logger, so reset state in case it was previously initialized.
        logger.reset_state()

    # Keep track of any options passed for formatters.
    formatter_options = {}
    # Keep track of formatters and list of streams specified.
    formatters = defaultdict(list)
    found = False
    found_stdout_logger = False
    if args is None:
        args = {}
    if not isinstance(args, dict):
        args = vars(args)

    if defaults is None:
        if sys.__stdout__.isatty():
            defaults = {"mach": sys.stdout}
        else:
            defaults = {"raw": sys.stdout}

    for name, values in six.iteritems(args):
        parts = name.split('_')
        if len(parts) > 3:
            continue
        # Our args will be ['log', <formatter>]
        #               or ['log', <formatter>, <option>]
        #               or ['valgrind']
        if parts[0] == 'log' and values is not None:
            if len(parts) == 1 or parts[1] not in log_formatters:
                continue
            if len(parts) == 2:
                _, formatter = parts
                for value in values:
                    found = True
                    if isinstance(value, six.string_types):
                        value = log_file(value)
                    if value == sys.stdout:
                        found_stdout_logger = True
                    formatters[formatter].append(value)
            if len(parts) == 3:
                _, formatter, opt = parts
                if formatter not in formatter_options:
                    formatter_options[formatter] = default_formatter_options(formatter,
                                                                             formatter_defaults)
                formatter_options[formatter][opt] = values

    # If there is no user-specified logging, go with the default options
    if not found:
        for name, value in six.iteritems(defaults):
            formatters[name].append(value)

    elif not found_stdout_logger and sys.stdout in list(defaults.values()):
        for name, value in six.iteritems(defaults):
            if value == sys.stdout:
                formatters[name].append(value)

    for name in formatters:
        if name not in formatter_options:
            formatter_options[name] = default_formatter_options(name, formatter_defaults)

    # If the user specified --valgrind, add it as an option for all formatters
    if args.get('valgrind', None) is not None:
        for name in formatters:
            formatter_options[name]['valgrind'] = True
    setup_handlers(logger, formatters, formatter_options, allow_unused_options)
    set_default_logger(logger)

    return logger