DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (dcc6d7a0dc00)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
"""module for tooltool operations"""
import os
import sys

from mozharness.base.errors import PythonErrorList
from mozharness.base.log import ERROR, FATAL
from mozharness.mozilla.proxxy import Proxxy

from mozharness.lib.python.authentication import get_credentials_path

TooltoolErrorList = PythonErrorList + [{
    'substr': 'ERROR - ', 'level': ERROR
}]


TOOLTOOL_PY_URL = \
    "https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py"

TOOLTOOL_SERVERS = [
    'https://api.pub.build.mozilla.org/tooltool/',
]


class TooltoolMixin(object):
    """Mixin class for handling tooltool manifests.
    To use a tooltool server other than the Mozilla server, override
    config['tooltool_servers'].  To specify a different authentication
    file than that used in releng automation,override
    config['tooltool_authentication_file']; set it to None to not pass
    any authentication information (OK for public files)
    """
    def _get_auth_file(self):
        # set the default authentication file based on platform; this
        # corresponds to where puppet puts the token
        if 'tooltool_authentication_file' in self.config:
            fn = self.config['tooltool_authentication_file']
        elif self._is_windows():
            fn = r'c:\builds\relengapi.tok'
        else:
            fn = '/builds/relengapi.tok'

        # if the file doesn't exist, don't pass it to tooltool (it will just
        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
        # take care of auth.  Everywhere else, we'll get auth failures if
        # necessary.
        if os.path.exists(fn):
            return fn

    def tooltool_fetch(self, manifest,
                       output_dir=None, privileged=False, cache=None):
        """docstring for tooltool_fetch"""

        if self.config.get("download_tooltool"):
            cmd = [sys.executable, self._fetch_tooltool_py()]
        else:
            cmd = self.query_exe('tooltool.py', return_type='list')

        # get the tooltool servers from configuration
        default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)

        # add slashes (bug 1155630)
        def add_slash(url):
            return url if url.endswith('/') else (url + '/')
        default_urls = [add_slash(u) for u in default_urls]

        # proxxy-ify
        proxxy = Proxxy(self.config, self.log_obj)
        proxxy_urls = proxxy.get_proxies_and_urls(default_urls)

        for proxyied_url in proxxy_urls:
            cmd.extend(['--url', proxyied_url])

        # handle authentication file, if given
        auth_file = self._get_auth_file()
        if auth_file and os.path.exists(auth_file):
            cmd.extend(['--authentication-file', auth_file])

        cmd.extend(['fetch', '-m', manifest, '-o'])

        if cache:
            cmd.extend(['-c', cache])

        self.retry(
            self.run_command,
            args=(cmd, ),
            kwargs={'cwd': output_dir,
                    'error_list': TooltoolErrorList,
                    'privileged': privileged,
                    },
            good_statuses=(0, ),
            error_message="Tooltool %s fetch failed!" % manifest,
            error_level=FATAL,
        )

    def _fetch_tooltool_py(self):
        """ Retrieve tooltool.py
        """
        dirs = self.query_abs_dirs()
        file_path = os.path.join(dirs['abs_work_dir'], "tooltool.py")
        self.download_file(TOOLTOOL_PY_URL, file_path)
        if not os.path.exists(file_path):
            self.fatal("We can't get tooltool.py")
        self.chmod(file_path, 0755)
        return file_path

    def create_tooltool_manifest(self, contents, path=None):
        """ Currently just creates a manifest, given the contents.
        We may want a template and individual values in the future?
        """
        if path is None:
            dirs = self.query_abs_dirs()
            path = os.path.join(dirs['abs_work_dir'], 'tooltool.tt')
        self.write_to_file(path, contents, error_level=FATAL)
        return path