DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (27a812186ff4)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import mozcrash
import threading
import os
import Queue
import re
import shutil
import tempfile
import time
import traceback

from automation import Automation
from devicemanager import NetworkTools
from mozprocess import ProcessHandlerMixin


class StdOutProc(ProcessHandlerMixin):
    """Process handler for b2g which puts all output in a Queue.
    """

    def __init__(self, cmd, queue, **kwargs):
        self.queue = queue
        kwargs.setdefault('processOutputLine', []).append(self.handle_output)
        ProcessHandlerMixin.__init__(self, cmd, **kwargs)

    def handle_output(self, line):
        self.queue.put_nowait(line)


class B2GRemoteAutomation(Automation):
    _devicemanager = None

    def __init__(self, deviceManager, appName='', remoteLog=None,
                 marionette=None, context_chrome=True):
        self._devicemanager = deviceManager
        self._appName = appName
        self._remoteProfile = None
        self._remoteLog = remoteLog
        self.marionette = marionette
        self.context_chrome = context_chrome
        self._is_emulator = False
        self.test_script = None
        self.test_script_args = None

        # Default our product to b2g
        self._product = "b2g"
        self.lastTestSeen = "b2gautomation.py"
        # Default log finish to mochitest standard
        self.logFinish = 'INFO SimpleTest FINISHED'
        Automation.__init__(self)

    def setEmulator(self, is_emulator):
        self._is_emulator = is_emulator

    def setDeviceManager(self, deviceManager):
        self._devicemanager = deviceManager

    def setAppName(self, appName):
        self._appName = appName

    def setRemoteProfile(self, remoteProfile):
        self._remoteProfile = remoteProfile

    def setProduct(self, product):
        self._product = product

    def setRemoteLog(self, logfile):
        self._remoteLog = logfile

    def installExtension(self, extensionSource, profileDir, extensionID=None):
        # Bug 827504 - installing special-powers extension separately causes problems in B2G
        if extensionID != "special-powers@mozilla.org":
            Automation.installExtension(self, extensionSource, profileDir, extensionID)

    # Set up what we need for the remote environment
    def environment(self, env=None, xrePath=None, crashreporter=True):
        # Because we are running remote, we don't want to mimic the local env
        # so no copying of os.environ
        if env is None:
            env = {}

        if crashreporter:
            env['MOZ_CRASHREPORTER'] = '1'
            env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'

        # We always hide the results table in B2G; it's much slower if we don't.
        env['MOZ_HIDE_RESULTS_TABLE'] = '1'
        return env

    def waitForNet(self):
        active = False
        time_out = 0
        while not active and time_out < 40:
            data = self._devicemanager._runCmd(['shell', '/system/bin/netcfg']).stdout.readlines()
            data.pop(0)
            for line in data:
                if (re.search(r'UP\s+(?:[0-9]{1,3}\.){3}[0-9]{1,3}', line)):
                    active = True
                    break
            time_out += 1
            time.sleep(1)
        return active

    def checkForCrashes(self, directory, symbolsPath):
        crashed = False
        remote_dump_dir = self._remoteProfile + '/minidumps'
        print "checking for crashes in '%s'" % remote_dump_dir
        if self._devicemanager.dirExists(remote_dump_dir):
            local_dump_dir = tempfile.mkdtemp()
            self._devicemanager.getDirectory(remote_dump_dir, local_dump_dir)
            try:
                crashed = mozcrash.check_for_crashes(local_dump_dir, symbolsPath, test_name=self.lastTestSeen)
            except:
                traceback.print_exc()
            finally:
                shutil.rmtree(local_dump_dir)
                self._devicemanager.removeDir(remote_dump_dir)
        return crashed

    def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
        # if remote profile is specified, use that instead
        if (self._remoteProfile):
            profileDir = self._remoteProfile

        cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)

        return app, args

    def getLanIp(self):
        nettools = NetworkTools()
        return nettools.getLanIp()

    def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime,
                      debuggerInfo, symbolsPath):
        """ Wait for tests to finish (as evidenced by a signature string
            in logcat), or for a given amount of time to elapse with no
            output.
        """
        timeout = timeout or 120
        while True:
            currentlog = proc.getStdoutLines(timeout)
            if currentlog:
                print currentlog
                # Match the test filepath from the last TEST-START line found in the new
                # log content. These lines are in the form:
                # ... INFO TEST-START | /filepath/we/wish/to/capture.html\n
                testStartFilenames = re.findall(r"TEST-START \| ([^\s]*)", currentlog)
                if testStartFilenames:
                    self.lastTestSeen = testStartFilenames[-1]
                if hasattr(self, 'logFinish') and self.logFinish in currentlog:
                    return 0
            else:
                self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed "
                              "out after %d seconds with no output",
                              self.lastTestSeen, int(timeout))
                return 1

    def getDeviceStatus(self, serial=None):
        # Get the current status of the device.  If we know the device
        # serial number, we look for that, otherwise we use the (presumably
        # only) device shown in 'adb devices'.
        serial = serial or self._devicemanager._deviceSerial
        status = 'unknown'

        for line in self._devicemanager._runCmd(['devices']).stdout.readlines():
            result = re.match('(.*?)\t(.*)', line)
            if result:
                thisSerial = result.group(1)
                if not serial or thisSerial == serial:
                    serial = thisSerial
                    status = result.group(2)

        return (serial, status)

    def restartB2G(self):
        # TODO hangs in subprocess.Popen without this delay
        time.sleep(5)
        self._devicemanager._checkCmd(['shell', 'stop', 'b2g'])
        # Wait for a bit to make sure B2G has completely shut down.
        time.sleep(10)
        self._devicemanager._checkCmd(['shell', 'start', 'b2g'])
        if self._is_emulator:
            self.marionette.emulator.wait_for_port()

    def rebootDevice(self):
        # find device's current status and serial number
        serial, status = self.getDeviceStatus()

        # reboot!
        self._devicemanager._runCmd(['shell', '/system/bin/reboot'])

        # The above command can return while adb still thinks the device is
        # connected, so wait a little bit for it to disconnect from adb.
        time.sleep(10)

        # wait for device to come back to previous status
        print 'waiting for device to come back online after reboot'
        start = time.time()
        rserial, rstatus = self.getDeviceStatus(serial)
        while rstatus != 'device':
            if time.time() - start > 120:
                # device hasn't come back online in 2 minutes, something's wrong
                raise Exception("Device %s (status: %s) not back online after reboot" % (serial, rstatus))
            time.sleep(5)
            rserial, rstatus = self.getDeviceStatus(serial)
        print 'device:', serial, 'status:', rstatus

    def Process(self, cmd, stdout=None, stderr=None, env=None, cwd=None):
        # On a desktop or fennec run, the Process method invokes a gecko
        # process in which to the tests.  For B2G, we simply
        # reboot the device (which was configured with a test profile
        # already), wait for B2G to start up, and then navigate to the
        # test url using Marionette.  There doesn't seem to be any way
        # to pass env variables into the B2G process, but this doesn't
        # seem to matter.

        # reboot device so it starts up with the mochitest profile
        # XXX:  We could potentially use 'stop b2g' + 'start b2g' to achieve
        # a similar effect; will see which is more stable while attempting
        # to bring up the continuous integration.
        if not self._is_emulator:
            self.rebootDevice()
            time.sleep(5)
            #wait for wlan to come up
            if not self.waitForNet():
                raise Exception("network did not come up, please configure the network" +
                                " prior to running before running the automation framework")

        # stop b2g
        self._devicemanager._runCmd(['shell', 'stop', 'b2g'])
        time.sleep(5)

        # relaunch b2g inside b2g instance
        instance = self.B2GInstance(self._devicemanager, env=env)

        time.sleep(5)

        # Set up port forwarding again for Marionette, since any that
        # existed previously got wiped out by the reboot.
        if not self._is_emulator:
            self._devicemanager._checkCmd(['forward',
                                           'tcp:%s' % self.marionette.port,
                                           'tcp:%s' % self.marionette.port])

        if self._is_emulator:
            self.marionette.emulator.wait_for_port()
        else:
            time.sleep(5)

        # start a marionette session
        session = self.marionette.start_session()
        if 'b2g' not in session:
            raise Exception("bad session value %s returned by start_session" % session)

        if self._is_emulator:
            # Disable offline status management (bug 777145), otherwise the network
            # will be 'offline' when the mochitests start.  Presumably, the network
            # won't be offline on a real device, so we only do this for emulators.
            self.marionette.set_context(self.marionette.CONTEXT_CHROME)
            self.marionette.execute_script("""
                Components.utils.import("resource://gre/modules/Services.jsm");
                Services.io.manageOfflineStatus = false;
                Services.io.offline = false;
                """)

        if self.context_chrome:
            self.marionette.set_context(self.marionette.CONTEXT_CHROME)
        else:
            self.marionette.set_context(self.marionette.CONTEXT_CONTENT)

        # run the script that starts the tests
        if self.test_script:
            if os.path.isfile(self.test_script):
                script = open(self.test_script, 'r')
                self.marionette.execute_script(script.read(), script_args=self.test_script_args)
                script.close()
            elif isinstance(self.test_script, basestring):
                self.marionette.execute_script(self.test_script, script_args=self.test_script_args)
        else:
            # assumes the tests are started on startup automatically
            pass

        return instance

    # be careful here as this inner class doesn't have access to outer class members
    class B2GInstance(object):
        """Represents a B2G instance running on a device, and exposes
           some process-like methods/properties that are expected by the
           automation.
        """

        def __init__(self, dm, env=None):
            self.dm = dm
            self.env = env or {}
            self.stdout_proc = None
            self.queue = Queue.Queue()

            # Launch b2g in a separate thread, and dump all output lines
            # into a queue.  The lines in this queue are
            # retrieved and returned by accessing the stdout property of
            # this class.
            cmd = [self.dm._adbPath]
            if self.dm._deviceSerial:
                cmd.extend(['-s', self.dm._deviceSerial])
            cmd.append('shell')
            for k, v in self.env.iteritems():
                cmd.append("%s=%s" % (k, v))
            cmd.append('/system/bin/b2g.sh')
            proc = threading.Thread(target=self._save_stdout_proc, args=(cmd, self.queue))
            proc.daemon = True
            proc.start()

        def _save_stdout_proc(self, cmd, queue):
            self.stdout_proc = StdOutProc(cmd, queue)
            self.stdout_proc.run()
            if hasattr(self.stdout_proc, 'processOutput'):
                self.stdout_proc.processOutput()
            self.stdout_proc.wait()
            self.stdout_proc = None

        @property
        def pid(self):
            # a dummy value to make the automation happy
            return 0

        def getStdoutLines(self, timeout):
            # Return any lines in the queue used by the
            # b2g process handler.
            lines = []
            # get all of the lines that are currently available
            while True:
                try:
                    lines.append(self.queue.get_nowait())
                except Queue.Empty:
                    break

            # wait 'timeout' for any additional lines
            try:
                lines.append(self.queue.get(True, timeout))
            except Queue.Empty:
                pass
            return '\n'.join(lines)

        def wait(self, timeout=None):
            # this should never happen
            raise Exception("'wait' called on B2GInstance")

        def kill(self):
            # this should never happen
            raise Exception("'kill' called on B2GInstance")