Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/python3 -u
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
"""
This script downloads the latest chromium build (or a manually
defined version) for a given platform. It then uploads the build,
with the revision of the build stored in a REVISION file.
"""
import argparse
import errno
import os
import shutil
import subprocess
import tempfile
import requests
from redo import retriable
LAST_CHANGE_URL = (
# formatted with platform
"chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media"
)
CHROMIUM_BASE_URL = (
# formatted with (platform/revision/archive)
"chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media"
)
CHROMIUM_INFO = {
"linux": {
"platform": "Linux_x64",
"chromium": "chrome-linux.zip",
"dir": "chrome-linux",
"result": "chromium-linux.tar.bz2",
"chromedriver": "chromedriver_linux64.zip",
},
"win32": {
"platform": "Win",
"chromium": "chrome-win.zip",
"dir": "chrome-win",
"result": "chromium-win32.tar.bz2",
"chromedriver": "chromedriver_win32.zip",
},
"win64": {
"platform": "Win",
"chromium": "chrome-win.zip",
"dir": "chrome-win",
"result": "chromium-win64.tar.bz2",
"chromedriver": "chromedriver_win32.zip",
},
"mac": {
"platform": "Mac",
"chromium": "chrome-mac.zip",
"dir": "chrome-mac",
"result": "chromium-mac.tar.bz2",
"chromedriver": "chromedriver_mac64.zip",
},
"mac-arm": {
"platform": "Mac_Arm",
"chromium": "chrome-mac.zip",
"dir": "chrome-mac",
"result": "chromium-mac-arm.tar.bz2",
"chromedriver": "chromedriver_mac64.zip",
},
}
def log(msg):
print("build-chromium: %s" % msg)
@retriable(attempts=7, sleeptime=5, sleepscale=2)
def fetch_file(url, filepath):
"""Download a file from the given url to a given file."""
size = 4096
r = requests.get(url, stream=True)
r.raise_for_status()
with open(filepath, "wb") as fd:
for chunk in r.iter_content(size):
fd.write(chunk)
def unzip(zippath, target):
"""Unzips an archive to the target location."""
log("Unpacking archive at: %s to: %s" % (zippath, target))
unzip_command = ["unzip", "-q", "-o", zippath, "-d", target]
subprocess.check_call(unzip_command)
@retriable(attempts=7, sleeptime=5, sleepscale=2)
def fetch_chromium_revision(platform):
"""Get the revision of the latest chromium build."""
chromium_platform = CHROMIUM_INFO[platform]["platform"]
revision_url = LAST_CHANGE_URL.format(chromium_platform)
log("Getting revision number for latest %s chromium build..." % chromium_platform)
# Expecting a file with a single number indicating the latest
# chromium build with a chromedriver that we can download
r = requests.get(revision_url, timeout=30)
r.raise_for_status()
chromium_revision = r.content.decode("utf-8")
return chromium_revision.strip()
def fetch_chromedriver(platform, revision, chromium_dir):
"""Get the chromedriver for the given revision and repackage it."""
if not revision:
revision = fetch_chromium_revision(platform)
download_url = CHROMIUM_BASE_URL.format(
CHROMIUM_INFO[platform]["platform"],
revision,
CHROMIUM_INFO[platform]["chromedriver"],
)
tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip")
log("Downloading chromedriver from %s" % download_url)
fetch_file(download_url, tmpzip)
tmppath = tempfile.mkdtemp()
unzip(tmpzip, tmppath)
# Find the chromedriver then copy it to the chromium directory
cd_path = None
for dirpath, _, filenames in os.walk(tmppath):
for filename in filenames:
if filename == "chromedriver" or filename == "chromedriver.exe":
cd_path = os.path.join(dirpath, filename)
break
if cd_path is not None:
break
if cd_path is None:
raise Exception("Could not find chromedriver binary in %s" % tmppath)
log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir))
shutil.copy(cd_path, chromium_dir)
return revision
def build_chromium_archive(platform, revision=None):
"""
Download and store a chromium build for a given platform.
Retrieves either the latest version, or uses a pre-defined version if
the `--revision` option is given a revision.
"""
upload_dir = os.environ.get("UPLOAD_DIR")
if upload_dir:
# Create the upload directory if it doesn't exist.
try:
log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
os.makedirs(upload_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Make a temporary location for the file
tmppath = tempfile.mkdtemp()
# Create the directory format expected for browsertime setup in taskgraph transform
artifact_dir = CHROMIUM_INFO[platform]["dir"]
chromium_dir = os.path.join(tmppath, artifact_dir)
os.mkdir(chromium_dir)
# Store the revision number and chromedriver
revision = fetch_chromedriver(platform, revision, chromium_dir)
revision_file = os.path.join(chromium_dir, ".REVISION")
with open(revision_file, "w+") as f:
f.write(str(revision))
tar_file = CHROMIUM_INFO[platform]["result"]
tar_command = ["tar", "cjf", tar_file, "-C", tmppath, artifact_dir]
log("Added revision to %s file." % revision_file)
log("Tarring with the command: %s" % str(tar_command))
subprocess.check_call(tar_command)
upload_dir = os.environ.get("UPLOAD_DIR")
if upload_dir:
# Move the tarball to the output directory for upload.
log("Moving %s to the upload directory..." % tar_file)
shutil.copy(tar_file, os.path.join(upload_dir, tar_file))
shutil.rmtree(tmppath)
def parse_args():
"""Read command line arguments and return options."""
parser = argparse.ArgumentParser()
parser.add_argument(
"--platform", help="Platform version of chromium to build.", required=True
)
parser.add_argument(
"--revision",
help="Revision of chromium to build to get. "
"(Defaults to the newest chromium build).",
default=None,
)
return parser.parse_args()
if __name__ == "__main__":
args = vars(parse_args())
build_chromium_archive(**args)