Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import sys
sys.path.append("/builds/worker/checkouts/gecko/third_party/python")
sys.path.append(".")
import base64
import os
import platform
import signal
import stat
import subprocess
import requests
import taskcluster
# Bump this number when you need to cause a commit for the job to re-run: 21
if len(sys.argv) < 3:
print("Usage:", sys.argv[0], "gecko-dev-path updatebot-path [moz-fetches-dir]")
sys.exit(1)
GECKO_DEV_PATH = sys.argv[1].replace("/", os.path.sep)
UPDATEBOT_PATH = sys.argv[2].replace("/", os.path.sep)
# Only needed on Windows
if len(sys.argv) > 3:
FETCHES_PATH = sys.argv[3].replace("/", os.path.sep)
else:
FETCHES_PATH = None
HOME_PATH = os.path.expanduser("~")
OPERATING_MODE = (
"prod"
if os.environ.get("GECKO_HEAD_REPOSITORY", "")
else "dev"
)
phabricator_url = DEV_PHAB_URL if OPERATING_MODE == "dev" else PROD_PHAB_URL
def log(*args):
print(*args)
def get_secret(name):
secret = None
if "TASK_ID" in os.environ:
secrets_url = (
+ ("3" if OPERATING_MODE == "prod" else "2")
+ "/"
+ name
)
res = requests.get(secrets_url)
res.raise_for_status()
secret = res.json()
else:
secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" + name)
secret = secret["secret"] if "secret" in secret else None
secret = secret["value"] if "value" in secret else None
return secret
# Get TC Secrets =======================================
log("Operating mode is ", OPERATING_MODE)
log("Getting secrets...")
bugzilla_api_key = get_secret("bugzilla-api-key")
phabricator_token = get_secret("phabricator-token")
try_sshkey = get_secret("try-sshkey")
database_config = get_secret("database-password")
sentry_url = get_secret("sentry-url")
sql_proxy_config = get_secret("sql-proxy-config")
# Update Updatebot =======================================
if OPERATING_MODE == "dev":
"""
If we are in development mode, we will update from github.
(This command will probably only work if we checkout a branch FWIW.)
This allows us to iterate faster by committing to github and
re-running the cron job on Taskcluster, without rebuilding the
Docker image.
However, this mechanism is bypassing the security feature we
have in-tree, where upstream out-of-tree code is fixed at a known
revision and cannot be changed without a commit to m-c.
Therefore, we only do this in dev mode when running on try.
"""
os.chdir(UPDATEBOT_PATH)
log("Performing git repo update...")
command = ["git", "symbolic-ref", "-q", "HEAD"]
r = subprocess.run(command)
if r.returncode == 0:
# This indicates we are on a branch, and not a specific revision
subprocess.check_call(["git", "pull", "origin"])
# Set Up SSH & Phabricator ==============================
os.chdir(HOME_PATH)
log("Setting up ssh and phab keys...")
with open("id_rsa", "w") as sshkey:
sshkey.write(try_sshkey)
os.chmod("id_rsa", stat.S_IRUSR | stat.S_IWUSR)
arc_filename = ".arcrc"
if platform.system() == "Windows":
arc_path = os.path.join(FETCHES_PATH, "..", "AppData", "Roaming")
os.makedirs(arc_path, exist_ok=True)
os.chdir(arc_path)
log("Writing %s to %s" % (arc_filename, arc_path))
else:
os.chdir(HOME_PATH)
arcrc = open(arc_filename, "w")
towrite = """
{
"hosts": {
"PHAB_URL_HERE": {
"token": "TOKENHERE"
}
}
}
""".replace(
"TOKENHERE", phabricator_token
).replace(
"PHAB_URL_HERE", phabricator_url + "api/"
)
arcrc.write(towrite)
arcrc.close()
os.chmod(arc_filename, stat.S_IRUSR | stat.S_IWUSR)
# Set up the Cloud SQL Proxy =============================
os.chdir(HOME_PATH)
log("Setting up cloud_sql_proxy...")
with open("sql-proxy-key", "w") as proxy_key_file:
proxy_key_file.write(
base64.b64decode(sql_proxy_config["key-value"]).decode("utf-8")
)
instance_name = sql_proxy_config["instance-name"]
if platform.system() == "Linux":
sql_proxy_command = "cloud_sql_proxy"
else:
sql_proxy_command = os.path.join(UPDATEBOT_PATH, "..", "cloud_sql_proxy.exe")
sql_proxy_command += (
" -instances=" + instance_name + "=tcp:3306 -credential_file=sql-proxy-key"
)
sql_proxy_args = {
"stdout": subprocess.PIPE,
"stderr": subprocess.PIPE,
"shell": True,
"start_new_session": True,
}
if platform.system() == "Windows":
si = subprocess.STARTUPINFO()
si.dwFlags = subprocess.CREATE_NEW_PROCESS_GROUP
sql_proxy_args["startupinfo"] = si
sql_proxy = subprocess.Popen((sql_proxy_command), **sql_proxy_args)
try:
(stdout, stderr) = sql_proxy.communicate(input=None, timeout=2)
log("sql proxy stdout:", stdout.decode("utf-8"))
log("sql proxy stderr:", stderr.decode("utf-8"))
except subprocess.TimeoutExpired:
log("no sqlproxy output in 2 seconds, this means it probably didn't error.")
log("sqlproxy pid:", sql_proxy.pid)
database_config["host"] = "127.0.0.1"
# Vendor =================================================
log("Getting Updatebot ready...")
os.chdir(UPDATEBOT_PATH)
localconfig = {
"General": {
"env": OPERATING_MODE,
"gecko-path": GECKO_DEV_PATH,
"soft_timeout": 3600,
},
"Logging": {
"local": True,
"sentry": True,
"sentry_config": {"url": sentry_url, "debug": False},
},
"Database": database_config,
"Bugzilla": {
"apikey": bugzilla_api_key,
},
"Taskcluster": {
},
}
log("Writing local config file")
config = open("localconfig.py", "w")
config.write("localconfig = " + str(localconfig))
config.close()
log("Running updatebot")
# On Windows, Updatebot is run by windows-setup.sh
if platform.system() == "Linux":
subprocess.check_call(["python3", "-m", "poetry", "run", "./automation.py"])
# Clean up ===============================================
log("Killing cloud_sql_proxy")
os.kill(sql_proxy.pid, signal.SIGTERM)