DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (9eb7e5b5f54d)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
#!/bin/bash
#
# build.sh [-d dxr-branch] tree1 [ [tree2] ... ]
#
PATH=/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin
umask 0002

usage () {
    echo "Usage: $0 [-d <dxr branch>] tree [ [tree1] ... ]" 1>&2
    exit 1
}

SITE=prod
while getopts ":d:s" opt; do
    case ${opt} in
        s)
            SITE=staging
            ;;
        d)
            DXR_BRANCH=${OPTARG}
            ;;
        *)
            usage
            ;;
    esac
done
shift $((OPTIND-1))
[ $# -eq 0 ] && usage

# -e: exit on cmd fail
# -E: ERR trap inherited
# -f: disable globs
# -x: echo commands
# -o pipfail: failure return code for any part of pipeline
set -eEfx -o pipefail

BUILD="${SITE}-6hour"
DXR_CONFIG="$(dirname $0)/dxr-config/dxr-${SITE}.config"

trap "echo \"Setup of ${BUILD} failed, check logs\" \
    | /bin/mailx -s \"dxr ${BUILD} failed\" klibby@mozilla.com,erik@mozilla.com" ERR

DXR_REPO="https://github.com/mozilla/dxr.git"
HG_ROOT="https://hg.mozilla.org/"

## This the path to the root of all building
ROOTDIR="/data/builds"

BUILDDIR="/targetdata/dxr-build-env"

MOCK_DIR="$(dirname $0)/mock_mozilla"
## mock build dependencies should be in mock_mozilla/$BUILD.cfg
if [ ! -f "${MOCK_DIR}/${BUILD}.cfg" ]; then
    echo "Missing config file: ${MOCK_DIR}/${BUILD}.cfg"
    exit 2
fi

## These are options to pass to all invocations of mock_mozilla
MOCK_OPTS="-q --configdir=${MOCK_DIR} -r ${BUILD}"

## Initialize/update the chroot
if [ -d "${ROOTDIR}/mock_mozilla/${BUILD}" ]; then
    echo "Updating up mock environment"
    mock_mozilla ${MOCK_OPTS} --update
else
    echo "Setting up mock environment"
    mock_mozilla ${MOCK_OPTS} --init
    # TODO: npm runs into an issue install phantomjs
    # to fix by hand after a re-init, do
    # cp -p /data/builds/phantomjs-1.9.7-linux-x86_64.tar.bz2 \
    #     /data/builds/mock_mozilla/prod-6hour/root/tmp/phantomjs/phantomjs-1.9.7-linux-x86_64.tar.bz2
fi

## Fix permissions on the directory that is bound to the chroot
mock_mozilla ${MOCK_OPTS} --chroot "chown -R mock_mozilla:mock_mozilla /targetdata"

## create mock's hardcoded HOME so npm works
mock_mozilla ${MOCK_OPTS} --chroot "chown -R mock_mozilla:mock_mozilla /builds"

## Setup DXR
echo -e "\nSetting up dxr build env"
# absolute path of builddir outside of chroot
cd ${ROOTDIR}/mock_mozilla/${BUILD}/targetdata
rm -rf dxr-build-env; mkdir dxr-build-env

# local copies since we don't have access to
# http://runtime-binaries.pvt.build.mozilla.org/tooltool
tar xf ${ROOTDIR}/clang-3.3.tar.bz2 -C dxr-build-env

# set path to use clang from tarball
MOCK_PATH=/usr/local/bin:${BUILDDIR}/clang/bin:${PATH}

echo -e "\nCloning DXR and trilite"
cd dxr-build-env

# --recursive also pulls trilite
git clone -q --recursive${DXR_BRANCH:+ -b $DXR_BRANCH} $DXR_REPO dxr

# dxr/format contains the instance version number
VER=0
if [ -f dxr/format ]; then
    VER=$(grep -E "^([a-zA-A0-9_-])+$" dxr/format)
fi

## This is the path to the index location on NFS
DXR_INDEX="/data/www_${SITE}/instances/${VER}"

cp -f ${DXR_CONFIG} dxr.config
TARGET=$(awk -F= '/^target_folder/ {print $2}' dxr.config)
[ -z "${TARGET}" ] && (echo "missing target, source or object definition"; exit 1)
[ -d "${TARGET}" ] || mkdir -p ${TARGET}

# Build DXR binary plugins
mock_mozilla ${MOCK_OPTS} --cwd=${BUILDDIR} --chroot --unpriv "/bin/env \
    PATH=${MOCK_PATH} CC=clang CXX=clang++ make -C dxr"

# Build a virtualenv for dxr
mock_mozilla ${MOCK_OPTS} --cwd=${BUILDDIR} --chroot --unpriv \
    "/usr/local/bin/virtualenv dxr-venv"
mock_mozilla ${MOCK_OPTS} --cwd=${BUILDDIR}/dxr --chroot --unpriv \
    "../dxr-venv/bin/pip install -r requirements.txt"
mock_mozilla ${MOCK_OPTS} --cwd=${BUILDDIR}/dxr --chroot --unpriv \
    "../dxr-venv/bin/python setup.py install"

for TREE in $@
do

    trap "echo \"build of ${TREE} failed, check logs\" \
        | /bin/mailx -s \"dxr ${BUILD} failed\" klibby@mozilla.com,erik@mozilla.com" ERR

    HG_REPO="${HG_ROOT}/${TREE}"

    date
    echo "Starting DXR${DXR_BRANCH:+ ($DXR_BRANCH)} build of ${TREE}"

    # Bug 1052119 - use path:repo in lieu of path/repo
    if [[ ${TREE} =~ / ]]; then
        TREE=$(echo ${TREE} | tr '/' ':')
    fi

    SOURCE=$(perl -MConfig::Tiny -e '$C=Config::Tiny->read("dxr.config"); print $C->{"'$TREE'"}->{"source_folder"};')
    OBJECT=$(perl -MConfig::Tiny -e '$C=Config::Tiny->read("dxr.config"); print $C->{"'$TREE'"}->{"object_folder"};')

    [ -z "${SOURCE}" -o -z "${OBJECT}" ] \
        && (echo "missing target, source or object definition"; exit 1)
    [ -d "${SOURCE}" ] || mkdir -p ${SOURCE}
    [ -d "${OBJECT}" ] || mkdir -p ${OBJECT}

    echo -e "\nCloning ${TREE}"
    # pull the repo
    if [ -d ${SOURCE}/.hg ]; then
        hg pull -u -R ${SOURCE}
    else
        hg clone ${HG_REPO} ${SOURCE}
    fi

    if [ "${TREE}" = 'comm-central' ]; then
        echo "\nDownloading comm-central modules"
        pushd ${SOURCE}
        #python client.py checkout --comm-rev 8f57b0340428 --mozilla-rev 85135c5c6ba8
        python client.py checkout
        popd
    fi

    echo -e "\nConfiguring build"
    if [ "${TREE}" = 'mozilla-central' ]; then
        echo "ac_add_options --enable-debug"              >  ${SOURCE}/.mozconfig
        echo "ac_add_options --enable-stdcxx-compat"     >> ${SOURCE}/.mozconfig
    elif [ "${TREE}" = 'comm-central' ]; then
        echo "ac_add_options --enable-debug"              >  ${SOURCE}/.mozconfig
        echo "ac_add_options --enable-application=suite" >> ${SOURCE}/.mozconfig
        echo "ac_add_options --disable-optimize"         >> ${SOURCE}/.mozconfig
        echo "ac_add_options --disable-stdcxx-compat"    >> ${SOURCE}/.mozconfig
        echo "ac_add_options --disable-elf-hack"         >> ${SOURCE}/.mozconfig
        echo "mk_add_options MOZ_OBJDIR=${BUILDDIR}/${OBJECT}" >> ${SOURCE}/.mozconfig
    fi

    echo -e "\nStarting build"
    # lib path needs setting until libtrilite is packaged and installed by mock
    mock_mozilla ${MOCK_OPTS} --cwd=${BUILDDIR} --chroot --unpriv \
        "/bin/env PATH=${MOCK_PATH} LD_LIBRARY_PATH=${BUILDDIR}/dxr/trilite \
            dxr-venv/bin/python dxr/bin/dxr-build.py -j6 -f dxr.config -t ${TREE} -v" 2>&1

done

find ${TARGET} -type d -exec chmod o+rx {} \;
find ${TARGET} -type f -exec chmod o+r {} \;

if [ -d "${DXR_INDEX}" ]; then
    # use symlink && mv -T to make swap atomic; choose between two dirs and swap link
    LINK=$(readlink ${DXR_INDEX}/target)
    if [ "${LINK}" = "www.a" ]; then
        rm -rf ${DXR_INDEX}/www.b
        mkdir -p ${DXR_INDEX}/www.b
        cd ${TARGET}; tar cpf - . | (cd ${DXR_INDEX}/www.b; tar xpf -)
        cd ${DXR_INDEX}
        ln -s www.b target.new
        mv -T target.new target
        rm -rf www.a
    else
        rm -rf ${DXR_INDEX}/www.a
        mkdir -p ${DXR_INDEX}/www.a
        cd ${TARGET}; tar cpf - . | (cd ${DXR_INDEX}/www.a; tar xpf -)
        cd ${DXR_INDEX}
        ln -s www.a target.new
        mv -T target.new target
        rm -rf www.b
    fi
else
    # must be a new format; special care must be taken:
    #   creating a new instance can create a race condition between copying
    #   the index in and the webheads deploying a new version (dependant on
    #   the not-yet copied new index version)
    mkdir -p ${DXR_INDEX}.new/www.a
    cd ${TARGET}; tar cpf - . | (cd ${DXR_INDEX}.new/www.a; tar xpf -)
    pushd ${DXR_INDEX}.new
    ln -s www.a target
    popd
    mv ${DXR_INDEX}.new ${DXR_INDEX}
fi

echo -e "\nFinished."
date