DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (409f3966645a)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
Parses a JSON file listing the known Certificate Transparency logs
(log_list.json) and generates a C++ header file to be included in Firefox.

The current log_list.json file available under security/manager/tools
was originally downloaded from
https://www.certificate-transparency.org/known-logs
and edited to include the disqualification time for the disqualified logs using
https://cs.chromium.org/chromium/src/net/cert/ct_known_logs_static-inc.h
"""

from __future__ import print_function
from string import Template
import argparse
import base64
import datetime
import json
import os.path
import sys
import textwrap
import urllib2


OUTPUT_TEMPLATE = """\
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/* This file was automatically generated by $prog. */

#ifndef $include_guard
#define $include_guard

#include "CTLog.h"

#include <stddef.h>

struct CTLogInfo
{
  // See bug 1338873 about making these fields const.
  const char* name;
  // Index within kCTLogOperatorList.
  mozilla::ct::CTLogStatus status;
  // 0 for qualified logs, disqualification time for disqualified logs
  // (in milliseconds, measured since the epoch, ignoring leap seconds).
  uint64_t disqualificationTime;
  size_t operatorIndex;
  const char* key;
  size_t keyLength;
};

struct CTLogOperatorInfo
{
  // See bug 1338873 about making these fields const.
  const char* name;
  mozilla::ct::CTLogOperatorId id;
};

const CTLogInfo kCTLogList[] = {
$logs
};

const CTLogOperatorInfo kCTLogOperatorList[] = {
$operators
};

#endif // $include_guard
"""


def get_disqualification_time(time_str):
    """
    Convert a time string such as "2017-01-01T00:00:00Z" to an integer
    representing milliseconds since the epoch.
    Timezones in the string are not supported and will result in an exception.
    """
    t = datetime.datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ")
    epoch = datetime.datetime.utcfromtimestamp(0)
    seconds_since_epoch = (t - epoch).total_seconds()
    return int(seconds_since_epoch * 1000)


def get_hex_lines(blob, width):
    """ Convert a binary string to a multiline text of C escape sequences. """
    text = "".join(["\\x{:02x}".format(ord(c)) for c in blob])
    # When escaped, a single byte takes 4 chars (e.g. "\x00").
    # Make sure we don't break an escaped byte between the lines.
    return textwrap.wrap(text, width - width % 4)


def get_operator_and_index(json_data, operator_id):
    """ Return operator's entry from the JSON along with its array index. """
    matches = [(operator, index) for (index, operator) in enumerate(
        json_data["operators"]) if operator["id"] == operator_id]
    assert len(matches) != 0, "No operators with id {0} defined.".format(
        operator_id)
    assert len(matches) == 1, "Found multiple operators with id {0}.".format(
        operator_id)
    return matches[0]


def get_log_info_structs(json_data):
    """ Return array of CTLogInfo initializers for the known logs. """
    tmpl = Template(textwrap.dedent("""\
          { $description,
            $status,
            $disqualification_time, // $disqualification_time_comment
            $operator_index, // $operator_comment
        $indented_log_key,
            $log_key_len }"""))
    initializers = []
    for log in json_data["logs"]:
        log_key = base64.decodestring(log["key"])
        # "operated_by" is a list, we assume here it always contains one item.
        operated_by = log["operated_by"]
        assert len(operated_by) == 1, "operated_by must contain one item."
        operator, operator_index = get_operator_and_index(json_data,
                                                          operated_by[0])
        if "disqualification_time" in log:
            status = "mozilla::ct::CTLogStatus::Disqualified"
            disqualification_time = get_disqualification_time(
                log["disqualification_time"])
            disqualification_time_comment = 'Date.parse("{0}")'.format(
                log["disqualification_time"])
        else:
            status = "mozilla::ct::CTLogStatus::Included"
            disqualification_time = 0
            disqualification_time_comment = "no disqualification time"
        is_test_log = "test_only" in operator and operator["test_only"]
        prefix = ""
        suffix = ","
        if is_test_log:
            prefix = "#ifdef DEBUG\n"
            suffix = ",\n#endif // DEBUG"
        toappend = tmpl.substitute(
            # Use json.dumps for C-escaping strings.
            # Not perfect but close enough.
            description=json.dumps(log["description"]),
            operator_index=operator_index,
            operator_comment="operated by {0}".
            # The comment must not contain "/".
            format(operator["name"]).replace("/", "|"),
            status=status,
            disqualification_time=disqualification_time,
            disqualification_time_comment=disqualification_time_comment,
            # Maximum line width is 80.
            indented_log_key="\n".
            join(['    "{0}"'.format(l) for l in get_hex_lines(log_key, 74)]),
            log_key_len=len(log_key))
        initializers.append(prefix + toappend + suffix)
    return initializers


def get_log_operator_structs(json_data):
    """ Return array of CTLogOperatorInfo initializers. """
    tmpl = Template("  { $name, $id }")
    initializers = []
    for operator in json_data["operators"]:
        prefix = ""
        suffix = ","
        is_test_log = "test_only" in operator and operator["test_only"]
        if is_test_log:
            prefix = "#ifdef DEBUG\n"
            suffix = ",\n#endif // DEBUG"
        toappend = tmpl.substitute(
            name=json.dumps(operator["name"]),
            id=operator["id"])
        initializers.append(prefix + toappend + suffix)
    return initializers


def generate_cpp_header_file(json_data, out_file):
    """ Generate the C++ header file for the known logs. """
    filename = os.path.basename(out_file.name)
    include_guard = filename.replace(".", "_").replace("/", "_")
    log_info_initializers = get_log_info_structs(json_data)
    operator_info_initializers = get_log_operator_structs(json_data)
    out_file.write(Template(OUTPUT_TEMPLATE).substitute(
        prog=os.path.basename(sys.argv[0]),
        include_guard=include_guard,
        logs="\n".join(log_info_initializers),
        operators="\n".join(operator_info_initializers)))


def patch_in_test_logs(json_data):
    """ Insert Mozilla-specific test log data. """
    max_id = 0
    for operator in json_data["operators"]:
        if operator["id"] > max_id:
            max_id = operator["id"]
    mozilla_test_operator_1 = {"name": "Mozilla Test Org 1", "id": max_id + 1,
                               "test_only": True}
    mozilla_test_operator_2 = {"name": "Mozilla Test Org 2", "id": max_id + 2,
                               "test_only": True}
    json_data["operators"].append(mozilla_test_operator_1)
    json_data["operators"].append(mozilla_test_operator_2)
    # The easiest way to get this is
    # `openssl x509 -noout -pubkey -in <path/to/default-ee.pem>`
    mozilla_rsa_log_1 = {"description": "Mozilla Test RSA Log 1",
                         "key": """
            MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuohRqESOFtZB/W62iAY2
            ED08E9nq5DVKtOz1aFdsJHvBxyWo4NgfvbGcBptuGobya+KvWnVramRxCHqlWqdF
            h/cc1SScAn7NQ/weadA4ICmTqyDDSeTbuUzCa2wO7RWCD/F+rWkasdMCOosqQe6n
            cOAPDY39ZgsrsCSSpH25iGF5kLFXkD3SO8XguEgfqDfTiEPvJxbYVbdmWqp+ApAv
            OnsQgAYkzBxsl62WYVu34pYSwHUxowyR3bTK9/ytHSXTCe+5Fw6naOGzey8ib2nj
            tIqVYR3uJtYlnauRCE42yxwkBCy/Fosv5fGPmRcxuLP+SSP6clHEMdUDrNoYCjXt
            jQIDAQAB
        """,
                         "operated_by": [max_id + 1]}
    # Similarly,
    # `openssl x509 -noout -pubkey -in <path/to/other-test-ca.pem>`
    mozilla_rsa_log_2 = {"description": "Mozilla Test RSA Log 2",
                         "key": """
            MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXXGUmYJn3cIKmeR8bh2
            w39c5TiwbErNIrHL1G+mWtoq3UHIwkmKxKOzwfYUh/QbaYlBvYClHDwSAkTFhKTE
            SDMF5ROMAQbPCL6ahidguuai6PNvI8XZgxO53683g0XazlHU1tzSpss8xwbrzTBw
            7JjM5AqlkdcpWn9xxb5maR0rLf7ISURZC8Wj6kn9k7HXU0BfF3N2mZWGZiVHl+1C
            aQiICBFCIGmYikP+5Izmh4HdIramnNKDdRMfkysSjOKG+n0lHAYq0n7wFvGHzdVO
            gys1uJMPdLqQqovHYWckKrH9bWIUDRjEwLjGj8N0hFcyStfehuZVLx0eGR1xIWjT
            uwIDAQAB
        """,
                         "operated_by": [max_id + 2]}
    # `openssl x509 -noout -pubkey -in <path/to/root_secp256r1_256.pem`
    mozilla_ec_log = {"description": "Mozilla Test EC Log",
                      "key": """
            MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAET7+7u2Hg+PmxpgpZrIcE4uwFC0I+
            PPcukj8sT3lLRVwqadIzRWw2xBGdBwbgDu3I0ZOQ15kbey0HowTqoEqmwA==
        """,
                      "operated_by": [max_id + 1]}
    json_data["logs"].append(mozilla_rsa_log_1)
    json_data["logs"].append(mozilla_rsa_log_2)
    json_data["logs"].append(mozilla_ec_log)


def run(args):
    """
    Load the input JSON file and generate the C++ header according to the
    command line arguments.
    """
    if args.file:
        print("Reading file: ", args.file)
        with open(args.file, "rb") as json_file:
            json_text = json_file.read()
    elif args.url:
        print("Fetching URL: ", args.url)
        json_request = urllib2.urlopen(args.url)
        try:
            json_text = json_request.read()
        finally:
            json_request.close()

    json_data = json.loads(json_text)

    print("Writing output: ", args.out)

    patch_in_test_logs(json_data)

    with open(args.out, "w") as out_file:
        generate_cpp_header_file(json_data, out_file)

    print("Done.")


def parse_arguments_and_run():
    """ Parse the command line arguments and run the program. """
    arg_parser = argparse.ArgumentParser(
        description="Parses a JSON file listing the known "
        "Certificate Transparency logs and generates "
        "a C++ header file to be included in Firefox.",
        epilog="Example: python %s --url" % os.path.basename(sys.argv[0]))

    source_group = arg_parser.add_mutually_exclusive_group(required=True)
    source_group.add_argument("--file", nargs="?",
                              const="log_list.json",
                              help="Read the known CT logs JSON data from the "
                              "specified local file (%(const)s by default).")
    source_group.add_argument("--url",
                              help="Download the known CT logs JSON file "
                              "from the specified URL.")

    arg_parser.add_argument("--out",
                            default="../../certverifier/CTKnownLogs.h",
                            help="Path and filename of the header file "
                            "to be generated. Defaults to %(default)s")

    run(arg_parser.parse_args())


if __name__ == "__main__":
    parse_arguments_and_run()