Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
"""
Parses a JSON file listing the known Certificate Transparency logs
(log_list.json) and generates a C++ header file to be included in Firefox.
The current log_list.json file available under security/manager/tools
was originally downloaded from
and edited to include the disqualification time for the disqualified logs using
"""
import argparse
import base64
import datetime
import json
import os.path
import sys
import textwrap
from string import Template
import six
import urllib3
def decodebytes(s):
if six.PY3:
return base64.decodebytes(six.ensure_binary(s))
return base64.decodestring(s)
OUTPUT_TEMPLATE = """\
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
/* This file was automatically generated by $prog. */
#ifndef $include_guard
#define $include_guard
#include "CTLog.h"
#include <stddef.h>
struct CTLogInfo
{
const char* name;
// Index within kCTLogOperatorList.
mozilla::ct::CTLogStatus status;
// 0 for qualified logs, disqualification time for disqualified logs
// (in milliseconds, measured since the epoch, ignoring leap seconds).
uint64_t disqualificationTime;
size_t operatorIndex;
const char* key;
size_t keyLength;
};
struct CTLogOperatorInfo
{
const char* name;
mozilla::ct::CTLogOperatorId id;
};
const CTLogInfo kCTLogList[] = {
$logs
};
const CTLogOperatorInfo kCTLogOperatorList[] = {
$operators
};
#endif // $include_guard
"""
def get_disqualification_time(time_str):
"""
Convert a time string such as "2017-01-01T00:00:00Z" to an integer
representing milliseconds since the epoch.
Timezones in the string are not supported and will result in an exception.
"""
t = datetime.datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ")
epoch = datetime.datetime.utcfromtimestamp(0)
seconds_since_epoch = (t - epoch).total_seconds()
return int(seconds_since_epoch * 1000)
def get_hex_lines(blob, width):
"""Convert a binary string to a multiline text of C escape sequences."""
text = "".join(["\\x{:02x}".format(c) for c in blob])
# When escaped, a single byte takes 4 chars (e.g. "\x00").
# Make sure we don't break an escaped byte between the lines.
return textwrap.wrap(text, width - width % 4)
def get_operator_index(json_data, target_name):
"""Return operator's entry from the JSON along with its array index."""
matches = [
(operator, index)
for (index, operator) in enumerate(json_data["operators"])
if operator["name"] == target_name
]
assert len(matches) != 0, "No operators with id {0} defined.".format(target_name)
assert len(matches) == 1, "Found multiple operators with id {0}.".format(
target_name
)
return matches[0][1]
def get_log_info_structs(json_data):
"""Return array of CTLogInfo initializers for the known logs."""
tmpl = Template(
textwrap.dedent(
"""\
{ $description,
$status,
$disqualification_time, // $disqualification_time_comment
$operator_index, // $operator_comment
$indented_log_key,
$log_key_len }"""
)
)
initializers = []
for operator in json_data["operators"]:
operator_name = operator["name"]
for log in operator["logs"]:
log_key = decodebytes(log["key"])
operator_index = get_operator_index(json_data, operator_name)
if "disqualification_time" in log:
status = "mozilla::ct::CTLogStatus::Disqualified"
disqualification_time = get_disqualification_time(
log["disqualification_time"]
)
disqualification_time_comment = 'Date.parse("{0}")'.format(
log["disqualification_time"]
)
else:
status = "mozilla::ct::CTLogStatus::Included"
disqualification_time = 0
disqualification_time_comment = "no disqualification time"
is_test_log = "test_only" in operator and operator["test_only"]
prefix = ""
suffix = ","
if is_test_log:
prefix = "#ifdef DEBUG\n"
suffix = ",\n#endif // DEBUG"
toappend = tmpl.substitute(
# Use json.dumps for C-escaping strings.
# Not perfect but close enough.
description=json.dumps(log["description"]),
operator_index=operator_index,
operator_comment="operated by {0}".
# The comment must not contain "/".
format(operator_name).replace("/", "|"),
status=status,
disqualification_time=disqualification_time,
disqualification_time_comment=disqualification_time_comment,
# Maximum line width is 80.
indented_log_key="\n".join(
[' "{0}"'.format(l) for l in get_hex_lines(log_key, 74)]
),
log_key_len=len(log_key),
)
initializers.append(prefix + toappend + suffix)
return initializers
def get_log_operator_structs(json_data):
"""Return array of CTLogOperatorInfo initializers."""
tmpl = Template(" { $name, $id }")
initializers = []
currentId = 0
for operator in json_data["operators"]:
prefix = ""
suffix = ","
is_test_log = "test_only" in operator and operator["test_only"]
if is_test_log:
prefix = "#ifdef DEBUG\n"
suffix = ",\n#endif // DEBUG"
toappend = tmpl.substitute(name=json.dumps(operator["name"]), id=currentId)
currentId += 1
initializers.append(prefix + toappend + suffix)
return initializers
def generate_cpp_header_file(json_data, out_file):
"""Generate the C++ header file for the known logs."""
filename = os.path.basename(out_file.name)
include_guard = filename.replace(".", "_").replace("/", "_")
log_info_initializers = get_log_info_structs(json_data)
operator_info_initializers = get_log_operator_structs(json_data)
out_file.write(
Template(OUTPUT_TEMPLATE).substitute(
prog=os.path.basename(sys.argv[0]),
include_guard=include_guard,
logs="\n".join(log_info_initializers),
operators="\n".join(operator_info_initializers),
)
)
def patch_in_test_logs(json_data):
"""Insert Mozilla-specific test log data."""
max_id = len(json_data["operators"])
mozilla_test_operator_1 = {
"name": "Mozilla Test Org 1",
"id": max_id + 1,
"test_only": True,
"logs": [
{
"description": "Mozilla Test RSA Log 1",
# `openssl x509 -noout -pubkey -in <path/to/default-ee.pem>`
"key": """
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuohRqESOFtZB/W62iAY2
ED08E9nq5DVKtOz1aFdsJHvBxyWo4NgfvbGcBptuGobya+KvWnVramRxCHqlWqdF
h/cc1SScAn7NQ/weadA4ICmTqyDDSeTbuUzCa2wO7RWCD/F+rWkasdMCOosqQe6n
cOAPDY39ZgsrsCSSpH25iGF5kLFXkD3SO8XguEgfqDfTiEPvJxbYVbdmWqp+ApAv
OnsQgAYkzBxsl62WYVu34pYSwHUxowyR3bTK9/ytHSXTCe+5Fw6naOGzey8ib2nj
tIqVYR3uJtYlnauRCE42yxwkBCy/Fosv5fGPmRcxuLP+SSP6clHEMdUDrNoYCjXt
jQIDAQAB
""",
"operated_by": [max_id + 1],
},
{
"description": "Mozilla Test EC Log",
# `openssl x509 -noout -pubkey -in <path/to/root_secp256r1_256.pem`
"key": """
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAET7+7u2Hg+PmxpgpZrIcE4uwFC0I+
PPcukj8sT3lLRVwqadIzRWw2xBGdBwbgDu3I0ZOQ15kbey0HowTqoEqmwA==
""",
"operated_by": [max_id + 1],
},
],
}
mozilla_test_operator_2 = {
"name": "Mozilla Test Org 2",
"id": max_id + 2,
"test_only": True,
"logs": [
{
"description": "Mozilla Test RSA Log 2",
# `openssl x509 -noout -pubkey -in <path/to/other-test-ca.pem>`
"key": """
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXXGUmYJn3cIKmeR8bh2
w39c5TiwbErNIrHL1G+mWtoq3UHIwkmKxKOzwfYUh/QbaYlBvYClHDwSAkTFhKTE
SDMF5ROMAQbPCL6ahidguuai6PNvI8XZgxO53683g0XazlHU1tzSpss8xwbrzTBw
7JjM5AqlkdcpWn9xxb5maR0rLf7ISURZC8Wj6kn9k7HXU0BfF3N2mZWGZiVHl+1C
aQiICBFCIGmYikP+5Izmh4HdIramnNKDdRMfkysSjOKG+n0lHAYq0n7wFvGHzdVO
gys1uJMPdLqQqovHYWckKrH9bWIUDRjEwLjGj8N0hFcyStfehuZVLx0eGR1xIWjT
uwIDAQAB
""",
"operated_by": [max_id + 2],
}
],
}
json_data["operators"].append(mozilla_test_operator_1)
json_data["operators"].append(mozilla_test_operator_2)
def run(args):
"""
Load the input JSON file and generate the C++ header according to the
command line arguments.
"""
if args.file:
print("Reading file: ", args.file)
with open(args.file, "rb") as json_file:
json_text = json_file.read()
elif args.url:
print("Fetching URL: ", args.url)
json_request = urllib3.urlopen(args.url)
try:
json_text = json_request.read()
finally:
json_request.close()
json_data = json.loads(json_text)
print("Writing output: ", args.out)
patch_in_test_logs(json_data)
with open(args.out, "w") as out_file:
generate_cpp_header_file(json_data, out_file)
print("Done.")
def parse_arguments_and_run():
"""Parse the command line arguments and run the program."""
arg_parser = argparse.ArgumentParser(
description="Parses a JSON file listing the known "
"Certificate Transparency logs and generates "
"a C++ header file to be included in Firefox.",
epilog="Example: python %s --url" % os.path.basename(sys.argv[0]),
)
source_group = arg_parser.add_mutually_exclusive_group(required=True)
source_group.add_argument(
"--file",
nargs="?",
const="log_list.json",
help="Read the known CT logs JSON data from the "
"specified local file (%(const)s by default).",
)
source_group.add_argument(
"--url", help="Download the known CT logs JSON file " "from the specified URL."
)
arg_parser.add_argument(
"--out",
default="../../certverifier/CTKnownLogs.h",
help="Path and filename of the header file "
"to be generated. Defaults to %(default)s",
)
run(arg_parser.parse_args())
if __name__ == "__main__":
parse_arguments_and_run()