DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (4a74609752d2)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

var EXPORTED_SYMBOLS = [ "BookmarkJSONUtils" ];

const {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm");
const {OS} = ChromeUtils.import("resource://gre/modules/osfile.jsm");
const {PlacesUtils} = ChromeUtils.import("resource://gre/modules/PlacesUtils.jsm");

Cu.importGlobalProperties(["fetch"]);

ChromeUtils.defineModuleGetter(this, "PlacesBackups",
  "resource://gre/modules/PlacesBackups.jsm");

// This is used to translate old folder pseudonyms in queries with their newer
// guids.
const OLD_BOOKMARK_QUERY_TRANSLATIONS = {
  "PLACES_ROOT": PlacesUtils.bookmarks.rootGuid,
  "BOOKMARKS_MENU": PlacesUtils.bookmarks.menuGuid,
  "TAGS": PlacesUtils.bookmarks.tagsGuid,
  "UNFILED_BOOKMARKS": PlacesUtils.bookmarks.unfiledGuid,
  "TOOLBAR": PlacesUtils.bookmarks.toolbarGuid,
  "MOBILE_BOOKMARKS": PlacesUtils.bookmarks.mobileGuid,
};

/**
 * Generates an hash for the given string.
 *
 * @note The generated hash is returned in base64 form.  Mind the fact base64
 * is case-sensitive if you are going to reuse this code.
 */
function generateHash(aString) {
  let cryptoHash = Cc["@mozilla.org/security/hash;1"]
                     .createInstance(Ci.nsICryptoHash);
  cryptoHash.init(Ci.nsICryptoHash.MD5);
  let stringStream = Cc["@mozilla.org/io/string-input-stream;1"]
                       .createInstance(Ci.nsIStringInputStream);
  stringStream.data = aString;
  cryptoHash.updateFromStream(stringStream, -1);
  // base64 allows the '/' char, but we can't use it for filenames.
  return cryptoHash.finish(true).replace(/\//g, "-");
}

var BookmarkJSONUtils = Object.freeze({
  /**
   * Import bookmarks from a url.
   *
   * @param {string} aSpec
   *        url of the bookmark data.
   * @param {boolean} [options.replace]
   *        Whether we should erase existing bookmarks before importing.
   * @param {PlacesUtils.bookmarks.SOURCES} [options.source]
   *        The bookmark change source, used to determine the sync status for
   *        imported bookmarks. Defaults to `RESTORE` if `replace = true`, or
   *        `IMPORT` otherwise.
   *
   * @return {Promise}
   * @resolves When the new bookmarks have been created.
   * @rejects JavaScript exception.
   */
  async importFromURL(aSpec, {
    replace: aReplace = false,
    source: aSource = aReplace ? PlacesUtils.bookmarks.SOURCES.RESTORE :
                                 PlacesUtils.bookmarks.SOURCES.IMPORT,
  } = {}) {
    notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aReplace);
    try {
      let importer = new BookmarkImporter(aReplace, aSource);
      await importer.importFromURL(aSpec);

      notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, aReplace);
    } catch (ex) {
      Cu.reportError("Failed to restore bookmarks from " + aSpec + ": " + ex);
      notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, aReplace);
      throw ex;
    }
  },

  /**
   * Restores bookmarks and tags from a JSON file.
   *
   * @param aFilePath
   *        OS.File path string of bookmarks in JSON or JSONlz4 format to be restored.
   * @param [options.replace]
   *        Whether we should erase existing bookmarks before importing.
   * @param [options.source]
   *        The bookmark change source, used to determine the sync status for
   *        imported bookmarks. Defaults to `RESTORE` if `replace = true`, or
   *        `IMPORT` otherwise.
   *
   * @return {Promise}
   * @resolves When the new bookmarks have been created.
   * @rejects JavaScript exception.
   */
  async importFromFile(aFilePath, {
    replace: aReplace = false,
    source: aSource = aReplace ? PlacesUtils.bookmarks.SOURCES.RESTORE :
                                 PlacesUtils.bookmarks.SOURCES.IMPORT,
  } = {}) {
    notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aReplace);
    try {
      if (!(await OS.File.exists(aFilePath)))
        throw new Error("Cannot restore from nonexisting json file");

      let importer = new BookmarkImporter(aReplace, aSource);
      if (aFilePath.endsWith("jsonlz4")) {
        await importer.importFromCompressedFile(aFilePath);
      } else {
        await importer.importFromURL(OS.Path.toFileURI(aFilePath));
      }
      notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, aReplace);
    } catch (ex) {
      Cu.reportError("Failed to restore bookmarks from " + aFilePath + ": " + ex);
      notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, aReplace);
      throw ex;
    }
  },

  /**
   * Serializes bookmarks using JSON, and writes to the supplied file path.
   *
   * @param aFilePath
   *        OS.File path string for the bookmarks file to be created.
   * @param [optional] aOptions
   *        Object containing options for the export:
   *         - failIfHashIs: if the generated file would have the same hash
   *                         defined here, will reject with ex.becauseSameHash
   *         - compress: if true, writes file using lz4 compression
   * @return {Promise}
   * @resolves once the file has been created, to an object with the
   *           following properties:
   *            - count: number of exported bookmarks
   *            - hash: file hash for contents comparison
   * @rejects JavaScript exception.
   */
  async exportToFile(aFilePath, aOptions = {}) {
    let [bookmarks, count] = await PlacesBackups.getBookmarksTree();
    let startTime = Date.now();
    let jsonString = JSON.stringify(bookmarks);
    // Report the time taken to convert the tree to JSON.
    try {
      Services.telemetry
              .getHistogramById("PLACES_BACKUPS_TOJSON_MS")
              .add(Date.now() - startTime);
    } catch (ex) {
      Cu.reportError("Unable to report telemetry.");
    }

    let hash = generateHash(jsonString);

    if (hash === aOptions.failIfHashIs) {
      let e = new Error("Hash conflict");
      e.becauseSameHash = true;
      throw e;
    }

    // Do not write to the tmp folder, otherwise if it has a different
    // filesystem writeAtomic will fail.  Eventual dangling .tmp files should
    // be cleaned up by the caller.
    let writeOptions = { tmpPath: OS.Path.join(aFilePath + ".tmp") };
    if (aOptions.compress)
      writeOptions.compression = "lz4";

    await OS.File.writeAtomic(aFilePath, jsonString, writeOptions);
    return { count, hash };
  },
});

function BookmarkImporter(aReplace, aSource) {
  this._replace = aReplace;
  this._source = aSource;
}
BookmarkImporter.prototype = {
  /**
   * Import bookmarks from a url.
   *
   * @param {string} aSpec
   *        url of the bookmark data.
   *
   * @return {Promise}
   * @resolves When the new bookmarks have been created.
   * @rejects JavaScript exception.
   */
  async importFromURL(spec) {
    if (!spec.startsWith("chrome://") &&
        !spec.startsWith("file://")) {
      throw new Error("importFromURL can only be used with chrome:// and file:// URLs");
    }
    let nodes = await (await fetch(spec)).json();

    if (!nodes.children || !nodes.children.length) {
      return;
    }

    await this.import(nodes);
  },

  /**
   * Import bookmarks from a compressed file.
   *
   * @param aFilePath
   *        OS.File path string of the bookmark data.
   *
   * @return {Promise}
   * @resolves When the new bookmarks have been created.
   * @rejects JavaScript exception.
   */
  importFromCompressedFile: async function BI_importFromCompressedFile(aFilePath) {
      let aResult = await OS.File.read(aFilePath, { compression: "lz4" });
      let decoder = new TextDecoder();
      let jsonString = decoder.decode(aResult);
      await this.importFromJSON(jsonString);
  },

  /**
   * Import bookmarks from a JSON string.
   *
   * @param {String} aString JSON string of serialized bookmark data.
   * @return {Promise}
   * @resolves When the new bookmarks have been created.
   * @rejects JavaScript exception.
   */
  async importFromJSON(aString) {
    let nodes =
      PlacesUtils.unwrapNodes(aString, PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER);

    if (nodes.length == 0 || !nodes[0].children ||
        nodes[0].children.length == 0) {
      return;
    }

    await this.import(nodes[0]);
  },

  async import(rootNode) {
    // Change to rootNode.children as we don't import the root, and also filter
    // out any obsolete "tagsFolder" sections.
    let nodes = rootNode.children.filter(node => node.root !== "tagsFolder");

    // If we're replacing, then erase existing bookmarks first.
    if (this._replace) {
      await PlacesUtils.bookmarks.eraseEverything({ source: this._source });
    }

    let folderIdToGuidMap = {};

    // Now do some cleanup on the imported nodes so that the various guids
    // match what we need for insertTree, and we also have mappings of folders
    // so we can repair any searches after inserting the bookmarks (see bug 824502).
    for (let node of nodes) {
      if (!node.children || node.children.length == 0)
        continue; // Nothing to restore for this root

      // Ensure we set the source correctly.
      node.source = this._source;

      // Translate the node for insertTree.
      let folders = translateTreeTypes(node);

      folderIdToGuidMap = Object.assign(folderIdToGuidMap, folders);
    }

    // Now we can add the actual nodes to the database.
    for (let node of nodes) {
      // Drop any nodes without children, we can't insert them.
      if (!node.children || node.children.length == 0) {
        continue;
      }

      // Drop any roots whose guid we don't recognise - we don't support anything
      // apart from the built-in roots.
      if (!PlacesUtils.bookmarks.userContentRoots.includes(node.guid)) {
        continue;
      }

      fixupSearchQueries(node, folderIdToGuidMap);

      await PlacesUtils.bookmarks.insertTree(node, { fixupOrSkipInvalidEntries: true });

      // Now add any favicons.
      try {
        insertFaviconsForTree(node);
      } catch (ex) {
        Cu.reportError(`Failed to insert favicons: ${ex}`);
      }
    }
  },
};

function notifyObservers(topic, replace) {
  Services.obs.notifyObservers(null, topic, replace ? "json" : "json-append");
}

/**
 * Iterates through a node, fixing up any place: URL queries that are found. This
 * replaces any old (pre Firefox 62) queries that contain "folder=<id>" parts with
 * "parent=<guid>".
 *
 * @param {Object} aNode The node to search.
 * @param {Array} aFolderIdMap An array mapping of old folder IDs to new folder GUIDs.
 */
function fixupSearchQueries(aNode, aFolderIdMap) {
  if (aNode.url && aNode.url.startsWith("place:")) {
    aNode.url = fixupQuery(aNode.url, aFolderIdMap);
  }
  if (aNode.children) {
    for (let child of aNode.children) {
      fixupSearchQueries(child, aFolderIdMap);
    }
  }
}

/**
 * Replaces imported folder ids with their local counterparts in a place: URI.
 *
 * @param   {String} aQueryURL
 *          A place: URI with folder ids.
 * @param   {Object} aFolderIdMap
 *          An array mapping of old folder IDs to new folder GUIDs.
 * @return {String} the fixed up URI if all matched. If some matched, it returns
 *         the URI with only the matching folders included. If none matched
 *         it returns the input URI unchanged.
 */
function fixupQuery(aQueryURL, aFolderIdMap) {
  let invalid = false;
  let convert = function(str, existingFolderId) {
    let guid;
    if (Object.keys(OLD_BOOKMARK_QUERY_TRANSLATIONS).includes(existingFolderId)) {
      guid = OLD_BOOKMARK_QUERY_TRANSLATIONS[existingFolderId];
    } else {
      guid = aFolderIdMap[existingFolderId];
      if (!guid) {
        invalid = true;
        return `invalidOldParentId=${existingFolderId}`;
      }
    }
    return `parent=${guid}`;
  };

  let url = aQueryURL.replace(/folder=([A-Za-z0-9_]+)/g, convert);
  if (invalid) {
    // One or more of the folders don't exist, cause an empty query so that
    // we don't try to display the whole database.
    url += "&excludeItems=1";
  }
  return url;
}

/**
 * A mapping of root folder names to Guids. To help fixupRootFolderGuid.
 */
const rootToFolderGuidMap = {
  "placesRoot": PlacesUtils.bookmarks.rootGuid,
  "bookmarksMenuFolder": PlacesUtils.bookmarks.menuGuid,
  "unfiledBookmarksFolder": PlacesUtils.bookmarks.unfiledGuid,
  "toolbarFolder": PlacesUtils.bookmarks.toolbarGuid,
  "mobileFolder": PlacesUtils.bookmarks.mobileGuid,
};

/**
 * Updates a bookmark node from the json version to the places GUID. This
 * will only change GUIDs for the built-in folders. Other folders will remain
 * unchanged.
 *
 * @param {Object} A bookmark node that is updated with the new GUID if necessary.
 */
function fixupRootFolderGuid(node) {
  if (!node.guid && node.root && node.root in rootToFolderGuidMap) {
    node.guid = rootToFolderGuidMap[node.root];
  }
}

/**
 * Translates the JSON types for a node and its children into Places compatible
 * types. Also handles updating of other parameters e.g. dateAdded and lastModified.
 *
 * @param {Object} node A node to be updated. If it contains children, they will
 *                      be updated as well.
 * @return {Array} An array containing two items:
 *       - {Object} A map of current folder ids to GUIDS
 *       - {Array} An array of GUIDs for nodes that contain query URIs
 */
function translateTreeTypes(node) {
  let folderIdToGuidMap = {};

  // Do the uri fixup first, so we can be consistent in this function.
  if (node.uri) {
    node.url = node.uri;
    delete node.uri;
  }

  switch (node.type) {
    case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
      node.type = PlacesUtils.bookmarks.TYPE_FOLDER;

      // Older type mobile folders have a random guid with an annotation. We need
      // to make sure those go into the proper mobile folder.
      let isMobileFolder = node.annos &&
                           node.annos.some(anno => anno.name == PlacesUtils.MOBILE_ROOT_ANNO);
      if (isMobileFolder) {
        node.guid = PlacesUtils.bookmarks.mobileGuid;
      } else {
        // In case the Guid is broken, we need to fix it up.
        fixupRootFolderGuid(node);
      }

      // Record the current id and the guid so that we can update any search
      // queries later.
      folderIdToGuidMap[node.id] = node.guid;
      break;
    case PlacesUtils.TYPE_X_MOZ_PLACE:
      node.type = PlacesUtils.bookmarks.TYPE_BOOKMARK;
      break;
    case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
      node.type = PlacesUtils.bookmarks.TYPE_SEPARATOR;
      if ("title" in node) {
        delete node.title;
      }
      break;
    default:
      // No need to throw/reject here, insertTree will remove this node automatically.
      Cu.reportError(`Unexpected bookmark type ${node.type}`);
      break;
  }

  if (node.dateAdded) {
    node.dateAdded = PlacesUtils.toDate(node.dateAdded);
  }

  if (node.lastModified) {
    let lastModified = PlacesUtils.toDate(node.lastModified);
    // Ensure we get a last modified date that's later or equal to the dateAdded
    // so that we don't upset the Bookmarks API.
    if (lastModified >= node.dateAdded) {
      node.lastModified = lastModified;
    } else {
      delete node.lastModified;
    }
  }

  if (node.tags) {
     // Separate any tags into an array, and ignore any that are too long.
    node.tags = node.tags.split(",").filter(aTag =>
      aTag.length > 0 && aTag.length <= PlacesUtils.bookmarks.MAX_TAG_LENGTH);

    // If we end up with none, then delete the property completely.
    if (!node.tags.length) {
      delete node.tags;
    }
  }

  // Sometimes postData can be null, so delete it to make the validators happy.
  if (node.postData == null) {
    delete node.postData;
  }

  // Now handle any children.
  if (!node.children) {
    return folderIdToGuidMap;
  }

  // First sort the children by index.
  node.children = node.children.sort((a, b) => {
    return a.index - b.index;
  });

  // Now do any adjustments required for the children.
  for (let child of node.children) {
    let folders = translateTreeTypes(child);
    folderIdToGuidMap = Object.assign(folderIdToGuidMap, folders);
  }

  return folderIdToGuidMap;
}

/**
 * Handles inserting favicons into the database for a bookmark node.
 * It is assumed the node has already been inserted into the bookmarks
 * database.
 *
 * @param {Object} node The bookmark node for icons to be inserted.
 */
function insertFaviconForNode(node) {
  if (node.icon) {
    try {
      // Create a fake faviconURI to use (FIXME: bug 523932)
      let faviconURI = Services.io.newURI("fake-favicon-uri:" + node.url);
      PlacesUtils.favicons.replaceFaviconDataFromDataURL(
        faviconURI, node.icon, 0,
        Services.scriptSecurityManager.getSystemPrincipal());
      PlacesUtils.favicons.setAndFetchFaviconForPage(
        Services.io.newURI(node.url), faviconURI, false,
        PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, null,
        Services.scriptSecurityManager.getSystemPrincipal());
    } catch (ex) {
      Cu.reportError("Failed to import favicon data:" + ex);
    }
  }

  if (!node.iconUri) {
    return;
  }

  try {
    PlacesUtils.favicons.setAndFetchFaviconForPage(
      Services.io.newURI(node.url), Services.io.newURI(node.iconUri), false,
      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, null,
      Services.scriptSecurityManager.getSystemPrincipal());
  } catch (ex) {
    Cu.reportError("Failed to import favicon URI:" + ex);
  }
}

/**
 * Handles inserting favicons into the database for a bookmark tree - a node
 * and its children.
 *
 * It is assumed the nodes have already been inserted into the bookmarks
 * database.
 *
 * @param {Object} nodeTree The bookmark node tree for icons to be inserted.
 */
function insertFaviconsForTree(nodeTree) {
  insertFaviconForNode(nodeTree);

  if (nodeTree.children) {
    for (let child of nodeTree.children) {
      insertFaviconsForTree(child);
    }
  }
}