Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
/*
* Pretenuring.
*
* Some kinds of GC cells can be allocated in either the nursery or the tenured
* heap. The pretenuring system decides where to allocate such cells based on
* their expected lifetime with the aim of minimising total collection time.
*
* Lifetime is predicted based on data gathered about the cells' allocation
* site. This data is gathered in the middle JIT tiers, after code has stopped
* executing in the interpreter and before we generate fully optimized code.
*/
#ifndef gc_Pretenuring_h
#define gc_Pretenuring_h
#include <algorithm>
#include "gc/AllocKind.h"
#include "js/TypeDecls.h"
class JS_PUBLIC_API JSTracer;
namespace JS {
enum class GCReason;
} // namespace JS
namespace js::gc {
class GCRuntime;
class PretenuringNursery;
// Number of trace kinds supportd by the nursery. These are arranged at the
// start of JS::TraceKind.
static constexpr size_t NurseryTraceKinds = 3;
enum class CatchAllAllocSite { Unknown, Optimized };
// Information about an allocation site.
//
// Nursery cells contain a pointer to one of these in their cell header (stored
// before the cell). The site can relate to either a specific JS bytecode
// instruction, a specific WebAssembly type, or can be a catch-all instance for
// unknown sites or JS JIT optimized code.
class AllocSite {
public:
enum class State : uint32_t { ShortLived = 0, Unknown = 1, LongLived = 2 };
// The JIT depends on being able to tell the states apart by checking a single
// bit.
static constexpr int32_t LONG_LIVED_BIT = int32_t(State::LongLived);
static_assert((LONG_LIVED_BIT & int32_t(State::Unknown)) == 0);
static_assert((AllocSite::LONG_LIVED_BIT & int32_t(State::ShortLived)) == 0);
private:
JS::Zone* zone_ = nullptr;
// Word storing JSScript pointer and site state.
//
// The script pointer is the script that owns this allocation site, a special
// sentinel script for wasm sites, or null for unknown sites. This is used
// when we need to invalidate the script.
uintptr_t scriptAndState = uintptr_t(State::Unknown);
static constexpr uintptr_t STATE_MASK = BitMask(2);
// Next pointer forming a linked list of sites at which nursery allocation
// happened since the last nursery collection.
AllocSite* nextNurseryAllocated = nullptr;
// Bytecode offset of this allocation site. Only valid if hasScript().
// Note that the offset does not need to correspond with the script stored in
// this AllocSite, because if we're doing trial-inlining, the script will be
// the outer script and the pc offset can be in an inlined script.
static constexpr uint32_t InvalidPCOffset = UINT32_MAX;
uint32_t pcOffset_ = InvalidPCOffset;
// Number of nursery allocations at this site since last nursery collection.
uint32_t nurseryAllocCount = 0;
// Number of nursery allocations that survived. Used during collection.
uint32_t nurseryPromotedCount : 24;
// Number of times the script has been invalidated.
uint32_t invalidationCount : 4;
// The trace kind of the allocation. Only kinds up to NurseryTraceKinds are
// allowed.
uint32_t traceKind_ : 4;
static AllocSite* const EndSentinel;
// Sentinel script for wasm sites.
static JSScript* const WasmScript;
friend class PretenuringZone;
friend class PretenuringNursery;
uintptr_t rawScript() const { return scriptAndState & ~STATE_MASK; }
public:
AllocSite() : nurseryPromotedCount(0), invalidationCount(0), traceKind_(0) {}
// Create a dummy site to use for unknown allocations.
explicit AllocSite(JS::Zone* zone, JS::TraceKind kind)
: zone_(zone),
nurseryPromotedCount(0),
invalidationCount(0),
traceKind_(uint32_t(kind)) {
MOZ_ASSERT(traceKind_ < NurseryTraceKinds);
}
// Create a site for an opcode in the given script.
AllocSite(JS::Zone* zone, JSScript* script, uint32_t pcOffset,
JS::TraceKind kind)
: AllocSite(zone, kind) {
MOZ_ASSERT(script != WasmScript);
setScript(script);
pcOffset_ = pcOffset;
}
void initUnknownSite(JS::Zone* zone, JS::TraceKind kind) {
MOZ_ASSERT(!zone_ && scriptAndState == uintptr_t(State::Unknown));
zone_ = zone;
nurseryPromotedCount = 0;
invalidationCount = 0;
traceKind_ = uint32_t(kind);
MOZ_ASSERT(traceKind_ < NurseryTraceKinds);
}
// Initialize a site to be a wasm site.
void initWasm(JS::Zone* zone) {
MOZ_ASSERT(!zone_ && scriptAndState == uintptr_t(State::Unknown));
zone_ = zone;
setScript(WasmScript);
nurseryPromotedCount = 0;
invalidationCount = 0;
traceKind_ = uint32_t(JS::TraceKind::Object);
}
JS::Zone* zone() const { return zone_; }
JS::TraceKind traceKind() const { return JS::TraceKind(traceKind_); }
State state() const { return State(scriptAndState & STATE_MASK); }
// Whether this site has a script associated with it. This is not true if
// this site is for a wasm site.
bool hasScript() const {
return rawScript() && rawScript() != uintptr_t(WasmScript);
}
JSScript* script() const {
MOZ_ASSERT(hasScript());
return reinterpret_cast<JSScript*>(rawScript());
}
uint32_t pcOffset() const {
MOZ_ASSERT(hasScript());
MOZ_ASSERT(pcOffset_ != InvalidPCOffset);
return pcOffset_;
}
// Whether this site is not an unknown or optimized site.
bool isNormal() const { return rawScript() != 0; }
enum class Kind : uint32_t { Normal, Unknown, Optimized };
Kind kind() const;
bool isInAllocatedList() const { return nextNurseryAllocated; }
// Whether allocations at this site should be allocated in the nursery or the
// tenured heap.
Heap initialHeap() const {
return state() == State::LongLived ? Heap::Tenured : Heap::Default;
}
bool hasNurseryAllocations() const {
return nurseryAllocCount != 0 || nurseryPromotedCount != 0;
}
void resetNurseryAllocations() {
nurseryAllocCount = 0;
nurseryPromotedCount = 0;
}
uint32_t incAllocCount() { return ++nurseryAllocCount; }
uint32_t* nurseryAllocCountAddress() { return &nurseryAllocCount; }
void incPromotedCount() {
// The nursery is not large enough for this to overflow.
nurseryPromotedCount++;
MOZ_ASSERT(nurseryPromotedCount != 0);
}
size_t allocCount() const {
return std::max(nurseryAllocCount, nurseryPromotedCount);
}
// Called for every active alloc site after minor GC.
enum SiteResult { NoChange, WasPretenured, WasPretenuredAndInvalidated };
SiteResult processSite(GCRuntime* gc, size_t attentionThreshold,
bool reportInfo, size_t reportThreshold);
void processCatchAllSite(bool reportInfo, size_t reportThreshold);
void updateStateOnMinorGC(double promotionRate);
// Reset the state to 'Unknown' unless we have reached the invalidation limit
// for this site. Return whether the state was reset.
bool maybeResetState();
bool invalidationLimitReached() const;
bool invalidateScript(GCRuntime* gc);
void trace(JSTracer* trc);
static void printInfoHeader(JS::GCReason reason, double promotionRate);
static void printInfoFooter(size_t sitesCreated, size_t sitesActive,
size_t sitesPretenured, size_t sitesInvalidated);
void printInfo(bool hasPromotionRate, double promotionRate,
bool wasInvalidated) const;
static constexpr size_t offsetOfScriptAndState() {
return offsetof(AllocSite, scriptAndState);
}
static constexpr size_t offsetOfNurseryAllocCount() {
return offsetof(AllocSite, nurseryAllocCount);
}
static constexpr size_t offsetOfNextNurseryAllocated() {
return offsetof(AllocSite, nextNurseryAllocated);
}
private:
void setScript(JSScript* newScript) {
MOZ_ASSERT((uintptr_t(newScript) & STATE_MASK) == 0);
scriptAndState = uintptr_t(newScript) | uintptr_t(state());
}
void setState(State newState) {
MOZ_ASSERT((uintptr_t(newState) & ~STATE_MASK) == 0);
scriptAndState = rawScript() | uintptr_t(newState);
}
const char* stateName() const;
};
// Pretenuring information stored per zone.
class PretenuringZone {
public:
// Catch-all allocation site instance used when the actual site is unknown, or
// when optimized JIT code allocates a GC thing that's not handled by the
// pretenuring system.
AllocSite unknownAllocSites[NurseryTraceKinds];
// Catch-all allocation instance used by optimized JIT code when allocating GC
// things that are handled by the pretenuring system. Allocation counts are
// not recorded by optimized JIT code.
AllocSite optimizedAllocSite;
// Allocation sites used for nursery cells promoted to the next nursery
// generation that didn't come from optimized alloc sites.
AllocSite promotedAllocSites[NurseryTraceKinds];
// Count of tenured cell allocations made between each major collection and
// how many survived.
uint32_t allocCountInNewlyCreatedArenas = 0;
uint32_t survivorCountInNewlyCreatedArenas = 0;
// Count of successive collections that had a low young tenured survival
// rate. Used to discard optimized code if we get the pretenuring decision
// wrong.
uint32_t lowYoungTenuredSurvivalCount = 0;
// Count of successive nursery collections that had a high survival rate for
// objects allocated by optimized code. Used to discard optimized code if we
// get the pretenuring decision wrong.
uint32_t highNurserySurvivalCount = 0;
// Total allocation count by trace kind (ignoring optimized
// allocations). Calculated during nursery collection.
uint32_t nurseryAllocCounts[NurseryTraceKinds] = {0};
explicit PretenuringZone(JS::Zone* zone)
: optimizedAllocSite(zone, JS::TraceKind::Object) {
for (uint32_t i = 0; i < NurseryTraceKinds; i++) {
unknownAllocSites[i].initUnknownSite(zone, JS::TraceKind(i));
promotedAllocSites[i].initUnknownSite(zone, JS::TraceKind(i));
}
}
AllocSite& unknownAllocSite(JS::TraceKind kind) {
size_t i = size_t(kind);
MOZ_ASSERT(i < NurseryTraceKinds);
return unknownAllocSites[i];
}
AllocSite& promotedAllocSite(JS::TraceKind kind) {
size_t i = size_t(kind);
MOZ_ASSERT(i < NurseryTraceKinds);
return promotedAllocSites[i];
}
void clearCellCountsInNewlyCreatedArenas() {
allocCountInNewlyCreatedArenas = 0;
survivorCountInNewlyCreatedArenas = 0;
}
void updateCellCountsInNewlyCreatedArenas(uint32_t allocCount,
uint32_t survivorCount) {
allocCountInNewlyCreatedArenas += allocCount;
survivorCountInNewlyCreatedArenas += survivorCount;
}
bool calculateYoungTenuredSurvivalRate(double* rateOut);
void noteLowYoungTenuredSurvivalRate(bool lowYoungSurvivalRate);
void noteHighNurserySurvivalRate(bool highNurserySurvivalRate);
// Recovery: if code behaviour change we may need to reset allocation site
// state and invalidate JIT code.
bool shouldResetNurseryAllocSites();
bool shouldResetPretenuredAllocSites();
uint32_t& nurseryAllocCount(JS::TraceKind kind) {
size_t i = size_t(kind);
MOZ_ASSERT(i < NurseryTraceKinds);
return nurseryAllocCounts[i];
}
uint32_t nurseryAllocCount(JS::TraceKind kind) const {
return const_cast<PretenuringZone*>(this)->nurseryAllocCount(kind);
}
};
// Pretenuring information stored as part of the the GC nursery.
class PretenuringNursery {
gc::AllocSite* allocatedSites;
size_t allocSitesCreated = 0;
uint32_t totalAllocCount_ = 0;
public:
PretenuringNursery() : allocatedSites(AllocSite::EndSentinel) {}
bool hasAllocatedSites() const {
return allocatedSites != AllocSite::EndSentinel;
}
bool canCreateAllocSite();
void noteAllocSiteCreated() { allocSitesCreated++; }
void insertIntoAllocatedList(AllocSite* site) {
MOZ_ASSERT(!site->isInAllocatedList());
site->nextNurseryAllocated = allocatedSites;
allocatedSites = site;
}
size_t doPretenuring(GCRuntime* gc, JS::GCReason reason,
bool validPromotionRate, double promotionRate,
bool reportInfo, size_t reportThreshold);
void maybeStopPretenuring(GCRuntime* gc);
uint32_t totalAllocCount() const { return totalAllocCount_; }
void* addressOfAllocatedSites() { return &allocatedSites; }
private:
void updateTotalAllocCounts(AllocSite* site);
};
} // namespace js::gc
#endif /* gc_Pretenuring_h */