Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#ifdef XP_WIN
# include "objbase.h"
#endif
#include "mozilla/dom/HTMLMediaElement.h"
#include <unordered_map>
#include "AudioDeviceInfo.h"
#include "AudioStreamTrack.h"
#include "AutoplayPolicy.h"
#include "ChannelMediaDecoder.h"
#include "CrossGraphPort.h"
#include "DOMMediaStream.h"
#include "DecoderDoctorDiagnostics.h"
#include "DecoderDoctorLogger.h"
#include "DecoderTraits.h"
#include "FrameStatistics.h"
#include "GMPCrashHelper.h"
#include "GVAutoplayPermissionRequest.h"
#ifdef MOZ_ANDROID_HLS_SUPPORT
# include "HLSDecoder.h"
#endif
#include "HTMLMediaElement.h"
#include "ImageContainer.h"
#include "MP4Decoder.h"
#include "MediaContainerType.h"
#include "MediaError.h"
#include "MediaManager.h"
#include "MediaMetadataManager.h"
#include "MediaResource.h"
#include "MediaShutdownManager.h"
#include "MediaSourceDecoder.h"
#include "MediaStreamError.h"
#include "MediaTrackGraphImpl.h"
#include "MediaTrackListener.h"
#include "MediaStreamWindowCapturer.h"
#include "MediaTrack.h"
#include "MediaTrackList.h"
#include "Navigator.h"
#include "TimeRanges.h"
#include "VideoFrameContainer.h"
#include "VideoOutput.h"
#include "VideoStreamTrack.h"
#include "base/basictypes.h"
#include "jsapi.h"
#include "js/PropertyAndElement.h" // JS_DefineProperty
#include "mozilla/AppShutdown.h"
#include "mozilla/ArrayUtils.h"
#include "mozilla/AsyncEventDispatcher.h"
#include "mozilla/EMEUtils.h"
#include "mozilla/EventDispatcher.h"
#include "mozilla/FloatingPoint.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/NotNull.h"
#include "mozilla/Preferences.h"
#include "mozilla/PresShell.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/SchedulerGroup.h"
#include "mozilla/Sprintf.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/SVGObserverUtils.h"
#include "mozilla/Telemetry.h"
#include "mozilla/dom/AudioTrack.h"
#include "mozilla/dom/AudioTrackList.h"
#include "mozilla/dom/BlobURLProtocolHandler.h"
#include "mozilla/dom/ContentMediaController.h"
#include "mozilla/dom/ElementInlines.h"
#include "mozilla/dom/FeaturePolicyUtils.h"
#include "mozilla/dom/HTMLAudioElement.h"
#include "mozilla/dom/HTMLInputElement.h"
#include "mozilla/dom/HTMLMediaElementBinding.h"
#include "mozilla/dom/HTMLSourceElement.h"
#include "mozilla/dom/HTMLVideoElement.h"
#include "mozilla/dom/MediaControlUtils.h"
#include "mozilla/dom/MediaDevices.h"
#include "mozilla/dom/MediaEncryptedEvent.h"
#include "mozilla/dom/MediaErrorBinding.h"
#include "mozilla/dom/MediaSource.h"
#include "mozilla/dom/PlayPromise.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/dom/TextTrack.h"
#include "mozilla/dom/UserActivation.h"
#include "mozilla/dom/VideoPlaybackQuality.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/VideoTrackList.h"
#include "mozilla/dom/WakeLock.h"
#include "mozilla/dom/WindowGlobalChild.h"
#include "mozilla/dom/power/PowerManagerService.h"
#include "mozilla/net/UrlClassifierFeatureFactory.h"
#include "nsAttrValueInlines.h"
#include "nsContentPolicyUtils.h"
#include "nsContentUtils.h"
#include "nsCycleCollectionParticipant.h"
#include "nsDisplayList.h"
#include "nsDocShell.h"
#include "nsError.h"
#include "nsGenericHTMLElement.h"
#include "nsGkAtoms.h"
#include "nsIAsyncVerifyRedirectCallback.h"
#include "nsICachingChannel.h"
#include "nsIClassOfService.h"
#include "nsIContentPolicy.h"
#include "nsIDocShell.h"
#include "mozilla/dom/Document.h"
#include "nsIFrame.h"
#include "nsIHttpChannel.h"
#include "nsIObserverService.h"
#include "nsIRequest.h"
#include "nsIScriptError.h"
#include "nsISupportsPrimitives.h"
#include "nsIThreadRetargetableStreamListener.h"
#include "nsITimer.h"
#include "nsJSUtils.h"
#include "nsLayoutUtils.h"
#include "nsMediaFragmentURIParser.h"
#include "nsMimeTypes.h"
#include "nsNetUtil.h"
#include "nsNodeInfoManager.h"
#include "nsPresContext.h"
#include "nsQueryObject.h"
#include "nsRange.h"
#include "nsSize.h"
#include "nsThreadUtils.h"
#include "nsURIHashKey.h"
#include "nsURLHelper.h"
#include "nsVideoFrame.h"
#include "ReferrerInfo.h"
#include "TimeUnits.h"
#include "xpcpublic.h"
#include <algorithm>
#include <cmath>
#include <limits>
#include <type_traits>
mozilla::LazyLogModule gMediaElementLog("HTMLMediaElement");
mozilla::LazyLogModule gMediaElementEventsLog("HTMLMediaElementEvents");
extern mozilla::LazyLogModule gAutoplayPermissionLog;
#define AUTOPLAY_LOG(msg, ...) \
MOZ_LOG(gAutoplayPermissionLog, LogLevel::Debug, (msg, ##__VA_ARGS__))
// avoid redefined macro in unified build
#undef MEDIACONTROL_LOG
#define MEDIACONTROL_LOG(msg, ...) \
MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
("HTMLMediaElement=%p, " msg, this, ##__VA_ARGS__))
#undef CONTROLLER_TIMER_LOG
#define CONTROLLER_TIMER_LOG(element, msg, ...) \
MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
("HTMLMediaElement=%p, " msg, element, ##__VA_ARGS__))
#define LOG(type, msg) MOZ_LOG(gMediaElementLog, type, msg)
#define LOG_EVENT(type, msg) MOZ_LOG(gMediaElementEventsLog, type, msg)
using namespace mozilla::layers;
using mozilla::net::nsMediaFragmentURIParser;
using namespace mozilla::dom::HTMLMediaElement_Binding;
namespace mozilla::dom {
using AudibleState = AudioChannelService::AudibleState;
using SinkInfoPromise = MediaDevices::SinkInfoPromise;
// Number of milliseconds between progress events as defined by spec
static const uint32_t PROGRESS_MS = 350;
// Number of milliseconds of no data before a stall event is fired as defined by
// spec
static const uint32_t STALL_MS = 3000;
// Used by AudioChannel for suppresssing the volume to this ratio.
#define FADED_VOLUME_RATIO 0.25
// These constants are arbitrary
// Minimum playbackRate for a media
static const double MIN_PLAYBACKRATE = 1.0 / 16;
// Maximum playbackRate for a media
static const double MAX_PLAYBACKRATE = 16.0;
static double ClampPlaybackRate(double aPlaybackRate) {
MOZ_ASSERT(aPlaybackRate >= 0.0);
if (aPlaybackRate == 0.0) {
return aPlaybackRate;
}
if (aPlaybackRate < MIN_PLAYBACKRATE) {
return MIN_PLAYBACKRATE;
}
if (aPlaybackRate > MAX_PLAYBACKRATE) {
return MAX_PLAYBACKRATE;
}
return aPlaybackRate;
}
// Media error values. These need to match the ones in MediaError.webidl.
static const unsigned short MEDIA_ERR_ABORTED = 1;
static const unsigned short MEDIA_ERR_NETWORK = 2;
static const unsigned short MEDIA_ERR_DECODE = 3;
static const unsigned short MEDIA_ERR_SRC_NOT_SUPPORTED = 4;
/**
* EventBlocker helps media element to postpone the event delivery by storing
* the event runner, and execute them once media element decides not to postpone
* the event delivery. If media element never resumes the event delivery, then
* those runner would be cancelled.
* For example, we postpone the event delivery when media element entering to
* the bf-cache.
*/
class HTMLMediaElement::EventBlocker final : public nsISupports {
public:
NS_DECL_CYCLE_COLLECTING_ISUPPORTS_FINAL
NS_DECL_CYCLE_COLLECTION_CLASS(EventBlocker)
explicit EventBlocker(HTMLMediaElement* aElement) : mElement(aElement) {}
void SetBlockEventDelivery(bool aShouldBlock) {
MOZ_ASSERT(NS_IsMainThread());
if (mShouldBlockEventDelivery == aShouldBlock) {
return;
}
LOG_EVENT(LogLevel::Debug,
("%p %s event delivery", mElement.get(),
mShouldBlockEventDelivery ? "block" : "unblock"));
mShouldBlockEventDelivery = aShouldBlock;
if (!mShouldBlockEventDelivery) {
DispatchPendingMediaEvents();
}
}
void PostponeEvent(nsMediaEventRunner* aRunner) {
MOZ_ASSERT(NS_IsMainThread());
// Element has been CCed, which would break the weak pointer.
if (!mElement) {
return;
}
MOZ_ASSERT(mShouldBlockEventDelivery);
MOZ_ASSERT(mElement);
LOG_EVENT(LogLevel::Debug,
("%p postpone runner %s for %s", mElement.get(),
NS_ConvertUTF16toUTF8(aRunner->Name()).get(),
NS_ConvertUTF16toUTF8(aRunner->EventName()).get()));
mPendingEventRunners.AppendElement(aRunner);
}
void Shutdown() {
MOZ_ASSERT(NS_IsMainThread());
for (auto& runner : mPendingEventRunners) {
runner->Cancel();
}
mPendingEventRunners.Clear();
}
bool ShouldBlockEventDelivery() const {
MOZ_ASSERT(NS_IsMainThread());
return mShouldBlockEventDelivery;
}
size_t SizeOfExcludingThis(MallocSizeOf& aMallocSizeOf) const {
MOZ_ASSERT(NS_IsMainThread());
size_t total = 0;
for (const auto& runner : mPendingEventRunners) {
total += aMallocSizeOf(runner);
}
return total;
}
private:
~EventBlocker() = default;
void DispatchPendingMediaEvents() {
MOZ_ASSERT(mElement);
for (auto& runner : mPendingEventRunners) {
LOG_EVENT(LogLevel::Debug,
("%p execute runner %s for %s", mElement.get(),
NS_ConvertUTF16toUTF8(runner->Name()).get(),
NS_ConvertUTF16toUTF8(runner->EventName()).get()));
GetMainThreadSerialEventTarget()->Dispatch(runner.forget());
}
mPendingEventRunners.Clear();
}
WeakPtr<HTMLMediaElement> mElement;
bool mShouldBlockEventDelivery = false;
// Contains event runners which should not be run for now because we want
// to block all events delivery. They would be dispatched once media element
// decides unblocking them.
nsTArray<RefPtr<nsMediaEventRunner>> mPendingEventRunners;
};
NS_IMPL_CYCLE_COLLECTION(HTMLMediaElement::EventBlocker, mPendingEventRunners)
NS_IMPL_CYCLE_COLLECTING_ADDREF(HTMLMediaElement::EventBlocker)
NS_IMPL_CYCLE_COLLECTING_RELEASE(HTMLMediaElement::EventBlocker)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(HTMLMediaElement::EventBlocker)
NS_INTERFACE_MAP_ENTRY(nsISupports)
NS_INTERFACE_MAP_END
/**
* We use MediaControlKeyListener to listen to media control key in order to
* play and pause media element when user press media control keys and update
* media's playback and audible state to the media controller.
*
* Use `Start()` to start listening event and use `Stop()` to stop listening
* event. In addition, notifying any change to media controller MUST be done
* after successfully calling `Start()`.
*/
class HTMLMediaElement::MediaControlKeyListener final
: public ContentMediaControlKeyReceiver {
public:
NS_INLINE_DECL_REFCOUNTING(MediaControlKeyListener, override)
MOZ_INIT_OUTSIDE_CTOR explicit MediaControlKeyListener(
HTMLMediaElement* aElement)
: mElement(aElement) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aElement);
}
/**
* Start listening to the media control keys which would make media being able
* to be controlled via pressing media control keys.
*/
void Start() {
MOZ_ASSERT(NS_IsMainThread());
if (IsStarted()) {
// We have already been started, do not notify start twice.
return;
}
// Fail to init media agent, we are not able to notify the media controller
// any update and also are not able to receive media control key events.
if (!InitMediaAgent()) {
MEDIACONTROL_LOG("Failed to start due to not able to init media agent!");
return;
}
NotifyPlaybackStateChanged(MediaPlaybackState::eStarted);
// If owner has started playing before the listener starts, we should update
// the playing state as well. Eg. media starts inaudily and becomes audible
// later.
if (!Owner()->Paused()) {
NotifyMediaStartedPlaying();
}
if (StaticPrefs::media_mediacontrol_testingevents_enabled()) {
auto dispatcher = MakeRefPtr<AsyncEventDispatcher>(
Owner(), u"MozStartMediaControl"_ns, CanBubble::eYes,
ChromeOnlyDispatch::eYes);
dispatcher->PostDOMEvent();
}
}
/**
* Stop listening to the media control keys which would make media not be able
* to be controlled via pressing media control keys. If we haven't started
* listening to the media control keys, then nothing would happen.
*/
void StopIfNeeded() {
MOZ_ASSERT(NS_IsMainThread());
if (!IsStarted()) {
// We have already been stopped, do not notify stop twice.
return;
}
NotifyMediaStoppedPlaying();
NotifyPlaybackStateChanged(MediaPlaybackState::eStopped);
// Remove ourselves from media agent, which would stop receiving event.
mControlAgent->RemoveReceiver(this);
mControlAgent = nullptr;
}
bool IsStarted() const { return mState != MediaPlaybackState::eStopped; }
bool IsPlaying() const override {
return Owner() ? !Owner()->Paused() : false;
}
/**
* Following methods should only be used after starting listener.
*/
void NotifyMediaStartedPlaying() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(IsStarted());
if (mState == MediaPlaybackState::eStarted ||
mState == MediaPlaybackState::ePaused) {
NotifyPlaybackStateChanged(MediaPlaybackState::ePlayed);
// If media is `inaudible` in the beginning, then we don't need to notify
// the state, because notifying `inaudible` should always come after
// notifying `audible`.
if (mIsOwnerAudible) {
NotifyAudibleStateChanged(MediaAudibleState::eAudible);
}
}
}
void NotifyMediaStoppedPlaying() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(IsStarted());
if (mState == MediaPlaybackState::ePlayed) {
NotifyPlaybackStateChanged(MediaPlaybackState::ePaused);
// As media are going to be paused, so no sound is possible to be heard.
if (mIsOwnerAudible) {
NotifyAudibleStateChanged(MediaAudibleState::eInaudible);
}
}
}
// This method can be called before the listener starts, which would cache
// the audible state and update after the listener starts.
void UpdateMediaAudibleState(bool aIsOwnerAudible) {
MOZ_ASSERT(NS_IsMainThread());
if (mIsOwnerAudible == aIsOwnerAudible) {
return;
}
mIsOwnerAudible = aIsOwnerAudible;
MEDIACONTROL_LOG("Media becomes %s",
mIsOwnerAudible ? "audible" : "inaudible");
// If media hasn't started playing, it doesn't make sense to update media
// audible state. Therefore, in that case we would noitfy the audible state
// when media starts playing.
if (mState == MediaPlaybackState::ePlayed) {
NotifyAudibleStateChanged(mIsOwnerAudible
? MediaAudibleState::eAudible
: MediaAudibleState::eInaudible);
}
}
void SetPictureInPictureModeEnabled(bool aIsEnabled) {
MOZ_ASSERT(NS_IsMainThread());
if (mIsPictureInPictureEnabled == aIsEnabled) {
return;
}
// PIP state changes might happen before the listener starts or stops where
// we haven't call `InitMediaAgent()` yet. Eg. Reset the PIP video's src,
// then cancel the PIP. In addition, not like playback and audible state
// which should be restricted to update via the same agent in order to keep
// those states correct in each `ContextMediaInfo`, PIP state can be updated
// through any browsing context, so we would use `ContentMediaAgent::Get()`
// directly to update PIP state.
mIsPictureInPictureEnabled = aIsEnabled;
if (RefPtr<IMediaInfoUpdater> updater =
ContentMediaAgent::Get(GetCurrentBrowsingContext())) {
updater->SetIsInPictureInPictureMode(mOwnerBrowsingContextId,
mIsPictureInPictureEnabled);
}
}
void HandleMediaKey(MediaControlKey aKey) override {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(IsStarted());
MEDIACONTROL_LOG("HandleEvent '%s'", GetEnumString(aKey).get());
if (aKey == MediaControlKey::Play) {
Owner()->Play();
} else if (aKey == MediaControlKey::Pause) {
Owner()->Pause();
} else {
MOZ_ASSERT(aKey == MediaControlKey::Stop,
"Not supported key for media element!");
Owner()->Pause();
StopIfNeeded();
}
}
void UpdateOwnerBrowsingContextIfNeeded() {
// Has not notified any information about the owner context yet.
if (!IsStarted()) {
return;
}
BrowsingContext* currentBC = GetCurrentBrowsingContext();
MOZ_ASSERT(currentBC);
// Still in the same browsing context, no need to update.
if (currentBC->Id() == mOwnerBrowsingContextId) {
return;
}
MEDIACONTROL_LOG("Change browsing context from %" PRIu64 " to %" PRIu64,
mOwnerBrowsingContextId, currentBC->Id());
// This situation would happen when we start a media in an original browsing
// context, then we move it to another browsing context, such as an iframe,
// so its owner browsing context would be changed. Therefore, we should
// reset the media status for the previous browsing context by calling
// `Stop()`, in which the listener would notify `ePaused` (if it's playing)
// and `eStop`. Then calls `Start()`, in which the listener would notify
// `eStart` to the new browsing context. If the media was playing before,
// we would also notify `ePlayed`.
bool wasInPlayingState = mState == MediaPlaybackState::ePlayed;
StopIfNeeded();
Start();
if (wasInPlayingState) {
NotifyMediaStartedPlaying();
}
}
private:
~MediaControlKeyListener() = default;
// The media can be moved around different browsing contexts, so this context
// might be different from the one that we used to initialize
// `ContentMediaAgent`.
BrowsingContext* GetCurrentBrowsingContext() const {
// Owner has been CCed, which would break the link of the weaker pointer.
if (!Owner()) {
return nullptr;
}
nsPIDOMWindowInner* window = Owner()->OwnerDoc()->GetInnerWindow();
return window ? window->GetBrowsingContext() : nullptr;
}
bool InitMediaAgent() {
MOZ_ASSERT(NS_IsMainThread());
BrowsingContext* currentBC = GetCurrentBrowsingContext();
mControlAgent = ContentMediaAgent::Get(currentBC);
if (!mControlAgent) {
return false;
}
MOZ_ASSERT(currentBC);
mOwnerBrowsingContextId = currentBC->Id();
MEDIACONTROL_LOG("Init agent in browsing context %" PRIu64,
mOwnerBrowsingContextId);
mControlAgent->AddReceiver(this);
return true;
}
HTMLMediaElement* Owner() const {
// `mElement` would be clear during CC unlinked, but it would only happen
// after stopping the listener.
MOZ_ASSERT(mElement || !IsStarted());
return mElement.get();
}
void NotifyPlaybackStateChanged(MediaPlaybackState aState) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mControlAgent);
MEDIACONTROL_LOG("NotifyMediaState from state='%s' to state='%s'",
ToMediaPlaybackStateStr(mState),
ToMediaPlaybackStateStr(aState));
MOZ_ASSERT(mState != aState, "Should not notify same state again!");
mState = aState;
mControlAgent->NotifyMediaPlaybackChanged(mOwnerBrowsingContextId, mState);
}
void NotifyAudibleStateChanged(MediaAudibleState aState) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(IsStarted());
mControlAgent->NotifyMediaAudibleChanged(mOwnerBrowsingContextId, aState);
}
MediaPlaybackState mState = MediaPlaybackState::eStopped;
WeakPtr<HTMLMediaElement> mElement;
RefPtr<ContentMediaAgent> mControlAgent;
bool mIsPictureInPictureEnabled = false;
bool mIsOwnerAudible = false;
MOZ_INIT_OUTSIDE_CTOR uint64_t mOwnerBrowsingContextId;
};
class HTMLMediaElement::MediaStreamTrackListener
: public DOMMediaStream::TrackListener {
public:
explicit MediaStreamTrackListener(HTMLMediaElement* aElement)
: mElement(aElement) {}
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrackListener,
DOMMediaStream::TrackListener)
void NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack) override {
if (!mElement) {
return;
}
mElement->NotifyMediaStreamTrackAdded(aTrack);
}
void NotifyTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack) override {
if (!mElement) {
return;
}
mElement->NotifyMediaStreamTrackRemoved(aTrack);
}
void OnActive() {
MOZ_ASSERT(mElement);
// mediacapture-main says:
// Note that once ended equals true the HTMLVideoElement will not play media
// even if new MediaStreamTracks are added to the MediaStream (causing it to
// return to the active state) unless autoplay is true or the web
// application restarts the element, e.g., by calling play().
//
// This is vague on exactly how to go from becoming active to playing, when
// autoplaying. However, per the media element spec, to play an autoplaying
// media element, we must load the source and reach readyState
// HAVE_ENOUGH_DATA [1]. Hence, a MediaStream being assigned to a media
// element and becoming active runs the load algorithm, so that it can
// eventually be played.
//
// [1]
LOG(LogLevel::Debug, ("%p, mSrcStream %p became active, checking if we "
"need to run the load algorithm",
mElement.get(), mElement->mSrcStream.get()));
if (!mElement->IsPlaybackEnded()) {
return;
}
if (!mElement->Autoplay()) {
return;
}
LOG(LogLevel::Info, ("%p, mSrcStream %p became active on autoplaying, "
"ended element. Reloading.",
mElement.get(), mElement->mSrcStream.get()));
mElement->DoLoad();
}
void NotifyActive() override {
if (!mElement) {
return;
}
if (!mElement->IsVideo()) {
// Audio elements use NotifyAudible().
return;
}
OnActive();
}
void NotifyAudible() override {
if (!mElement) {
return;
}
if (mElement->IsVideo()) {
// Video elements use NotifyActive().
return;
}
OnActive();
}
void OnInactive() {
MOZ_ASSERT(mElement);
if (mElement->IsPlaybackEnded()) {
return;
}
LOG(LogLevel::Debug, ("%p, mSrcStream %p became inactive", mElement.get(),
mElement->mSrcStream.get()));
mElement->PlaybackEnded();
}
void NotifyInactive() override {
if (!mElement) {
return;
}
if (!mElement->IsVideo()) {
// Audio elements use NotifyInaudible().
return;
}
OnInactive();
}
void NotifyInaudible() override {
if (!mElement) {
return;
}
if (mElement->IsVideo()) {
// Video elements use NotifyInactive().
return;
}
OnInactive();
}
protected:
virtual ~MediaStreamTrackListener() = default;
RefPtr<HTMLMediaElement> mElement;
};
NS_IMPL_CYCLE_COLLECTION_INHERITED(HTMLMediaElement::MediaStreamTrackListener,
DOMMediaStream::TrackListener, mElement)
NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::MediaStreamTrackListener,
DOMMediaStream::TrackListener)
NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::MediaStreamTrackListener,
DOMMediaStream::TrackListener)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
HTMLMediaElement::MediaStreamTrackListener)
NS_INTERFACE_MAP_END_INHERITING(DOMMediaStream::TrackListener)
/**
* Helper class that manages audio and video outputs for all enabled tracks in a
* media element. It also manages calculating the current time when playing a
* MediaStream.
*/
class HTMLMediaElement::MediaStreamRenderer {
public:
NS_INLINE_DECL_REFCOUNTING(MediaStreamRenderer)
MediaStreamRenderer(AbstractThread* aMainThread,
VideoFrameContainer* aVideoContainer,
FirstFrameVideoOutput* aFirstFrameVideoOutput,
void* aAudioOutputKey)
: mVideoContainer(aVideoContainer),
mAudioOutputKey(aAudioOutputKey),
mWatchManager(this, aMainThread),
mFirstFrameVideoOutput(aFirstFrameVideoOutput) {
if (mFirstFrameVideoOutput) {
mWatchManager.Watch(mFirstFrameVideoOutput->mFirstFrameRendered,
&MediaStreamRenderer::SetFirstFrameRendered);
}
}
void Shutdown() {
for (const auto& t : mAudioTracks.Clone()) {
if (t) {
RemoveTrack(t->AsAudioStreamTrack());
}
}
if (mVideoTrack) {
RemoveTrack(mVideoTrack->AsVideoStreamTrack());
}
mWatchManager.Shutdown();
mFirstFrameVideoOutput = nullptr;
}
void UpdateGraphTime() {
mGraphTime =
mGraphTimeDummy->mTrack->Graph()->CurrentTime() - *mGraphTimeOffset;
}
void SetFirstFrameRendered() {
if (!mFirstFrameVideoOutput) {
return;
}
if (mVideoTrack) {
mVideoTrack->AsVideoStreamTrack()->RemoveVideoOutput(
mFirstFrameVideoOutput);
}
mWatchManager.Unwatch(mFirstFrameVideoOutput->mFirstFrameRendered,
&MediaStreamRenderer::SetFirstFrameRendered);
mFirstFrameVideoOutput = nullptr;
}
void SetProgressingCurrentTime(bool aProgress) {
if (aProgress == mProgressingCurrentTime) {
return;
}
MOZ_DIAGNOSTIC_ASSERT(mGraphTimeDummy);
mProgressingCurrentTime = aProgress;
MediaTrackGraph* graph = mGraphTimeDummy->mTrack->Graph();
if (mProgressingCurrentTime) {
mGraphTimeOffset = Some(graph->CurrentTime().Ref() - mGraphTime);
mWatchManager.Watch(graph->CurrentTime(),
&MediaStreamRenderer::UpdateGraphTime);
} else {
mWatchManager.Unwatch(graph->CurrentTime(),
&MediaStreamRenderer::UpdateGraphTime);
}
}
void Start() {
if (mRendering) {
return;
}
LOG(LogLevel::Info, ("MediaStreamRenderer=%p Start", this));
mRendering = true;
if (!mGraphTimeDummy) {
return;
}
for (const auto& t : mAudioTracks) {
if (t) {
t->AsAudioStreamTrack()->AddAudioOutput(mAudioOutputKey,
mAudioOutputSink);
t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
mAudioOutputVolume);
}
}
if (mVideoTrack) {
mVideoTrack->AsVideoStreamTrack()->AddVideoOutput(mVideoContainer);
}
}
void Stop() {
if (!mRendering) {
return;
}
LOG(LogLevel::Info, ("MediaStreamRenderer=%p Stop", this));
mRendering = false;
if (!mGraphTimeDummy) {
return;
}
for (const auto& t : mAudioTracks) {
if (t) {
t->AsAudioStreamTrack()->RemoveAudioOutput(mAudioOutputKey);
}
}
// There is no longer an audio output that needs the device so the
// device may not start. Ensure the promise is resolved.
ResolveAudioDevicePromiseIfExists(__func__);
if (mVideoTrack) {
mVideoTrack->AsVideoStreamTrack()->RemoveVideoOutput(mVideoContainer);
}
}
void SetAudioOutputVolume(float aVolume) {
if (mAudioOutputVolume == aVolume) {
return;
}
mAudioOutputVolume = aVolume;
if (!mRendering) {
return;
}
for (const auto& t : mAudioTracks) {
if (t) {
t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
mAudioOutputVolume);
}
}
}
RefPtr<GenericPromise> SetAudioOutputDevice(AudioDeviceInfo* aSink) {
MOZ_ASSERT(aSink);
MOZ_ASSERT(mAudioOutputSink != aSink);
LOG(LogLevel::Info,
("MediaStreamRenderer=%p SetAudioOutputDevice name=%s\n", this,
NS_ConvertUTF16toUTF8(aSink->Name()).get()));
mAudioOutputSink = aSink;
if (!mRendering) {
MOZ_ASSERT(mSetAudioDevicePromise.IsEmpty());
return GenericPromise::CreateAndResolve(true, __func__);
}
nsTArray<RefPtr<GenericPromise>> promises;
for (const auto& t : mAudioTracks) {
t->AsAudioStreamTrack()->RemoveAudioOutput(mAudioOutputKey);
promises.AppendElement(t->AsAudioStreamTrack()->AddAudioOutput(
mAudioOutputKey, mAudioOutputSink));
t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
mAudioOutputVolume);
}
if (!promises.Length()) {
// Not active track, save it for later
MOZ_ASSERT(mSetAudioDevicePromise.IsEmpty());
return GenericPromise::CreateAndResolve(true, __func__);
}
// Resolve any existing promise for a previous device so that promises
// resolve in order of setSinkId() invocation.
ResolveAudioDevicePromiseIfExists(__func__);
RefPtr promise = mSetAudioDevicePromise.Ensure(__func__);
GenericPromise::AllSettled(GetCurrentSerialEventTarget(), promises)
->Then(GetMainThreadSerialEventTarget(), __func__,
[self = RefPtr{this},
this](const GenericPromise::AllSettledPromiseType::
ResolveOrRejectValue& aValue) {
// This handler should have been disconnected if
// mSetAudioDevicePromise has been settled.
MOZ_ASSERT(!mSetAudioDevicePromise.IsEmpty());
mDeviceStartedRequest.Complete();
// The AudioStreamTrack::AddAudioOutput() promise is rejected
// either when the graph no longer needs the device, in which
// case this handler would have already been disconnected, or
// the graph is force shutdown.
// mSetAudioDevicePromise is resolved regardless of whether
// the AddAudioOutput() promises resolve or reject because
// the underlying device has been changed.
LOG(LogLevel::Info,
("MediaStreamRenderer=%p SetAudioOutputDevice settled",
this));
mSetAudioDevicePromise.Resolve(true, __func__);
})
->Track(mDeviceStartedRequest);
return promise;
}
void AddTrack(AudioStreamTrack* aTrack) {
MOZ_DIAGNOSTIC_ASSERT(!mAudioTracks.Contains(aTrack));
mAudioTracks.AppendElement(aTrack);
EnsureGraphTimeDummy();
if (mRendering) {
aTrack->AddAudioOutput(mAudioOutputKey, mAudioOutputSink);
aTrack->SetAudioOutputVolume(mAudioOutputKey, mAudioOutputVolume);
}
}
void AddTrack(VideoStreamTrack* aTrack) {
MOZ_DIAGNOSTIC_ASSERT(!mVideoTrack);
if (!mVideoContainer) {
return;
}
mVideoTrack = aTrack;
EnsureGraphTimeDummy();
if (mFirstFrameVideoOutput) {
// Add the first frame output even if we are rendering. It will only
// accept one frame. If we are rendering, then the main output will
// overwrite that with the same frame (and possibly more frames).
aTrack->AddVideoOutput(mFirstFrameVideoOutput);
}
if (mRendering) {
aTrack->AddVideoOutput(mVideoContainer);
}
}
void RemoveTrack(AudioStreamTrack* aTrack) {
MOZ_DIAGNOSTIC_ASSERT(mAudioTracks.Contains(aTrack));
if (mRendering) {
aTrack->RemoveAudioOutput(mAudioOutputKey);
}
mAudioTracks.RemoveElement(aTrack);
if (mAudioTracks.IsEmpty()) {
// There is no longer an audio output that needs the device so the
// device may not start. Ensure the promise is resolved.
ResolveAudioDevicePromiseIfExists(__func__);
}
}
void RemoveTrack(VideoStreamTrack* aTrack) {
MOZ_DIAGNOSTIC_ASSERT(mVideoTrack == aTrack);
if (!mVideoContainer) {
return;
}
if (mFirstFrameVideoOutput) {
aTrack->RemoveVideoOutput(mFirstFrameVideoOutput);
}
if (mRendering) {
aTrack->RemoveVideoOutput(mVideoContainer);
}
mVideoTrack = nullptr;
}
double CurrentTime() const {
if (!mGraphTimeDummy) {
return 0.0;
}
return mGraphTimeDummy->mTrack->GraphImpl()->MediaTimeToSeconds(mGraphTime);
}
Watchable<GraphTime>& CurrentGraphTime() { return mGraphTime; }
// Set if we're rendering video.
const RefPtr<VideoFrameContainer> mVideoContainer;
// Set if we're rendering audio, nullptr otherwise.
void* const mAudioOutputKey;
private:
~MediaStreamRenderer() { Shutdown(); }
void EnsureGraphTimeDummy() {
if (mGraphTimeDummy) {
return;
}
MediaTrackGraph* graph = nullptr;
for (const auto& t : mAudioTracks) {
if (t && !t->Ended()) {
graph = t->Graph();
break;
}
}
if (!graph && mVideoTrack && !mVideoTrack->Ended()) {
graph = mVideoTrack->Graph();
}
if (!graph) {
return;
}
// This dummy keeps `graph` alive and ensures access to it.
mGraphTimeDummy = MakeRefPtr<SharedDummyTrack>(
graph->CreateSourceTrack(MediaSegment::AUDIO));
}
void ResolveAudioDevicePromiseIfExists(const char* aMethodName) {
if (mSetAudioDevicePromise.IsEmpty()) {
return;
}
LOG(LogLevel::Info,
("MediaStreamRenderer=%p resolve audio device promise", this));
mSetAudioDevicePromise.Resolve(true, aMethodName);
mDeviceStartedRequest.Disconnect();
}
// True when all tracks are being rendered, i.e., when the media element is
// playing.
bool mRendering = false;
// True while we're progressing mGraphTime. False otherwise.
bool mProgressingCurrentTime = false;
// The audio output volume for all audio tracks.
float mAudioOutputVolume = 1.0f;
// The sink device for all audio tracks.
RefPtr<AudioDeviceInfo> mAudioOutputSink;
// The promise returned from SetAudioOutputDevice() when an output is
// active.
MozPromiseHolder<GenericPromise> mSetAudioDevicePromise;
// Request tracking the promise to indicate when the device passed to
// SetAudioOutputDevice() is running.
MozPromiseRequestHolder<GenericPromise::AllSettledPromiseType>
mDeviceStartedRequest;
// WatchManager for mGraphTime.
WatchManager<MediaStreamRenderer> mWatchManager;
// A dummy MediaTrack to guarantee a MediaTrackGraph is kept alive while
// we're actively rendering, so we can track the graph's current time. Set
// when the first track is added, never unset.
RefPtr<SharedDummyTrack> mGraphTimeDummy;
// Watchable that relays the graph's currentTime updates to the media element
// only while we're rendering. This is the current time of the rendering in
// GraphTime units.
Watchable<GraphTime> mGraphTime = {0, "MediaStreamRenderer::mGraphTime"};
// Nothing until a track has been added. Then, the current GraphTime at the
// time when we were last Start()ed.
Maybe<GraphTime> mGraphTimeOffset;
// Currently enabled (and rendered) audio tracks.
nsTArray<WeakPtr<MediaStreamTrack>> mAudioTracks;
// Currently selected (and rendered) video track.
WeakPtr<MediaStreamTrack> mVideoTrack;
// Holds a reference to the first-frame-getting video output attached to
// mVideoTrack. Set by the constructor, unset when the media element tells us
// it has rendered the first frame.
RefPtr<FirstFrameVideoOutput> mFirstFrameVideoOutput;
};
static uint32_t sDecoderCaptureSourceId = 0;
static uint32_t sStreamCaptureSourceId = 0;
class HTMLMediaElement::MediaElementTrackSource
: public MediaStreamTrackSource,
public MediaStreamTrackSource::Sink,
public MediaStreamTrackConsumer {
public:
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource,
MediaStreamTrackSource)
/* MediaDecoder track source */
MediaElementTrackSource(ProcessedMediaTrack* aTrack, nsIPrincipal* aPrincipal,
OutputMuteState aMuteState, bool aHasAlpha)
: MediaStreamTrackSource(
aPrincipal, nsString(),
TrackingId(TrackingId::Source::MediaElementDecoder,
sDecoderCaptureSourceId++,
TrackingId::TrackAcrossProcesses::Yes)),
mTrack(aTrack),
mIntendedElementMuteState(aMuteState),
mElementMuteState(aMuteState),
mMediaDecoderHasAlpha(Some(aHasAlpha)) {
MOZ_ASSERT(mTrack);
}
/* MediaStream track source */
MediaElementTrackSource(MediaStreamTrack* aCapturedTrack,
MediaStreamTrackSource* aCapturedTrackSource,
ProcessedMediaTrack* aTrack, MediaInputPort* aPort,
OutputMuteState aMuteState)
: MediaStreamTrackSource(
aCapturedTrackSource->GetPrincipal(), nsString(),
TrackingId(TrackingId::Source::MediaElementStream,
sStreamCaptureSourceId++,
TrackingId::TrackAcrossProcesses::Yes)),
mCapturedTrack(aCapturedTrack),
mCapturedTrackSource(aCapturedTrackSource),
mTrack(aTrack),
mPort(aPort),
mIntendedElementMuteState(aMuteState),
mElementMuteState(aMuteState) {
MOZ_ASSERT(mTrack);
MOZ_ASSERT(mCapturedTrack);
MOZ_ASSERT(mCapturedTrackSource);
MOZ_ASSERT(mPort);
mCapturedTrack->AddConsumer(this);
mCapturedTrackSource->RegisterSink(this);
}
void SetEnabled(bool aEnabled) {
if (!mTrack) {
return;
}
mTrack->SetDisabledTrackMode(aEnabled ? DisabledTrackMode::ENABLED
: DisabledTrackMode::SILENCE_FREEZE);
}
void SetPrincipal(RefPtr<nsIPrincipal> aPrincipal) {
mPrincipal = std::move(aPrincipal);
MediaStreamTrackSource::PrincipalChanged();
}
void SetMutedByElement(OutputMuteState aMuteState) {
if (mIntendedElementMuteState == aMuteState) {
return;
}
mIntendedElementMuteState = aMuteState;
GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
"MediaElementTrackSource::SetMutedByElement",
[self = RefPtr<MediaElementTrackSource>(this), this, aMuteState] {
mElementMuteState = aMuteState;
MediaStreamTrackSource::MutedChanged(Muted());
}));
}
void Destroy() override {
if (mCapturedTrack) {
mCapturedTrack->RemoveConsumer(this);
mCapturedTrack = nullptr;
}
if (mCapturedTrackSource) {
mCapturedTrackSource->UnregisterSink(this);
mCapturedTrackSource = nullptr;
}
if (mTrack && !mTrack->IsDestroyed()) {
mTrack->Destroy();
}
if (mPort) {
mPort->Destroy();
mPort = nullptr;
}
}
MediaSourceEnum GetMediaSource() const override {
return MediaSourceEnum::Other;
}
void Stop() override {
// Do nothing. There may appear new output streams
// that need tracks sourced from this source, so we
// cannot destroy things yet.
}
/**
* Do not keep the track source alive. The source lifetime is controlled by
* its associated tracks.
*/
bool KeepsSourceAlive() const override { return false; }
/**
* Do not keep the track source on. It is controlled by its associated tracks.
*/
bool Enabled() const override { return false; }
void Disable() override {}
void Enable() override {}
void PrincipalChanged() override {
if (!mCapturedTrackSource) {
// This could happen during shutdown.
return;
}
SetPrincipal(mCapturedTrackSource->GetPrincipal());
}
void MutedChanged(bool aNewState) override {
MediaStreamTrackSource::MutedChanged(Muted());
}
void OverrideEnded() override {
Destroy();
MediaStreamTrackSource::OverrideEnded();
}
void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled) override {
MediaStreamTrackSource::MutedChanged(Muted());
}
bool Muted() const {
return mElementMuteState == OutputMuteState::Muted ||
(mCapturedTrack &&
(mCapturedTrack->Muted() || !mCapturedTrack->Enabled()));
}
bool HasAlpha() const override {
if (mCapturedTrack) {
return mCapturedTrack->AsVideoStreamTrack()
? mCapturedTrack->AsVideoStreamTrack()->HasAlpha()
: false;
}
return mMediaDecoderHasAlpha.valueOr(false);
}
ProcessedMediaTrack* Track() const { return mTrack; }
private:
virtual ~MediaElementTrackSource() { Destroy(); };
RefPtr<MediaStreamTrack> mCapturedTrack;
RefPtr<MediaStreamTrackSource> mCapturedTrackSource;
const RefPtr<ProcessedMediaTrack> mTrack;
RefPtr<MediaInputPort> mPort;
// The mute state as intended by the media element.
OutputMuteState mIntendedElementMuteState;
// The mute state as applied to this track source. It is applied async, so
// needs to be tracked separately from the intended state.
OutputMuteState mElementMuteState;
// Some<bool> if this is a MediaDecoder track source.
const Maybe<bool> mMediaDecoderHasAlpha;
};
HTMLMediaElement::OutputMediaStream::OutputMediaStream(
RefPtr<DOMMediaStream> aStream, bool aCapturingAudioOnly,
bool aFinishWhenEnded)
: mStream(std::move(aStream)),
mCapturingAudioOnly(aCapturingAudioOnly),
mFinishWhenEnded(aFinishWhenEnded) {}
HTMLMediaElement::OutputMediaStream::~OutputMediaStream() = default;
void ImplCycleCollectionTraverse(nsCycleCollectionTraversalCallback& aCallback,
HTMLMediaElement::OutputMediaStream& aField,
const char* aName, uint32_t aFlags) {
ImplCycleCollectionTraverse(aCallback, aField.mStream, "mStream", aFlags);
ImplCycleCollectionTraverse(aCallback, aField.mLiveTracks, "mLiveTracks",
aFlags);
ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedLoadingSrc,
"mFinishWhenEndedLoadingSrc", aFlags);
ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedAttrStream,
"mFinishWhenEndedAttrStream", aFlags);
ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedMediaSource,
"mFinishWhenEndedMediaSource", aFlags);
}
void ImplCycleCollectionUnlink(HTMLMediaElement::OutputMediaStream& aField) {
ImplCycleCollectionUnlink(aField.mStream);
ImplCycleCollectionUnlink(aField.mLiveTracks);
ImplCycleCollectionUnlink(aField.mFinishWhenEndedLoadingSrc);
ImplCycleCollectionUnlink(aField.mFinishWhenEndedAttrStream);
ImplCycleCollectionUnlink(aField.mFinishWhenEndedMediaSource);
}
NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::MediaElementTrackSource,
MediaStreamTrackSource)
NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::MediaElementTrackSource,
MediaStreamTrackSource)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
HTMLMediaElement::MediaElementTrackSource)
NS_INTERFACE_MAP_END_INHERITING(MediaStreamTrackSource)
NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::MediaElementTrackSource)
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(
HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
tmp->Destroy();
NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrack)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrackSource)
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(
HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrack)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrackSource)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
/**
* There is a reference cycle involving this class: MediaLoadListener
* holds a reference to the HTMLMediaElement, which holds a reference
* to an nsIChannel, which holds a reference to this listener.
* We break the reference cycle in OnStartRequest by clearing mElement.
*/
class HTMLMediaElement::MediaLoadListener final
: public nsIChannelEventSink,
public nsIInterfaceRequestor,
public nsIObserver,
public nsIThreadRetargetableStreamListener {
~MediaLoadListener() = default;
NS_DECL_THREADSAFE_ISUPPORTS
NS_DECL_NSIREQUESTOBSERVER
NS_DECL_NSISTREAMLISTENER
NS_DECL_NSICHANNELEVENTSINK
NS_DECL_NSIOBSERVER
NS_DECL_NSIINTERFACEREQUESTOR
NS_DECL_NSITHREADRETARGETABLESTREAMLISTENER
public:
explicit MediaLoadListener(HTMLMediaElement* aElement)
: mElement(aElement), mLoadID(aElement->GetCurrentLoadID()) {
MOZ_ASSERT(mElement, "Must pass an element to call back");
}
private:
RefPtr<HTMLMediaElement> mElement;
nsCOMPtr<nsIStreamListener> mNextListener;
const uint32_t mLoadID;
};
NS_IMPL_ISUPPORTS(HTMLMediaElement::MediaLoadListener, nsIRequestObserver,
nsIStreamListener, nsIChannelEventSink, nsIInterfaceRequestor,
nsIObserver, nsIThreadRetargetableStreamListener)
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::Observe(nsISupports* aSubject,
const char* aTopic,
const char16_t* aData) {
nsContentUtils::UnregisterShutdownObserver(this);
// Clear mElement to break cycle so we don't leak on shutdown
mElement = nullptr;
return NS_OK;
}
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::OnStartRequest(nsIRequest* aRequest) {
nsContentUtils::UnregisterShutdownObserver(this);
if (!mElement) {
// We've been notified by the shutdown observer, and are shutting down.
return NS_BINDING_ABORTED;
}
// The element is only needed until we've had a chance to call
// InitializeDecoderForChannel. So make sure mElement is cleared here.
RefPtr<HTMLMediaElement> element;
element.swap(mElement);
if (mLoadID != element->GetCurrentLoadID()) {
// The channel has been cancelled before we had a chance to create
// a decoder. Abort, don't dispatch an "error" event, as the new load
// may not be in an error state.
return NS_BINDING_ABORTED;
}
// Don't continue to load if the request failed or has been canceled.
nsresult status;
nsresult rv = aRequest->GetStatus(&status);
NS_ENSURE_SUCCESS(rv, rv);
if (NS_FAILED(status)) {
if (element) {
// Handle media not loading error because source was a tracking URL (or
// fingerprinting, cryptomining, etc).
// We make a note of this media node by including it in a dedicated
// array of blocked tracking nodes under its parent document.
if (net::UrlClassifierFeatureFactory::IsClassifierBlockingErrorCode(
status)) {
element->OwnerDoc()->AddBlockedNodeByClassifier(element);
}
element->NotifyLoadError(
nsPrintfCString("%u: %s", uint32_t(status), "Request failed"));
}
return status;
}
nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(aRequest);
bool succeeded;
if (hc && NS_SUCCEEDED(hc->GetRequestSucceeded(&succeeded)) && !succeeded) {
uint32_t responseStatus = 0;
Unused << hc->GetResponseStatus(&responseStatus);
nsAutoCString statusText;
Unused << hc->GetResponseStatusText(statusText);
// we need status text for resist fingerprinting mode's message allowlist
if (statusText.IsEmpty()) {
net_GetDefaultStatusTextForCode(responseStatus, statusText);
}
element->NotifyLoadError(
nsPrintfCString("%u: %s", responseStatus, statusText.get()));
nsAutoString code;
code.AppendInt(responseStatus);
nsAutoString src;
element->GetCurrentSrc(src);
AutoTArray<nsString, 2> params = {code, src};
element->ReportLoadError("MediaLoadHttpError", params);
return NS_BINDING_ABORTED;
}
nsCOMPtr<nsIChannel> channel = do_QueryInterface(aRequest);
if (channel &&
NS_SUCCEEDED(rv = element->InitializeDecoderForChannel(
channel, getter_AddRefs(mNextListener))) &&
mNextListener) {
rv = mNextListener->OnStartRequest(aRequest);
} else {
// If InitializeDecoderForChannel() returned an error, fire a network error.
if (NS_FAILED(rv) && !mNextListener) {
// Load failed, attempt to load the next candidate resource. If there
// are none, this will trigger a MEDIA_ERR_SRC_NOT_SUPPORTED error.
element->NotifyLoadError("Failed to init decoder"_ns);
}
// If InitializeDecoderForChannel did not return a listener (but may
// have otherwise succeeded), we abort the connection since we aren't
// interested in keeping the channel alive ourselves.
rv = NS_BINDING_ABORTED;
}
return rv;
}
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::OnStopRequest(nsIRequest* aRequest,
nsresult aStatus) {
if (mNextListener) {
return mNextListener->OnStopRequest(aRequest, aStatus);
}
return NS_OK;
}
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::OnDataAvailable(nsIRequest* aRequest,
nsIInputStream* aStream,
uint64_t aOffset,
uint32_t aCount) {
if (!mNextListener) {
NS_ERROR(
"Must have a chained listener; OnStartRequest should have "
"canceled this request");
return NS_BINDING_ABORTED;
}
return mNextListener->OnDataAvailable(aRequest, aStream, aOffset, aCount);
}
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::OnDataFinished(nsresult aStatus) {
if (!mNextListener) {
return NS_ERROR_FAILURE;
}
nsCOMPtr<nsIThreadRetargetableStreamListener> retargetable =
do_QueryInterface(mNextListener);
if (retargetable) {
return retargetable->OnDataFinished(aStatus);
}
return NS_OK;
}
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::AsyncOnChannelRedirect(
nsIChannel* aOldChannel, nsIChannel* aNewChannel, uint32_t aFlags,
nsIAsyncVerifyRedirectCallback* cb) {
// TODO is this really correct?? See bug #579329.
if (mElement) {
mElement->OnChannelRedirect(aOldChannel, aNewChannel, aFlags);
}
nsCOMPtr<nsIChannelEventSink> sink = do_QueryInterface(mNextListener);
if (sink) {
return sink->AsyncOnChannelRedirect(aOldChannel, aNewChannel, aFlags, cb);
}
cb->OnRedirectVerifyCallback(NS_OK);
return NS_OK;
}
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::CheckListenerChain() {
MOZ_ASSERT(mNextListener);
nsCOMPtr<nsIThreadRetargetableStreamListener> retargetable =
do_QueryInterface(mNextListener);
if (retargetable) {
return retargetable->CheckListenerChain();
}
return NS_ERROR_NO_INTERFACE;
}
NS_IMETHODIMP
HTMLMediaElement::MediaLoadListener::GetInterface(const nsIID& aIID,
void** aResult) {
return QueryInterface(aIID, aResult);
}
void HTMLMediaElement::ReportLoadError(const char* aMsg,
const nsTArray<nsString>& aParams) {
ReportToConsole(nsIScriptError::warningFlag, aMsg, aParams);
}
void HTMLMediaElement::ReportToConsole(
uint32_t aErrorFlags, const char* aMsg,
const nsTArray<nsString>& aParams) const {
nsContentUtils::ReportToConsole(aErrorFlags, "Media"_ns, OwnerDoc(),
nsContentUtils::eDOM_PROPERTIES, aMsg,
aParams);
}
class HTMLMediaElement::AudioChannelAgentCallback final
: public nsIAudioChannelAgentCallback {
public:
NS_DECL_CYCLE_COLLECTING_ISUPPORTS
NS_DECL_CYCLE_COLLECTION_CLASS(AudioChannelAgentCallback)
explicit AudioChannelAgentCallback(HTMLMediaElement* aOwner)
: mOwner(aOwner),
mAudioChannelVolume(1.0),
mPlayingThroughTheAudioChannel(false),
mIsOwnerAudible(IsOwnerAudible()),
mIsShutDown(false) {
MOZ_ASSERT(mOwner);
MaybeCreateAudioChannelAgent();
}
void UpdateAudioChannelPlayingState() {
MOZ_ASSERT(!mIsShutDown);
bool playingThroughTheAudioChannel = IsPlayingThroughTheAudioChannel();
if (playingThroughTheAudioChannel != mPlayingThroughTheAudioChannel) {
if (!MaybeCreateAudioChannelAgent()) {
return;
}
mPlayingThroughTheAudioChannel = playingThroughTheAudioChannel;
if (mPlayingThroughTheAudioChannel) {
StartAudioChannelAgent();
} else {
StopAudioChanelAgent();
}
}
}
void NotifyPlayStateChanged() {
MOZ_ASSERT(!mIsShutDown);
UpdateAudioChannelPlayingState();
}
NS_IMETHODIMP WindowVolumeChanged(float aVolume, bool aMuted) override {
MOZ_ASSERT(mAudioChannelAgent);
MOZ_LOG(
AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
("HTMLMediaElement::AudioChannelAgentCallback, WindowVolumeChanged, "
"this = %p, aVolume = %f, aMuted = %s\n",
this, aVolume, aMuted ? "true" : "false"));
if (mAudioChannelVolume != aVolume) {
mAudioChannelVolume = aVolume;
mOwner->SetVolumeInternal();
}
const uint32_t muted = mOwner->mMuted;
if (aMuted && !mOwner->ComputedMuted()) {
mOwner->SetMutedInternal(muted | MUTED_BY_AUDIO_CHANNEL);
} else if (!aMuted && mOwner->ComputedMuted()) {
mOwner->SetMutedInternal(muted & ~MUTED_BY_AUDIO_CHANNEL);
}
return NS_OK;
}
NS_IMETHODIMP WindowSuspendChanged(SuspendTypes aSuspend) override {
// Currently this method is only be used for delaying autoplay, and we've
// separated related codes to `MediaPlaybackDelayPolicy`.
return NS_OK;
}
NS_IMETHODIMP WindowAudioCaptureChanged(bool aCapture) override {
MOZ_ASSERT(mAudioChannelAgent);
AudioCaptureTrackChangeIfNeeded();
return NS_OK;
}
void AudioCaptureTrackChangeIfNeeded() {
MOZ_ASSERT(!mIsShutDown);
if (!IsPlayingStarted()) {
return;
}
MOZ_ASSERT(mAudioChannelAgent);
bool isCapturing = mAudioChannelAgent->IsWindowAudioCapturingEnabled();
mOwner->AudioCaptureTrackChange(isCapturing);
}
void NotifyAudioPlaybackChanged(AudibleChangedReasons aReason) {
MOZ_ASSERT(!mIsShutDown);
AudibleState newAudibleState = IsOwnerAudible();
MOZ_LOG(AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
("HTMLMediaElement::AudioChannelAgentCallback, "
"NotifyAudioPlaybackChanged, this=%p, current=%s, new=%s",
this, AudibleStateToStr(mIsOwnerAudible),
AudibleStateToStr(newAudibleState)));
if (mIsOwnerAudible == newAudibleState) {
return;
}
mIsOwnerAudible = newAudibleState;
if (IsPlayingStarted()) {
mAudioChannelAgent->NotifyStartedAudible(mIsOwnerAudible, aReason);
}
}
void Shutdown() {
MOZ_ASSERT(!mIsShutDown);
if (mAudioChannelAgent && mAudioChannelAgent->IsPlayingStarted()) {
StopAudioChanelAgent();
}
mAudioChannelAgent = nullptr;
mIsShutDown = true;
}
float GetEffectiveVolume() const {
MOZ_ASSERT(!mIsShutDown);
return static_cast<float>(mOwner->Volume()) * mAudioChannelVolume;
}
private:
~AudioChannelAgentCallback() { MOZ_ASSERT(mIsShutDown); };
bool MaybeCreateAudioChannelAgent() {
if (mAudioChannelAgent) {
return true;
}
mAudioChannelAgent = new AudioChannelAgent();
nsresult rv =
mAudioChannelAgent->Init(mOwner->OwnerDoc()->GetInnerWindow(), this);
if (NS_WARN_IF(NS_FAILED(rv))) {
mAudioChannelAgent = nullptr;
MOZ_LOG(
AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
("HTMLMediaElement::AudioChannelAgentCallback, Fail to initialize "
"the audio channel agent, this = %p\n",
this));
return false;
}
return true;
}
void StartAudioChannelAgent() {
MOZ_ASSERT(mAudioChannelAgent);
MOZ_ASSERT(!mAudioChannelAgent->IsPlayingStarted());
if (NS_WARN_IF(NS_FAILED(
mAudioChannelAgent->NotifyStartedPlaying(IsOwnerAudible())))) {
return;
}
mAudioChannelAgent->PullInitialUpdate();
}
void StopAudioChanelAgent() {
MOZ_ASSERT(mAudioChannelAgent);
MOZ_ASSERT(mAudioChannelAgent->IsPlayingStarted());
mAudioChannelAgent->NotifyStoppedPlaying();
// If we have started audio capturing before, we have to tell media element
// to clear the output capturing track.
mOwner->AudioCaptureTrackChange(false);
}
bool IsPlayingStarted() {
if (MaybeCreateAudioChannelAgent()) {
return mAudioChannelAgent->IsPlayingStarted();
}
return false;
}
AudibleState IsOwnerAudible() const {
// paused media doesn't produce any sound.
if (mOwner->mPaused) {
return AudibleState::eNotAudible;
}
return mOwner->IsAudible() ? AudibleState::eAudible
: AudibleState::eNotAudible;
}
bool IsPlayingThroughTheAudioChannel() const {
// If we have an error, we are not playing.
if (mOwner->GetError()) {
return false;
}
// We should consider any bfcached page or inactive document as non-playing.
if (!mOwner->OwnerDoc()->IsActive()) {
return false;
}
// Media is suspended by the docshell.
if (mOwner->ShouldBeSuspendedByInactiveDocShell()) {
return false;
}
// Are we paused
if (mOwner->mPaused) {
return false;
}
// No audio track
if (!mOwner->HasAudio()) {
return false;
}
// A loop always is playing
if (mOwner->HasAttr(nsGkAtoms::loop)) {
return true;
}
// If we are actually playing...
if (mOwner->IsCurrentlyPlaying()) {
return true;
}
// If we are playing an external stream.
if (mOwner->mSrcAttrStream) {
return true;
}
return false;
}
RefPtr<AudioChannelAgent> mAudioChannelAgent;
HTMLMediaElement* mOwner;
// The audio channel volume
float mAudioChannelVolume;
// Is this media element playing?
bool mPlayingThroughTheAudioChannel;
// Indicate whether media element is audible for users.
AudibleState mIsOwnerAudible;
bool mIsShutDown;
};
NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::AudioChannelAgentCallback)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(
HTMLMediaElement::AudioChannelAgentCallback)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelAgent)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(
HTMLMediaElement::AudioChannelAgentCallback)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelAgent)
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
HTMLMediaElement::AudioChannelAgentCallback)
NS_INTERFACE_MAP_ENTRY(nsIAudioChannelAgentCallback)
NS_INTERFACE_MAP_END
NS_IMPL_CYCLE_COLLECTING_ADDREF(HTMLMediaElement::AudioChannelAgentCallback)
NS_IMPL_CYCLE_COLLECTING_RELEASE(HTMLMediaElement::AudioChannelAgentCallback)
class HTMLMediaElement::ChannelLoader final {
public:
NS_INLINE_DECL_REFCOUNTING(ChannelLoader);
void LoadInternal(HTMLMediaElement* aElement) {
if (mCancelled) {
return;
}
// determine what security checks need to be performed in AsyncOpen().
nsSecurityFlags securityFlags =
aElement->ShouldCheckAllowOrigin()
? nsILoadInfo::SEC_REQUIRE_CORS_INHERITS_SEC_CONTEXT
: nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT;
if (aElement->GetCORSMode() == CORS_USE_CREDENTIALS) {
securityFlags |= nsILoadInfo::SEC_COOKIES_INCLUDE;
}
securityFlags |= nsILoadInfo::SEC_ALLOW_CHROME;
MOZ_ASSERT(
aElement->IsAnyOfHTMLElements(nsGkAtoms::audio, nsGkAtoms::video));
nsContentPolicyType contentPolicyType =
aElement->IsHTMLElement(nsGkAtoms::audio)
? nsIContentPolicy::TYPE_INTERNAL_AUDIO
: nsIContentPolicy::TYPE_INTERNAL_VIDEO;
// If aElement has 'triggeringprincipal' attribute, we will use the value as
// triggeringPrincipal for the channel, otherwise it will default to use
// aElement->NodePrincipal().
// This function returns true when aElement has 'triggeringprincipal', so if
// setAttrs is true we will override the origin attributes on the channel
// later.
nsCOMPtr<nsIPrincipal> triggeringPrincipal;
bool setAttrs = nsContentUtils::QueryTriggeringPrincipal(
aElement, aElement->mLoadingSrcTriggeringPrincipal,
getter_AddRefs(triggeringPrincipal));
nsCOMPtr<nsILoadGroup> loadGroup = aElement->GetDocumentLoadGroup();
nsCOMPtr<nsIChannel> channel;
nsresult rv = NS_NewChannelWithTriggeringPrincipal(
getter_AddRefs(channel), aElement->mLoadingSrc,
static_cast<Element*>(aElement), triggeringPrincipal, securityFlags,
contentPolicyType,
nullptr, // aPerformanceStorage
loadGroup,
nullptr, // aCallbacks
nsICachingChannel::LOAD_BYPASS_LOCAL_CACHE_IF_BUSY |
nsIChannel::LOAD_MEDIA_SNIFFER_OVERRIDES_CONTENT_TYPE |
nsIChannel::LOAD_CALL_CONTENT_SNIFFERS);
if (NS_FAILED(rv)) {
// Notify load error so the element will try next resource candidate.
aElement->NotifyLoadError("Fail to create channel"_ns);
return;
}
nsCOMPtr<nsILoadInfo> loadInfo = channel->LoadInfo();
if (setAttrs) {
// The function simply returns NS_OK, so we ignore the return value.
Unused << loadInfo->SetOriginAttributes(
triggeringPrincipal->OriginAttributesRef());
}
loadInfo->SetIsMediaRequest(true);
loadInfo->SetIsMediaInitialRequest(true);
nsCOMPtr<nsIClassOfService> cos(do_QueryInterface(channel));
if (cos) {
if (aElement->mUseUrgentStartForChannel) {
cos->AddClassFlags(nsIClassOfService::UrgentStart);
// Reset the flag to avoid loading again without initiated by user
// interaction.
aElement->mUseUrgentStartForChannel = false;
}
// Unconditionally disable throttling since we want the media to fluently
// play even when we switch the tab to background.
cos->AddClassFlags(nsIClassOfService::DontThrottle);
}
// The listener holds a strong reference to us. This creates a
// reference cycle, once we've set mChannel, which is manually broken
// in the listener's OnStartRequest method after it is finished with
// the element. The cycle will also be broken if we get a shutdown
// notification before OnStartRequest fires. Necko guarantees that
// OnStartRequest will eventually fire if we don't shut down first.
RefPtr<MediaLoadListener> loadListener = new MediaLoadListener(aElement);
channel->SetNotificationCallbacks(loadListener);
nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(channel);
if (hc) {
// Use a byte range request from the start of the resource.
// This enables us to detect if the stream supports byte range
// requests, and therefore seeking, early.
rv = hc->SetRequestHeader("Range"_ns, "bytes=0-"_ns, false);
MOZ_ASSERT(NS_SUCCEEDED(rv));
aElement->SetRequestHeaders(hc);
}
rv = channel->AsyncOpen(loadListener);
if (NS_FAILED(rv)) {
// Notify load error so the element will try next resource candidate.
aElement->NotifyLoadError("Failed to open channel"_ns);
return;
}
// Else the channel must be open and starting to download. If it encounters
// a non-catastrophic failure, it will set a new task to continue loading
// another candidate. It's safe to set it as mChannel now.
mChannel = channel;
// loadListener will be unregistered either on shutdown or when
// OnStartRequest for the channel we just opened fires.
nsContentUtils::RegisterShutdownObserver(loadListener);
}
nsresult Load(HTMLMediaElement* aElement) {
MOZ_ASSERT(aElement);
// state. Defer NS_NewChannel() to a new regular runnable.
return aElement->OwnerDoc()->Dispatch(NewRunnableMethod<HTMLMediaElement*>(
"ChannelLoader::LoadInternal", this, &ChannelLoader::LoadInternal,
aElement));
}
void Cancel() {
mCancelled = true;
if (mChannel) {
mChannel->CancelWithReason(NS_BINDING_ABORTED,
"HTMLMediaElement::ChannelLoader::Cancel"_ns);
mChannel = nullptr;
}
}
void Done() {
MOZ_ASSERT(mChannel);
// Decoder successfully created, the decoder now owns the MediaResource
// which owns the channel.
mChannel = nullptr;
}
nsresult Redirect(nsIChannel* aChannel, nsIChannel* aNewChannel,
uint32_t aFlags) {
NS_ASSERTION(aChannel == mChannel, "Channels should match!");
mChannel = aNewChannel;
// Handle forwarding of Range header so that the intial detection
// of seeking support (via result code 206) works across redirects.
nsCOMPtr<nsIHttpChannel> http = do_QueryInterface(aChannel);
NS_ENSURE_STATE(http);
constexpr auto rangeHdr = "Range"_ns;
nsAutoCString rangeVal;
if (NS_SUCCEEDED(http->GetRequestHeader(rangeHdr, rangeVal))) {
NS_ENSURE_STATE(!rangeVal.IsEmpty());
http = do_QueryInterface(aNewChannel);
NS_ENSURE_STATE(http);
nsresult rv = http->SetRequestHeader(rangeHdr, rangeVal, false);
NS_ENSURE_SUCCESS(rv, rv);
}
return NS_OK;
}
private:
~ChannelLoader() { MOZ_ASSERT(!mChannel); }
// Holds a reference to the first channel we open to the media resource.
// Once the decoder is created, control over the channel passes to the
// decoder, and we null out this reference. We must store this in case
// we need to cancel the channel before control of it passes to the decoder.
nsCOMPtr<nsIChannel> mChannel;
bool mCancelled = false;
};
class HTMLMediaElement::ErrorSink {
public:
explicit ErrorSink(HTMLMediaElement* aOwner) : mOwner(aOwner) {
MOZ_ASSERT(mOwner);
}
void SetError(uint16_t aErrorCode, const nsACString& aErrorDetails) {
// Since we have multiple paths calling into DecodeError, e.g.
// MediaKeys::Terminated and EMEH264Decoder::Error. We should take the 1st
// one only in order not to fire multiple 'error' events.
if (mError) {
return;
}
if (!IsValidErrorCode(aErrorCode)) {
NS_ASSERTION(false, "Undefined MediaError codes!");
return;
}
mError = new MediaError(mOwner, aErrorCode, aErrorDetails);
mOwner->DispatchAsyncEvent(u"error"_ns);
if (mOwner->ReadyState() == HAVE_NOTHING &&
aErrorCode == MEDIA_ERR_ABORTED) {
// "If the media data fetching process is aborted by the user"
mOwner->DispatchAsyncEvent(u"abort"_ns);
mOwner->ChangeNetworkState(NETWORK_EMPTY);
mOwner->DispatchAsyncEvent(u"emptied"_ns);
if (mOwner->mDecoder) {
mOwner->ShutdownDecoder();
}
} else if (aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED) {
mOwner->ChangeNetworkState(NETWORK_NO_SOURCE);
} else {
mOwner->ChangeNetworkState(NETWORK_IDLE);
}
}
void ResetError() { mError = nullptr; }
RefPtr<MediaError> mError;
private:
bool IsValidErrorCode(const uint16_t& aErrorCode) const {
return (aErrorCode == MEDIA_ERR_DECODE || aErrorCode == MEDIA_ERR_NETWORK ||
aErrorCode == MEDIA_ERR_ABORTED ||
aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED);
}
// Media elememt's life cycle would be longer than error sink, so we use the
// raw pointer and this class would only be referenced by media element.
HTMLMediaElement* mOwner;
};
NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement,
nsGenericHTMLElement)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mStreamWindowCapturer)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaSource)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcMediaSource)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcStream)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcAttrStream)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourcePointer)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mLoadBlockedDoc)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourceLoadCandidate)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelWrapper)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mErrorSink->mError)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputTrackSources);
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlayed);
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTextTrackManager)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioTrackList)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVideoTrackList)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaStreamTrackListener)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mIncomingMediaKeys)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPendingPlayPromises)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSeekDOMPromise)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSetMediaKeysDOMPromise)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mEventBlocker)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement,
nsGenericHTMLElement)
tmp->RemoveMutationObserver(tmp);
if (tmp->mSrcStream) {
// Need to unhook everything that EndSrcMediaStreamPlayback would normally
// do, without creating any new strong references.
if (tmp->mSelectedVideoStreamTrack) {
tmp->mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(tmp);
}
if (tmp->mMediaStreamRenderer) {
tmp->mMediaStreamRenderer->Shutdown();
// We null out mMediaStreamRenderer here since Shutdown() will shut down
// its WatchManager, and UpdateSrcStreamPotentiallyPlaying() contains a
// guard for this.
tmp->mMediaStreamRenderer = nullptr;
}
if (tmp->mSecondaryMediaStreamRenderer) {
tmp->mSecondaryMediaStreamRenderer->Shutdown();
tmp->mSecondaryMediaStreamRenderer = nullptr;
}
if (tmp->mMediaStreamTrackListener) {
tmp->mSrcStream->UnregisterTrackListener(
tmp->mMediaStreamTrackListener.get());
}
}
NS_IMPL_CYCLE_COLLECTION_UNLINK(mStreamWindowCapturer)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcStream)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcAttrStream)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaSource)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcMediaSource)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourcePointer)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mLoadBlockedDoc)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourceLoadCandidate)
if (tmp->mAudioChannelWrapper) {
tmp->mAudioChannelWrapper->Shutdown();
}
NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelWrapper)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mErrorSink->mError)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputStreams)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputTrackSources)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mPlayed)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mTextTrackManager)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioTrackList)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mVideoTrackList)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaStreamTrackListener)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mIncomingMediaKeys)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mPendingPlayPromises)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSeekDOMPromise)
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSetMediaKeysDOMPromise)
if (tmp->mMediaControlKeyListener) {
tmp->mMediaControlKeyListener->StopIfNeeded();
}
if (tmp->mEventBlocker) {
tmp->mEventBlocker->Shutdown();
}
NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_PTR
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLMediaElement,
nsGenericHTMLElement)
void HTMLMediaElement::AddSizeOfExcludingThis(nsWindowSizes& aSizes,
size_t* aNodeSize) const {
nsGenericHTMLElement::AddSizeOfExcludingThis(aSizes, aNodeSize);
// There are many other fields that might be worth reporting, but as seen in
// sometimes, so at least report that.
if (mEventBlocker) {
*aNodeSize +=
mEventBlocker->SizeOfExcludingThis(aSizes.mState.mMallocSizeOf);
}
}
void HTMLMediaElement::ContentRemoved(nsIContent* aChild,
nsIContent* aPreviousSibling) {
if (aChild == mSourcePointer) {
mSourcePointer = aPreviousSibling;
}
}
already_AddRefed<MediaSource> HTMLMediaElement::GetMozMediaSourceObject()
const {
RefPtr<MediaSource> source = mMediaSource;
return source.forget();
}
already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugInfo(
ErrorResult& aRv) {
RefPtr<Promise> promise = CreateDOMPromise(aRv);
if (NS_WARN_IF(aRv.Failed())) {
return nullptr;
}
auto result = MakeUnique<dom::HTMLMediaElementDebugInfo>();
if (mMediaKeys) {
GetEMEInfo(result->mEMEInfo);
}
if (mVideoFrameContainer) {
result->mCompositorDroppedFrames =
mVideoFrameContainer->GetDroppedImageCount();
}
if (mDecoder) {
mDecoder->RequestDebugInfo(result->mDecoder)
->Then(
AbstractMainThread(), __func__,
[promise, ptr = std::move(result)]() {
promise->MaybeResolve(ptr.get());
},
[]() {
MOZ_ASSERT_UNREACHABLE("Unexpected RequestDebugInfo() rejection");
});
} else {
promise->MaybeResolve(result.get());
}
return promise.forget();
}
/* static */
void HTMLMediaElement::MozEnableDebugLog(const GlobalObject&) {
DecoderDoctorLogger::EnableLogging();
}
already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugLog(
ErrorResult& aRv) {
RefPtr<Promise> promise = CreateDOMPromise(aRv);
if (NS_WARN_IF(aRv.Failed())) {
return nullptr;
}
DecoderDoctorLogger::RetrieveMessages(this)->Then(
AbstractMainThread(), __func__,
[promise](const nsACString& aString) {
promise->MaybeResolve(NS_ConvertUTF8toUTF16(aString));
},
[promise](nsresult rv) { promise->MaybeReject(rv); });
return promise.forget();
}
void HTMLMediaElement::SetVisible(bool aVisible) {
mForcedHidden = !aVisible;
if (mDecoder) {
mDecoder->SetForcedHidden(!aVisible);
}
}
bool HTMLMediaElement::IsVideoDecodingSuspended() const {
return mDecoder && mDecoder->IsVideoDecodingSuspended();
}
double HTMLMediaElement::TotalVideoPlayTime() const {
return mDecoder ? mDecoder->GetTotalVideoPlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::TotalVideoHDRPlayTime() const {
return mDecoder ? mDecoder->GetTotalVideoHDRPlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::VisiblePlayTime() const {
return mDecoder ? mDecoder->GetVisibleVideoPlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::InvisiblePlayTime() const {
return mDecoder ? mDecoder->GetInvisibleVideoPlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::TotalAudioPlayTime() const {
return mDecoder ? mDecoder->GetTotalAudioPlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::AudiblePlayTime() const {
return mDecoder ? mDecoder->GetAudiblePlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::InaudiblePlayTime() const {
return mDecoder ? mDecoder->GetInaudiblePlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::MutedPlayTime() const {
return mDecoder ? mDecoder->GetMutedPlayTimeInSeconds() : -1.0;
}
double HTMLMediaElement::VideoDecodeSuspendedTime() const {
return mDecoder ? mDecoder->GetVideoDecodeSuspendedTimeInSeconds() : -1.0;
}
void HTMLMediaElement::SetFormatDiagnosticsReportForMimeType(
const nsAString& aMimeType, DecoderDoctorReportType aType) {
DecoderDoctorDiagnostics diagnostics;
diagnostics.SetDecoderDoctorReportType(aType);
diagnostics.StoreFormatDiagnostics(OwnerDoc(), aMimeType, false /* can play*/,
__func__);
}
void HTMLMediaElement::SetDecodeError(const nsAString& aError,
ErrorResult& aRv) {
// The reason we use this map-ish structure is because we can't use
// `CR.NS_ERROR.*` directly in test. In order to use them in test, we have to
// add them into `xpc.msg`. As we won't use `CR.NS_ERROR.*` in the production
// code, adding them to `xpc.msg` seems an overdesign and adding maintenance
// effort (exposing them in CR also needs to add a description, which is
// useless because we won't show them to users)
static struct {
const char* mName;
nsresult mResult;
} kSupportedErrorList[] = {
{"NS_ERROR_DOM_MEDIA_ABORT_ERR", NS_ERROR_DOM_MEDIA_ABORT_ERR},
{"NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR",
NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR},
{"NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR",
NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR},
{"NS_ERROR_DOM_MEDIA_DECODE_ERR", NS_ERROR_DOM_MEDIA_DECODE_ERR},
{"NS_ERROR_DOM_MEDIA_FATAL_ERR", NS_ERROR_DOM_MEDIA_FATAL_ERR},
{"NS_ERROR_DOM_MEDIA_METADATA_ERR", NS_ERROR_DOM_MEDIA_METADATA_ERR},
{"NS_ERROR_DOM_MEDIA_OVERFLOW_ERR", NS_ERROR_DOM_MEDIA_OVERFLOW_ERR},
{"NS_ERROR_DOM_MEDIA_MEDIASINK_ERR", NS_ERROR_DOM_MEDIA_MEDIASINK_ERR},
{"NS_ERROR_DOM_MEDIA_DEMUXER_ERR", NS_ERROR_DOM_MEDIA_DEMUXER_ERR},
{"NS_ERROR_DOM_MEDIA_CDM_ERR", NS_ERROR_DOM_MEDIA_CDM_ERR},
{"NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR",
NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR}};
for (auto& error : kSupportedErrorList) {
if (strcmp(error.mName, NS_ConvertUTF16toUTF8(aError).get()) == 0) {
DecoderDoctorDiagnostics diagnostics;
diagnostics.StoreDecodeError(OwnerDoc(), error.mResult, u""_ns, __func__);
return;
}
}
aRv.Throw(NS_ERROR_FAILURE);
}
void HTMLMediaElement::SetAudioSinkFailedStartup() {
DecoderDoctorDiagnostics diagnostics;
diagnostics.StoreEvent(OwnerDoc(),
{DecoderDoctorEvent::eAudioSinkStartup,
NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR},
__func__);
}
already_AddRefed<layers::Image> HTMLMediaElement::GetCurrentImage() {
MarkAsTainted();
ImageContainer* container = GetImageContainer();
if (!container) {
return nullptr;
}
AutoLockImage lockImage(container);
RefPtr<layers::Image> image = lockImage.GetImage(TimeStamp::Now());
return image.forget();
}
bool HTMLMediaElement::HasSuspendTaint() const {
MOZ_ASSERT(!mDecoder || (mDecoder->HasSuspendTaint() == mHasSuspendTaint));
return mHasSuspendTaint;
}
already_AddRefed<DOMMediaStream> HTMLMediaElement::GetSrcObject() const {
return do_AddRef(mSrcAttrStream);
}
void HTMLMediaElement::SetSrcObject(DOMMediaStream& aValue) {
SetSrcObject(&aValue);
}
void HTMLMediaElement::SetSrcObject(DOMMediaStream* aValue) {
for (auto& outputStream : mOutputStreams) {
if (aValue == outputStream.mStream) {
ReportToConsole(nsIScriptError::warningFlag,
"MediaElementStreamCaptureCycle");
return;
}
}
mSrcAttrStream = aValue;
UpdateAudioChannelPlayingState();
DoLoad();
}
bool HTMLMediaElement::Ended() {
return (mDecoder && mDecoder->IsEnded()) ||
(mSrcStream && mSrcStreamReportPlaybackEnded);
}
void HTMLMediaElement::GetCurrentSrc(nsAString& aCurrentSrc) {
nsAutoCString src;
GetCurrentSpec(src);
CopyUTF8toUTF16(src, aCurrentSrc);
}
nsresult HTMLMediaElement::OnChannelRedirect(nsIChannel* aChannel,
nsIChannel* aNewChannel,
uint32_t aFlags) {
MOZ_ASSERT(mChannelLoader);
return mChannelLoader->Redirect(aChannel, aNewChannel, aFlags);
}
void HTMLMediaElement::ShutdownDecoder() {
RemoveMediaElementFromURITable();
NS_ASSERTION(mDecoder, "Must have decoder to shut down");
mWaitingForKeyListener.DisconnectIfExists();
if (mMediaSource) {
mMediaSource->CompletePendingTransactions();
}
mDecoder->Shutdown();
DDUNLINKCHILD(mDecoder.get());
mDecoder = nullptr;
}
void HTMLMediaElement::AbortExistingLoads() {
// Abort any already-running instance of the resource selection algorithm.
mLoadWaitStatus = NOT_WAITING;
// Set a new load ID. This will cause events which were enqueued
// with a different load ID to silently be cancelled.
mCurrentLoadID++;
// Immediately reject or resolve the already-dispatched
// nsResolveOrRejectPendingPlayPromisesRunners. These runners won't be
// executed again later since the mCurrentLoadID had been changed.
for (auto& runner : mPendingPlayPromisesRunners) {
runner->ResolveOrReject();
}
mPendingPlayPromisesRunners.Clear();
if (mChannelLoader) {
mChannelLoader->Cancel();
mChannelLoader = nullptr;
}
bool fireTimeUpdate = false;
if (mDecoder) {
fireTimeUpdate = mDecoder->GetCurrentTime() != 0.0;
ShutdownDecoder();
}
if (mSrcStream) {
EndSrcMediaStreamPlayback();
}
if (mMediaSource) {
OwnerDoc()->RemoveMediaElementWithMSE();
}
RemoveMediaElementFromURITable();
mLoadingSrcTriggeringPrincipal = nullptr;
DDLOG(DDLogCategory::Property, "loading_src", "");
DDUNLINKCHILD(mMediaSource.get());
mMediaSource = nullptr;
if (mNetworkState == NETWORK_LOADING || mNetworkState == NETWORK_IDLE) {
DispatchAsyncEvent(u"abort"_ns);
}
bool hadVideo = HasVideo();
mErrorSink->ResetError();
mCurrentPlayRangeStart = -1.0;
mPlayed = new TimeRanges(ToSupports(OwnerDoc()));
mLoadedDataFired = false;
mCanAutoplayFlag = true;
mIsLoadingFromSourceChildren = false;
mSuspendedAfterFirstFrame = false;
mAllowSuspendAfterFirstFrame = true;
mHaveQueuedSelectResource = false;
mSuspendedForPreloadNone = false;
mDownloadSuspendedByCache = false;
mMediaInfo = MediaInfo();
mIsEncrypted = false;
mPendingEncryptedInitData.Reset();
mWaitingForKey = NOT_WAITING_FOR_KEY;
mSourcePointer = nullptr;
mIsBlessed = false;
SetAudibleState(false);
mTags = nullptr;
if (mNetworkState != NETWORK_EMPTY) {
NS_ASSERTION(!mDecoder && !mSrcStream,
"How did someone setup a new stream/decoder already?");
DispatchAsyncEvent(u"emptied"_ns);
// ChangeNetworkState() will call UpdateAudioChannelPlayingState()
// indirectly which depends on mPaused. So we need to update mPaused first.
if (!mPaused) {
mPaused = true;
PlayPromise::RejectPromises(TakePendingPlayPromises(),
NS_ERROR_DOM_MEDIA_ABORT_ERR);
}
ChangeNetworkState(NETWORK_EMPTY);
RemoveMediaTracks();
UpdateOutputTrackSources();
ChangeReadyState(HAVE_NOTHING);
if (mTextTrackManager) {
mTextTrackManager->GetTextTracks()->SetCuesInactive();
}
if (fireTimeUpdate) {
// Since we destroyed the decoder above, the current playback position
// will now be reported as 0. The playback position was non-zero when
// we destroyed the decoder, so fire a timeupdate event so that the
// change will be reflected in the controls.
FireTimeUpdate(TimeupdateType::eMandatory);
}
UpdateAudioChannelPlayingState();
}
if (IsVideo() && hadVideo) {
// Ensure we render transparent black after resetting video resolution.
Maybe<nsIntSize> size = Some(nsIntSize(0, 0));
Invalidate(ImageSizeChanged::Yes, size, ForceInvalidate::No);
}
// As aborting current load would stop current playback, so we have no need to
// resume a paused media element.
ClearResumeDelayedMediaPlaybackAgentIfNeeded();
mMediaControlKeyListener->StopIfNeeded();
// We may have changed mPaused, mCanAutoplayFlag, and other
// things which can affect AddRemoveSelfReference
AddRemoveSelfReference();
mIsRunningSelectResource = false;
AssertReadyStateIsNothing();
}
void HTMLMediaElement::NoSupportedMediaSourceError(
const nsACString& aErrorDetails) {
if (mDecoder) {
ShutdownDecoder();
}
bool isSameOriginLoad = false;
nsresult rv = NS_ERROR_NOT_AVAILABLE;
if (mSrcAttrTriggeringPrincipal && mLoadingSrc) {
rv = mSrcAttrTriggeringPrincipal->IsSameOrigin(mLoadingSrc,
&isSameOriginLoad);
}
if (NS_SUCCEEDED(rv) && !isSameOriginLoad) {
// aErrorDetails can include sensitive details like MimeType or HTTP Status
// Code. In case we're loading a 3rd party resource we should not leak this
// and pass a Generic Error Message
mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED,
"Failed to open media"_ns);
} else {
mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED, aErrorDetails);
}
RemoveMediaTracks();
ChangeDelayLoadStatus(false);
UpdateAudioChannelPlayingState();
PlayPromise::RejectPromises(TakePendingPlayPromises(),
NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
}
// Runs a "synchronous section", a function that must run once the event loop
// has reached a "stable state"
void HTMLMediaElement::RunInStableState(nsIRunnable* aRunnable) {
if (mShuttingDown) {
return;
}
nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
"HTMLMediaElement::RunInStableState",
[self = RefPtr<HTMLMediaElement>(this), loadId = GetCurrentLoadID(),
runnable = RefPtr<nsIRunnable>(aRunnable)]() {
if (self->GetCurrentLoadID() != loadId) {
return;
}
runnable->Run();
});
nsContentUtils::RunInStableState(task.forget());
}
void HTMLMediaElement::QueueLoadFromSourceTask() {
if (!mIsLoadingFromSourceChildren || mShuttingDown) {
return;
}
if (mDecoder) {
// Reset readyState to HAVE_NOTHING since we're going to load a new decoder.
ShutdownDecoder();
ChangeReadyState(HAVE_NOTHING);
}
AssertReadyStateIsNothing();
ChangeDelayLoadStatus(true);
ChangeNetworkState(NETWORK_LOADING);
RefPtr<Runnable> r =
NewRunnableMethod("HTMLMediaElement::LoadFromSourceChildren", this,
&HTMLMediaElement::LoadFromSourceChildren);
RunInStableState(r);
}
void HTMLMediaElement::QueueSelectResourceTask() {
// Don't allow multiple async select resource calls to be queued.
if (mHaveQueuedSelectResource) return;
mHaveQueuedSelectResource = true;
ChangeNetworkState(NETWORK_NO_SOURCE);
RefPtr<Runnable> r =
NewRunnableMethod("HTMLMediaElement::SelectResourceWrapper", this,
&HTMLMediaElement::SelectResourceWrapper);
RunInStableState(r);
}
static bool HasSourceChildren(nsIContent* aElement) {
for (nsIContent* child = aElement->GetFirstChild(); child;
child = child->GetNextSibling()) {
if (child->IsHTMLElement(nsGkAtoms::source)) {
return true;
}
}
return false;
}
static nsCString DocumentOrigin(Document* aDoc) {
if (!aDoc) {
return "null"_ns;
}
nsCOMPtr<nsIPrincipal> principal = aDoc->NodePrincipal();
if (!principal) {
return "null"_ns;
}
nsCString origin;
if (NS_FAILED(principal->GetOrigin(origin))) {
return "null"_ns;
}
return origin;
}
void HTMLMediaElement::Load() {
LOG(LogLevel::Debug,
("%p Load() hasSrcAttrStream=%d hasSrcAttr=%d hasSourceChildren=%d "
"handlingInput=%d hasAutoplayAttr=%d AllowedToPlay=%d "
"ownerDoc=%p (%s) ownerDocUserActivated=%d "
"muted=%d volume=%f",
this, !!mSrcAttrStream, HasAttr(nsGkAtoms::src), HasSourceChildren(this),
UserActivation::IsHandlingUserInput(), HasAttr(nsGkAtoms::autoplay),
AllowedToPlay(), OwnerDoc(), DocumentOrigin(OwnerDoc()).get(),
OwnerDoc()->HasBeenUserGestureActivated(), mMuted, mVolume));
if (mIsRunningLoadMethod) {
return;
}
mIsDoingExplicitLoad = true;
DoLoad();
}
void HTMLMediaElement::DoLoad() {
// Check if media is allowed for the docshell.
nsCOMPtr<nsIDocShell> docShell = OwnerDoc()->GetDocShell();
if (docShell && !docShell->GetAllowMedia()) {
LOG(LogLevel::Debug, ("%p Media not allowed", this));
return;
}
if (mIsRunningLoadMethod) {
return;
}
if (UserActivation::IsHandlingUserInput()) {
// Detect if user has interacted with element so that play will not be
// blocked when initiated by a script. This enables sites to capture user
// intent to play by calling load() in the click handler of a "catalog
// view" of a gallery of videos.
mIsBlessed = true;
// Mark the channel as urgent-start when autoplay so that it will play the
// media from src after loading enough resource.
if (HasAttr(nsGkAtoms::autoplay)) {
mUseUrgentStartForChannel = true;
}
}
SetPlayedOrSeeked(false);
mIsRunningLoadMethod = true;
AbortExistingLoads();
SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
QueueSelectResourceTask();
ResetState();
mIsRunningLoadMethod = false;
}
void HTMLMediaElement::ResetState() {
// There might be a pending MediaDecoder::PlaybackPositionChanged() which
// will overwrite |mMediaInfo.mVideo.mDisplay| in UpdateMediaSize() to give
// staled videoWidth and videoHeight. We have to call ForgetElement() here
// such that the staled callbacks won't reach us.
if (mVideoFrameContainer) {
mVideoFrameContainer->ForgetElement();
mVideoFrameContainer = nullptr;
}
if (mMediaStreamRenderer) {
// mMediaStreamRenderer, has a strong reference to mVideoFrameContainer.
mMediaStreamRenderer->Shutdown();
mMediaStreamRenderer = nullptr;
}
if (mSecondaryMediaStreamRenderer) {
// mSecondaryMediaStreamRenderer, has a strong reference to
// the secondary VideoFrameContainer.
mSecondaryMediaStreamRenderer->Shutdown();
mSecondaryMediaStreamRenderer = nullptr;
}
}
void HTMLMediaElement::SelectResourceWrapper() {
SelectResource();
MaybeBeginCloningVisually();
mIsRunningSelectResource = false;
mHaveQueuedSelectResource = false;
mIsDoingExplicitLoad = false;
}
void HTMLMediaElement::SelectResource() {
if (!mSrcAttrStream && !HasAttr(nsGkAtoms::src) && !HasSourceChildren(this)) {
// The media element has neither a src attribute nor any source
// element children, abort the load.
ChangeNetworkState(NETWORK_EMPTY);
ChangeDelayLoadStatus(false);
return;
}
ChangeDelayLoadStatus(true);
ChangeNetworkState(NETWORK_LOADING);
DispatchAsyncEvent(u"loadstart"_ns);
// Delay setting mIsRunningSeletResource until after UpdatePreloadAction
// so that we don't lose our state change by bailing out of the preload
// state update
UpdatePreloadAction();
mIsRunningSelectResource = true;
// If we have a 'src' attribute, use that exclusively.
nsAutoString src;
if (mSrcAttrStream) {
SetupSrcMediaStreamPlayback(mSrcAttrStream);
} else if (GetAttr(nsGkAtoms::src, src)) {
nsCOMPtr<nsIURI> uri;
MediaResult rv = NewURIFromString(src, getter_AddRefs(uri));
if (NS_SUCCEEDED(rv)) {
LOG(LogLevel::Debug, ("%p Trying load from src=%s", this,
NS_ConvertUTF16toUTF8(src).get()));
NS_ASSERTION(
!mIsLoadingFromSourceChildren,
"Should think we're not loading from source children by default");
RemoveMediaElementFromURITable();
if (!mSrcMediaSource) {
mLoadingSrc = uri;
} else {
mLoadingSrc = nullptr;
}
mLoadingSrcTriggeringPrincipal = mSrcAttrTriggeringPrincipal;
DDLOG(DDLogCategory::Property, "loading_src",
nsCString(NS_ConvertUTF16toUTF8(src)));
bool hadMediaSource = !!mMediaSource;
mMediaSource = mSrcMediaSource;
if (mMediaSource && !hadMediaSource) {
OwnerDoc()->AddMediaElementWithMSE();
}
DDLINKCHILD("mediasource", mMediaSource.get());
UpdatePreloadAction();
if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
// preload:none media, suspend the load here before we make any
// network requests.
SuspendLoad();
return;
}
rv = LoadResource();
if (NS_SUCCEEDED(rv)) {
return;
}
} else {
AutoTArray<nsString, 1> params = {src};
ReportLoadError("MediaLoadInvalidURI", params);
rv = MediaResult(rv.Code(), "MediaLoadInvalidURI");
}
// The media element has neither a src attribute nor a source element child:
// set the networkState to NETWORK_EMPTY, and abort these steps; the
// synchronous section ends.
GetMainThreadSerialEventTarget()->Dispatch(NewRunnableMethod<nsCString>(
"HTMLMediaElement::NoSupportedMediaSourceError", this,
&HTMLMediaElement::NoSupportedMediaSourceError, rv.Description()));
} else {
// Otherwise, the source elements will be used.
mIsLoadingFromSourceChildren = true;
LoadFromSourceChildren();
}
}
void HTMLMediaElement::NotifyLoadError(const nsACString& aErrorDetails) {
if (!mIsLoadingFromSourceChildren) {
LOG(LogLevel::Debug, ("NotifyLoadError(), no supported media error"));
NoSupportedMediaSourceError(aErrorDetails);
} else if (mSourceLoadCandidate) {
DispatchAsyncSourceError(mSourceLoadCandidate);
QueueLoadFromSourceTask();
} else {
NS_WARNING("Should know the source we were loading from!");
}
}
void HTMLMediaElement::NotifyMediaTrackAdded(dom::MediaTrack* aTrack) {
// The set of tracks changed.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
}
void HTMLMediaElement::NotifyMediaTrackRemoved(dom::MediaTrack* aTrack) {
// The set of tracks changed.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
}
void HTMLMediaElement::NotifyMediaTrackEnabled(dom::MediaTrack* aTrack) {
MOZ_ASSERT(aTrack);
if (!aTrack) {
return;
}
#ifdef DEBUG
nsString id;
aTrack->GetId(id);
LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s enabled", this,
aTrack->AsAudioTrack() ? "Audio" : "Video",
NS_ConvertUTF16toUTF8(id).get()));
#endif
MOZ_ASSERT((aTrack->AsAudioTrack() && aTrack->AsAudioTrack()->Enabled()) ||
(aTrack->AsVideoTrack() && aTrack->AsVideoTrack()->Selected()));
if (aTrack->AsAudioTrack()) {
SetMutedInternal(mMuted & ~MUTED_BY_AUDIO_TRACK);
} else if (aTrack->AsVideoTrack()) {
if (!IsVideo()) {
MOZ_ASSERT(false);
return;
}
mDisableVideo = false;
} else {
MOZ_ASSERT(false, "Unknown track type");
}
if (mSrcStream) {
if (AudioTrack* t = aTrack->AsAudioTrack()) {
if (mMediaStreamRenderer) {
mMediaStreamRenderer->AddTrack(t->GetAudioStreamTrack());
}
} else if (VideoTrack* t = aTrack->AsVideoTrack()) {
MOZ_ASSERT(!mSelectedVideoStreamTrack);
mSelectedVideoStreamTrack = t->GetVideoStreamTrack();
mSelectedVideoStreamTrack->AddPrincipalChangeObserver(this);
if (mMediaStreamRenderer) {
mMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
}
if (mSecondaryMediaStreamRenderer) {
mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
}
if (mMediaInfo.HasVideo()) {
mMediaInfo.mVideo.SetAlpha(mSelectedVideoStreamTrack->HasAlpha());
}
nsContentUtils::CombineResourcePrincipals(
&mSrcStreamVideoPrincipal, mSelectedVideoStreamTrack->GetPrincipal());
}
}
// The set of enabled/selected tracks changed.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
}
void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) {
MOZ_ASSERT(aTrack);
if (!aTrack) {
return;
}
nsString id;
aTrack->GetId(id);
LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s disabled", this,
aTrack->AsAudioTrack() ? "Audio" : "Video",
NS_ConvertUTF16toUTF8(id).get()));
MOZ_ASSERT((!aTrack->AsAudioTrack() || !aTrack->AsAudioTrack()->Enabled()) &&
(!aTrack->AsVideoTrack() || !aTrack->AsVideoTrack()->Selected()));
if (AudioTrack* t = aTrack->AsAudioTrack()) {
if (mSrcStream) {
if (mMediaStreamRenderer) {
mMediaStreamRenderer->RemoveTrack(t->GetAudioStreamTrack());
}
}
// If we don't have any live tracks, we don't need to mute MediaElement.
MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
if (AudioTracks()->Length() > 0) {
bool shouldMute = true;
for (uint32_t i = 0; i < AudioTracks()->Length(); ++i) {
if ((*AudioTracks())[i]->Enabled()) {
shouldMute = false;
break;
}
}
if (shouldMute) {
SetMutedInternal(mMuted | MUTED_BY_AUDIO_TRACK);
}
}
} else if (aTrack->AsVideoTrack()) {
if (mSrcStream) {
MOZ_DIAGNOSTIC_ASSERT(mSelectedVideoStreamTrack ==
aTrack->AsVideoTrack()->GetVideoStreamTrack());
if (mMediaStreamRenderer) {
mMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
}
if (mSecondaryMediaStreamRenderer) {
mSecondaryMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
}
mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
mSelectedVideoStreamTrack = nullptr;
}
}
// The set of enabled/selected tracks changed.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
}
void HTMLMediaElement::DealWithFailedElement(nsIContent* aSourceElement) {
if (mShuttingDown) {
return;
}
DispatchAsyncSourceError(aSourceElement);
GetMainThreadSerialEventTarget()->Dispatch(
NewRunnableMethod("HTMLMediaElement::QueueLoadFromSourceTask", this,
&HTMLMediaElement::QueueLoadFromSourceTask));
}
void HTMLMediaElement::LoadFromSourceChildren() {
NS_ASSERTION(mDelayingLoadEvent,
"Should delay load event (if in document) during load");
NS_ASSERTION(mIsLoadingFromSourceChildren,
"Must remember we're loading from source children");
AddMutationObserverUnlessExists(this);
RemoveMediaTracks();
while (true) {
HTMLSourceElement* child = GetNextSource();
if (!child) {
// Exhausted candidates, wait for more candidates to be appended to
// the media element.
mLoadWaitStatus = WAITING_FOR_SOURCE;
ChangeNetworkState(NETWORK_NO_SOURCE);
ChangeDelayLoadStatus(false);
ReportLoadError("MediaLoadExhaustedCandidates");
return;
}
// Must have src attribute.
nsAutoString src;
if (!child->GetAttr(nsGkAtoms::src, src)) {
ReportLoadError("MediaLoadSourceMissingSrc");
DealWithFailedElement(child);
return;
}
// If we have a type attribute, it must be a supported type.
nsAutoString type;
if (child->GetAttr(nsGkAtoms::type, type) && !type.IsEmpty()) {
DecoderDoctorDiagnostics diagnostics;
CanPlayStatus canPlay = GetCanPlay(type, &diagnostics);
diagnostics.StoreFormatDiagnostics(OwnerDoc(), type,
canPlay != CANPLAY_NO, __func__);
if (canPlay == CANPLAY_NO) {
// Check that at least one other source child exists and report that
// we will try to load that one next.
nsIContent* nextChild = mSourcePointer->GetNextSibling();
AutoTArray<nsString, 2> params = {type, src};
while (nextChild) {
if (nextChild && nextChild->IsHTMLElement(nsGkAtoms::source)) {
ReportLoadError("MediaLoadUnsupportedTypeAttributeLoadingNextChild",
params);
break;
}
nextChild = nextChild->GetNextSibling();
};
if (!nextChild) {
ReportLoadError("MediaLoadUnsupportedTypeAttribute", params);
}
DealWithFailedElement(child);
return;
}
}
nsAutoString media;
child->GetAttr(nsGkAtoms::media, media);
HTMLSourceElement* childSrc = HTMLSourceElement::FromNode(child);
MOZ_ASSERT(childSrc, "Expect child to be HTMLSourceElement");
if (childSrc && !childSrc->MatchesCurrentMedia()) {
AutoTArray<nsString, 2> params = {media, src};
ReportLoadError("MediaLoadSourceMediaNotMatched", params);
DealWithFailedElement(child);
LOG(LogLevel::Debug,
("%p Media did not match from <source>=%s type=%s media=%s", this,
NS_ConvertUTF16toUTF8(src).get(), NS_ConvertUTF16toUTF8(type).get(),
NS_ConvertUTF16toUTF8(media).get()));
return;
}
LOG(LogLevel::Debug,
("%p Trying load from <source>=%s type=%s media=%s", this,
NS_ConvertUTF16toUTF8(src).get(), NS_ConvertUTF16toUTF8(type).get(),
NS_ConvertUTF16toUTF8(media).get()));
nsCOMPtr<nsIURI> uri;
NewURIFromString(src, getter_AddRefs(uri));
if (!uri) {
AutoTArray<nsString, 1> params = {src};
ReportLoadError("MediaLoadInvalidURI", params);
DealWithFailedElement(child);
return;
}
RemoveMediaElementFromURITable();
mLoadingSrc = uri;
mLoadingSrcTriggeringPrincipal = child->GetSrcTriggeringPrincipal();
DDLOG(DDLogCategory::Property, "loading_src",
nsCString(NS_ConvertUTF16toUTF8(src)));
bool hadMediaSource = !!mMediaSource;
mMediaSource = child->GetSrcMediaSource();
if (mMediaSource && !hadMediaSource) {
OwnerDoc()->AddMediaElementWithMSE();
}
DDLINKCHILD("mediasource", mMediaSource.get());
NS_ASSERTION(mNetworkState == NETWORK_LOADING,
"Network state should be loading");
if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
// preload:none media, suspend the load here before we make any
// network requests.
SuspendLoad();
return;
}
if (NS_SUCCEEDED(LoadResource())) {
return;
}
// If we fail to load, loop back and try loading the next resource.
DispatchAsyncSourceError(child);
}
MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
}
void HTMLMediaElement::SuspendLoad() {
mSuspendedForPreloadNone = true;
ChangeNetworkState(NETWORK_IDLE);
ChangeDelayLoadStatus(false);
}
void HTMLMediaElement::ResumeLoad(PreloadAction aAction) {
NS_ASSERTION(mSuspendedForPreloadNone,
"Must be halted for preload:none to resume from preload:none "
"suspended load.");
mSuspendedForPreloadNone = false;
mPreloadAction = aAction;
ChangeDelayLoadStatus(true);
ChangeNetworkState(NETWORK_LOADING);
if (!mIsLoadingFromSourceChildren) {
// We were loading from the element's src attribute.
MediaResult rv = LoadResource();
if (NS_FAILED(rv)) {
NoSupportedMediaSourceError(rv.Description());
}
} else {
// We were loading from a child <source> element. Try to resume the
// load of that child, and if that fails, try the next child.
if (NS_FAILED(LoadResource())) {
LoadFromSourceChildren();
}
}
}
bool HTMLMediaElement::AllowedToPlay() const {
return media::AutoplayPolicy::IsAllowedToPlay(*this);
}
uint32_t HTMLMediaElement::GetPreloadDefault() const {
if (mMediaSource) {
return HTMLMediaElement::PRELOAD_ATTR_METADATA;
}
if (OnCellularConnection()) {
return Preferences::GetInt("media.preload.default.cellular",
HTMLMediaElement::PRELOAD_ATTR_NONE);
}
return Preferences::GetInt("media.preload.default",
HTMLMediaElement::PRELOAD_ATTR_METADATA);
}
uint32_t HTMLMediaElement::GetPreloadDefaultAuto() const {
if (OnCellularConnection()) {
return Preferences::GetInt("media.preload.auto.cellular",
HTMLMediaElement::PRELOAD_ATTR_METADATA);
}
return Preferences::GetInt("media.preload.auto",
HTMLMediaElement::PRELOAD_ENOUGH);
}
void HTMLMediaElement::UpdatePreloadAction() {
PreloadAction nextAction = PRELOAD_UNDEFINED;
// If autoplay is set, or we're playing, we should always preload data,
// as we'll need it to play.
if ((AllowedToPlay() && HasAttr(nsGkAtoms::autoplay)) || !mPaused) {
nextAction = HTMLMediaElement::PRELOAD_ENOUGH;
} else {
// Find the appropriate preload action by looking at the attribute.
const nsAttrValue* val =
mAttrs.GetAttr(nsGkAtoms::preload, kNameSpaceID_None);
// MSE doesn't work if preload is none, so it ignores the pref when src is
// from MSE.
uint32_t preloadDefault = GetPreloadDefault();
uint32_t preloadAuto = GetPreloadDefaultAuto();
if (!val) {
// Attribute is not set. Use the preload action specified by the
// media.preload.default pref, or just preload metadata if not present.
nextAction = static_cast<PreloadAction>(preloadDefault);
} else if (val->Type() == nsAttrValue::eEnum) {
PreloadAttrValue attr =
static_cast<PreloadAttrValue>(val->GetEnumValue());
if (attr == HTMLMediaElement::PRELOAD_ATTR_EMPTY ||
attr == HTMLMediaElement::PRELOAD_ATTR_AUTO) {
nextAction = static_cast<PreloadAction>(preloadAuto);
} else if (attr == HTMLMediaElement::PRELOAD_ATTR_METADATA) {
nextAction = HTMLMediaElement::PRELOAD_METADATA;
} else if (attr == HTMLMediaElement::PRELOAD_ATTR_NONE) {
nextAction = HTMLMediaElement::PRELOAD_NONE;
}
} else {
// Use the suggested "missing value default" of "metadata", or the value
// specified by the media.preload.default, if present.
nextAction = static_cast<PreloadAction>(preloadDefault);
}
}
if (nextAction == HTMLMediaElement::PRELOAD_NONE && mIsDoingExplicitLoad) {
nextAction = HTMLMediaElement::PRELOAD_METADATA;
}
mPreloadAction = nextAction;
if (nextAction == HTMLMediaElement::PRELOAD_ENOUGH) {
if (mSuspendedForPreloadNone) {
// Our load was previouly suspended due to the media having preload
// value "none". The preload value has changed to preload:auto, so
// resume the load.
ResumeLoad(PRELOAD_ENOUGH);
} else {
// Preload as much of the video as we can, i.e. don't suspend after
// the first frame.
StopSuspendingAfterFirstFrame();
}
} else if (nextAction == HTMLMediaElement::PRELOAD_METADATA) {
// Ensure that the video can be suspended after first frame.
mAllowSuspendAfterFirstFrame = true;
if (mSuspendedForPreloadNone) {
// Our load was previouly suspended due to the media having preload
// value "none". The preload value has changed to preload:metadata, so
// resume the load. We'll pause the load again after we've read the
// metadata.
ResumeLoad(PRELOAD_METADATA);
}
}
}
MediaResult HTMLMediaElement::LoadResource() {
NS_ASSERTION(mDelayingLoadEvent,
"Should delay load event (if in document) during load");
if (mChannelLoader) {
mChannelLoader->Cancel();
mChannelLoader = nullptr;
}
// Set the media element's CORS mode only when loading a resource
mCORSMode = AttrValueToCORSMode(GetParsedAttr(nsGkAtoms::crossorigin));
HTMLMediaElement* other = LookupMediaElementURITable(mLoadingSrc);
if (other && other->mDecoder) {
// Clone it.
// TODO: remove the cast by storing ChannelMediaDecoder in the URI table.
nsresult rv = InitializeDecoderAsClone(
static_cast<ChannelMediaDecoder*>(other->mDecoder.get()));
if (NS_SUCCEEDED(rv)) return rv;
}
if (mMediaSource) {
MediaDecoderInit decoderInit(
this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
ClampPlaybackRate(mPlaybackRate),
mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
HasAttr(nsGkAtoms::loop),
MediaContainerType(MEDIAMIMETYPE("application/x.mediasource")));
RefPtr<MediaSourceDecoder> decoder = new MediaSourceDecoder(decoderInit);
if (!mMediaSource->Attach(decoder)) {
// TODO: Handle failure: run "If the media data cannot be fetched at
// all, due to network errors, causing the user agent to give up
// trying to fetch the resource" section of resource fetch algorithm.
decoder->Shutdown();
return MediaResult(NS_ERROR_FAILURE, "Failed to attach MediaSource");
}
ChangeDelayLoadStatus(false);
nsresult rv = decoder->Load(mMediaSource->GetPrincipal());
if (NS_FAILED(rv)) {
decoder->Shutdown();
LOG(LogLevel::Debug,
("%p Failed to load for decoder %p", this, decoder.get()));
return MediaResult(rv, "Fail to load decoder");
}
rv = FinishDecoderSetup(decoder);
return MediaResult(rv, "Failed to set up decoder");
}
AssertReadyStateIsNothing();
RefPtr<ChannelLoader> loader = new ChannelLoader;
nsresult rv = loader->Load(this);
if (NS_SUCCEEDED(rv)) {
mChannelLoader = std::move(loader);
}
return MediaResult(rv, "Failed to load channel");
}
nsresult HTMLMediaElement::LoadWithChannel(nsIChannel* aChannel,
nsIStreamListener** aListener) {
NS_ENSURE_ARG_POINTER(aChannel);
NS_ENSURE_ARG_POINTER(aListener);
*aListener = nullptr;
// Make sure we don't reenter during synchronous abort events.
if (mIsRunningLoadMethod) return NS_OK;
mIsRunningLoadMethod = true;
AbortExistingLoads();
mIsRunningLoadMethod = false;
mLoadingSrcTriggeringPrincipal = nullptr;
nsresult rv = aChannel->GetOriginalURI(getter_AddRefs(mLoadingSrc));
NS_ENSURE_SUCCESS(rv, rv);
ChangeDelayLoadStatus(true);
rv = InitializeDecoderForChannel(aChannel, aListener);
if (NS_FAILED(rv)) {
ChangeDelayLoadStatus(false);
return rv;
}
SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
DispatchAsyncEvent(u"loadstart"_ns);
return NS_OK;
}
bool HTMLMediaElement::Seeking() const {
return mDecoder && mDecoder->IsSeeking();
}
double HTMLMediaElement::CurrentTime() const {
if (mMediaStreamRenderer) {
return ToMicrosecondResolution(mMediaStreamRenderer->CurrentTime());
}
if (mDefaultPlaybackStartPosition == 0.0 && mDecoder) {
return std::clamp(mDecoder->GetCurrentTime(), 0.0, mDecoder->GetDuration());
}
return mDefaultPlaybackStartPosition;
}
void HTMLMediaElement::FastSeek(double aTime, ErrorResult& aRv) {
LOG(LogLevel::Debug, ("%p FastSeek(%f) called by JS", this, aTime));
Seek(aTime, SeekTarget::PrevSyncPoint, IgnoreErrors());
}
already_AddRefed<Promise> HTMLMediaElement::SeekToNextFrame(ErrorResult& aRv) {
/* This will cause JIT code to be kept around longer, to help performance
* when using SeekToNextFrame to iterate through every frame of a video.
*/
nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
if (win) {
if (JSObject* obj = win->AsGlobal()->GetGlobalJSObject()) {
js::NotifyAnimationActivity(obj);
}
}
Seek(CurrentTime(), SeekTarget::NextFrame, aRv);
if (aRv.Failed()) {
return nullptr;
}
mSeekDOMPromise = CreateDOMPromise(aRv);
if (NS_WARN_IF(aRv.Failed())) {
return nullptr;
}
return do_AddRef(mSeekDOMPromise);
}
void HTMLMediaElement::SetCurrentTime(double aCurrentTime, ErrorResult& aRv) {
LOG(LogLevel::Debug,
("%p SetCurrentTime(%lf) called by JS", this, aCurrentTime));
Seek(aCurrentTime, SeekTarget::Accurate, IgnoreErrors());
}
/**
* Check if aValue is inside a range of aRanges, and if so returns true
* and puts the range index in aIntervalIndex. If aValue is not
* inside a range, returns false, and aIntervalIndex
* is set to the index of the range which starts immediately after aValue
* (and can be aRanges.Length() if aValue is after the last range).
*/
static bool IsInRanges(TimeRanges& aRanges, double aValue,
uint32_t& aIntervalIndex) {
uint32_t length = aRanges.Length();
for (uint32_t i = 0; i < length; i++) {
double start = aRanges.Start(i);
if (start > aValue) {
aIntervalIndex = i;
return false;
}
double end = aRanges.End(i);
if (aValue <= end) {
aIntervalIndex = i;
return true;
}
}
aIntervalIndex = length;
return false;
}
void HTMLMediaElement::Seek(double aTime, SeekTarget::Type aSeekType,
ErrorResult& aRv) {
// Note: Seek is called both by synchronous code that expects errors thrown in
// aRv, as well as asynchronous code that expects a promise. Make sure all
// synchronous errors are returned using aRv, not promise rejections.
// aTime should be non-NaN.
MOZ_ASSERT(!std::isnan(aTime));
// Seeking step1, Set the media element's show poster flag to false.
mShowPoster = false;
// Detect if user has interacted with element by seeking so that
// play will not be blocked when initiated by a script.
if (UserActivation::IsHandlingUserInput()) {
mIsBlessed = true;
}
StopSuspendingAfterFirstFrame();
if (mSrcAttrStream) {
// do nothing since media streams have an empty Seekable range.
aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
return;
}
if (mPlayed && mCurrentPlayRangeStart != -1.0) {
double rangeEndTime = CurrentTime();
LOG(LogLevel::Debug, ("%p Adding \'played\' a range : [%f, %f]", this,
mCurrentPlayRangeStart, rangeEndTime));
// Multiple seek without playing, or seek while playing.
if (mCurrentPlayRangeStart != rangeEndTime) {
// Don't round the left of the interval: it comes from script and needs
// to be exact.
mPlayed->Add(mCurrentPlayRangeStart, rangeEndTime);
}
// Reset the current played range start time. We'll re-set it once
// the seek completes.
mCurrentPlayRangeStart = -1.0;
}
if (mReadyState == HAVE_NOTHING) {
mDefaultPlaybackStartPosition = aTime;
aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
return;
}
if (!mDecoder) {
// mDecoder must always be set in order to reach this point.
NS_ASSERTION(mDecoder, "SetCurrentTime failed: no decoder");
aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
return;
}
// Clamp the seek target to inside the seekable ranges.
media::TimeRanges seekableRanges = mDecoder->GetSeekableTimeRanges();
if (seekableRanges.IsInvalid()) {
aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
return;
}
RefPtr<TimeRanges> seekable =
new TimeRanges(ToSupports(OwnerDoc()), seekableRanges);
uint32_t length = seekable->Length();
if (length == 0) {
aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
return;
}
// If the position we want to seek to is not in a seekable range, we seek
// to the closest position in the seekable ranges instead. If two positions
// are equally close, we seek to the closest position from the currentTime.
// See seeking spec, point 7 :
uint32_t range = 0;
bool isInRange = IsInRanges(*seekable, aTime, range);
if (!isInRange) {
if (range == 0) {
// aTime is before the first range in |seekable|, the closest point we can
// seek to is the start of the first range.
aTime = seekable->Start(0);
} else if (range == length) {
// Seek target is after the end last range in seekable data.
// Clamp the seek target to the end of the last seekable range.
aTime = seekable->End(length - 1);
} else {
double leftBound = seekable->End(range - 1);
double rightBound = seekable->Start(range);
double distanceLeft = Abs(leftBound - aTime);
double distanceRight = Abs(rightBound - aTime);
if (distanceLeft == distanceRight) {
double currentTime = CurrentTime();
distanceLeft = Abs(leftBound - currentTime);
distanceRight = Abs(rightBound - currentTime);
}
aTime = (distanceLeft < distanceRight) ? leftBound : rightBound;
}
}
// TODO: The spec requires us to update the current time to reflect the
// actual seek target before beginning the synchronous section, but
// that requires changing all MediaDecoderReaders to support telling
// us the fastSeek target, and it's currently not possible to get
// this information as we don't yet control the demuxer for all
// MediaDecoderReaders.
mPlayingBeforeSeek = IsPotentiallyPlaying();
// The media backend is responsible for dispatching the timeupdate
// event if it changes the playback position as a result of the seek.
LOG(LogLevel::Debug, ("%p SetCurrentTime(%f) starting seek", this, aTime));
mDecoder->Seek(aTime, aSeekType);
// We changed whether we're seeking so we need to AddRemoveSelfReference.
AddRemoveSelfReference();
}
double HTMLMediaElement::Duration() const {
if (mSrcStream) {
if (mSrcStreamPlaybackEnded) {
return CurrentTime();
}
return std::numeric_limits<double>::infinity();
}
if (mDecoder) {
return mDecoder->GetDuration();
}
return std::numeric_limits<double>::quiet_NaN();
}
already_AddRefed<TimeRanges> HTMLMediaElement::Seekable() const {
media::TimeRanges seekable =
mDecoder ? mDecoder->GetSeekableTimeRanges() : media::TimeRanges();
RefPtr<TimeRanges> ranges = new TimeRanges(
ToSupports(OwnerDoc()), seekable.ToMicrosecondResolution());
return ranges.forget();
}
already_AddRefed<TimeRanges> HTMLMediaElement::Played() {
RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()));
uint32_t timeRangeCount = 0;
if (mPlayed) {
timeRangeCount = mPlayed->Length();
}
for (uint32_t i = 0; i < timeRangeCount; i++) {
double begin = mPlayed->Start(i);
double end = mPlayed->End(i);
ranges->Add(begin, end);
}
if (mCurrentPlayRangeStart != -1.0) {
double now = CurrentTime();
if (mCurrentPlayRangeStart != now) {
// Don't round the left of the interval: it comes from script and needs
// to be exact.
ranges->Add(mCurrentPlayRangeStart, now);
}
}
ranges->Normalize();
return ranges.forget();
}
void HTMLMediaElement::Pause(ErrorResult& aRv) {
LOG(LogLevel::Debug, ("%p Pause() called by JS", this));
if (mNetworkState == NETWORK_EMPTY) {
LOG(LogLevel::Debug, ("Loading due to Pause()"));
DoLoad();
}
PauseInternal();
}
void HTMLMediaElement::PauseInternal() {
if (mDecoder && mNetworkState != NETWORK_EMPTY) {
mDecoder->Pause();
}
bool oldPaused = mPaused;
mPaused = true;
// Step 1,
mCanAutoplayFlag = false;
// We changed mPaused and mCanAutoplayFlag which can affect
// AddRemoveSelfReference
AddRemoveSelfReference();
UpdateSrcMediaStreamPlaying();
if (mAudioChannelWrapper) {
mAudioChannelWrapper->NotifyPlayStateChanged();
}
// We don't need to resume media which is paused explicitly by user.
ClearResumeDelayedMediaPlaybackAgentIfNeeded();
if (!oldPaused) {
FireTimeUpdate(TimeupdateType::eMandatory);
DispatchAsyncEvent(u"pause"_ns);
AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_ABORT_ERR);
}
}
void HTMLMediaElement::SetVolume(double aVolume, ErrorResult& aRv) {
LOG(LogLevel::Debug, ("%p SetVolume(%f) called by JS", this, aVolume));
if (aVolume < 0.0 || aVolume > 1.0) {
aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR);
return;
}
if (aVolume == mVolume) return;
mVolume = aVolume;
// Here we want just to update the volume.
SetVolumeInternal();
DispatchAsyncEvent(u"volumechange"_ns);
// We allow inaudible autoplay. But changing our volume may make this
// media audible. So pause if we are no longer supposed to be autoplaying.
PauseIfShouldNotBePlaying();
}
void HTMLMediaElement::MozGetMetadata(JSContext* aCx,
JS::MutableHandle<JSObject*> aResult,
ErrorResult& aRv) {
if (mReadyState < HAVE_METADATA) {
aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
return;
}
JS::Rooted<JSObject*> tags(aCx, JS_NewPlainObject(aCx));
if (!tags) {
aRv.Throw(NS_ERROR_FAILURE);
return;
}
if (mTags) {
for (const auto& entry : *mTags) {
nsString wideValue;
CopyUTF8toUTF16(entry.GetData(), wideValue);
JS::Rooted<JSString*> string(aCx,
JS_NewUCStringCopyZ(aCx, wideValue.Data()));
if (!string || !JS_DefineProperty(aCx, tags, entry.GetKey().Data(),
string, JSPROP_ENUMERATE)) {
NS_WARNING("couldn't create metadata object!");
aRv.Throw(NS_ERROR_FAILURE);
return;
}
}
}
aResult.set(tags);
}
void HTMLMediaElement::SetMutedInternal(uint32_t aMuted) {
uint32_t oldMuted = mMuted;
mMuted = aMuted;
if (!!aMuted == !!oldMuted) {
return;
}
SetVolumeInternal();
}
void HTMLMediaElement::PauseIfShouldNotBePlaying() {
if (GetPaused()) {
return;
}
if (!AllowedToPlay()) {
AUTOPLAY_LOG("pause because not allowed to play, element=%p", this);
ErrorResult rv;
Pause(rv);
}
}
void HTMLMediaElement::SetVolumeInternal() {
float effectiveVolume = ComputedVolume();
if (mDecoder) {
mDecoder->SetVolume(effectiveVolume);
} else if (mMediaStreamRenderer) {
mMediaStreamRenderer->SetAudioOutputVolume(effectiveVolume);
}
NotifyAudioPlaybackChanged(
AudioChannelService::AudibleChangedReasons::eVolumeChanged);
}
void HTMLMediaElement::SetMuted(bool aMuted) {
LOG(LogLevel::Debug, ("%p SetMuted(%d) called by JS", this, aMuted));
if (aMuted == Muted()) {
return;
}
if (aMuted) {
SetMutedInternal(mMuted | MUTED_BY_CONTENT);
} else {
SetMutedInternal(mMuted & ~MUTED_BY_CONTENT);
}
DispatchAsyncEvent(u"volumechange"_ns);
// We allow inaudible autoplay. But changing our mute status may make this
// media audible. So pause if we are no longer supposed to be autoplaying.
PauseIfShouldNotBePlaying();
}
void HTMLMediaElement::GetAllEnabledMediaTracks(
nsTArray<RefPtr<MediaTrack>>& aTracks) {
if (AudioTrackList* tracks = AudioTracks()) {
for (size_t i = 0; i < tracks->Length(); ++i) {
AudioTrack* track = (*tracks)[i];
if (track->Enabled()) {
aTracks.AppendElement(track);
}
}
}
if (IsVideo()) {
if (VideoTrackList* tracks = VideoTracks()) {
for (size_t i = 0; i < tracks->Length(); ++i) {
VideoTrack* track = (*tracks)[i];
if (track->Selected()) {
aTracks.AppendElement(track);
}
}
}
}
}
void HTMLMediaElement::SetCapturedOutputStreamsEnabled(bool aEnabled) {
for (const auto& entry : mOutputTrackSources.Values()) {
entry->SetEnabled(aEnabled);
}
}
HTMLMediaElement::OutputMuteState HTMLMediaElement::OutputTracksMuted() {
return mPaused || mReadyState <= HAVE_CURRENT_DATA ? OutputMuteState::Muted
: OutputMuteState::Unmuted;
}
void HTMLMediaElement::UpdateOutputTracksMuting() {
for (const auto& entry : mOutputTrackSources.Values()) {
entry->SetMutedByElement(OutputTracksMuted());
}
}
void HTMLMediaElement::AddOutputTrackSourceToOutputStream(
MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream,
AddTrackMode aMode) {
if (aOutputStream.mStream == mSrcStream) {
// Cycle detected. This can happen since tracks are added async.
// We avoid forwarding it to the output here or we'd get into an infloop.
LOG(LogLevel::Warning,
("NOT adding output track source %p to output stream "
"%p -- cycle detected",
aSource, aOutputStream.mStream.get()));
return;
}
LOG(LogLevel::Debug, ("Adding output track source %p to output stream %p",
aSource, aOutputStream.mStream.get()));
RefPtr<MediaStreamTrack> domTrack;
if (aSource->Track()->mType == MediaSegment::AUDIO) {
domTrack = new AudioStreamTrack(
aOutputStream.mStream->GetOwner(), aSource->Track(), aSource,
MediaStreamTrackState::Live, aSource->Muted());
} else {
domTrack = new VideoStreamTrack(
aOutputStream.mStream->GetOwner(), aSource->Track(), aSource,
MediaStreamTrackState::Live, aSource->Muted());
}
aOutputStream.mLiveTracks.AppendElement(domTrack);
switch (aMode) {
case AddTrackMode::ASYNC:
GetMainThreadSerialEventTarget()->Dispatch(
NewRunnableMethod<StoreRefPtrPassByPtr<MediaStreamTrack>>(
"DOMMediaStream::AddTrackInternal", aOutputStream.mStream,
&DOMMediaStream::AddTrackInternal, domTrack));
break;
case AddTrackMode::SYNC:
aOutputStream.mStream->AddTrackInternal(domTrack);
break;
default:
MOZ_CRASH("Unexpected mode");
}
LOG(LogLevel::Debug,
("Created capture %s track %p",
domTrack->AsAudioStreamTrack() ? "audio" : "video", domTrack.get()));
}
void HTMLMediaElement::UpdateOutputTrackSources() {
// This updates the track sources in mOutputTrackSources so they're in sync
// with the tracks being currently played, and state saying whether we should
// be capturing tracks. This method is long so here is a breakdown:
// - Figure out the tracks that should be captured
// - Diff those against currently captured tracks (mOutputTrackSources), into
// tracks-to-add, and tracks-to-remove
// - Remove the tracks in tracks-to-remove and dispatch "removetrack" and
// "ended" events for them
// - If playback has ended, or there is no longer a media provider object,
// remove any OutputMediaStreams that have the finish-when-ended flag set
// - Create track sources for, and add to OutputMediaStreams, the tracks in
// tracks-to-add
const bool shouldHaveTrackSources = mTracksCaptured.Ref() &&
!IsPlaybackEnded() &&
mReadyState >= HAVE_METADATA;
// Add track sources for all enabled/selected MediaTracks.
nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
if (!window) {
return;
}
if (mDecoder) {
if (!mTracksCaptured.Ref()) {
mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::None);
} else if (!AudioTracks() || !VideoTracks() || !shouldHaveTrackSources) {
// We've been unlinked, or tracks are not yet known.
mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::Halt);
} else {
mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::Capture,
mTracksCaptured.Ref().get());
}
}
// Start with all MediaTracks
AutoTArray<RefPtr<MediaTrack>, 4> mediaTracksToAdd;
if (shouldHaveTrackSources) {
GetAllEnabledMediaTracks(mediaTracksToAdd);
}
// ...and all MediaElementTrackSources.
auto trackSourcesToRemove =
ToTArray<AutoTArray<nsString, 4>>(mOutputTrackSources.Keys());
// Then work out the differences.
mediaTracksToAdd.RemoveLastElements(
mediaTracksToAdd.end() -
std::remove_if(mediaTracksToAdd.begin(), mediaTracksToAdd.end(),
[this, &trackSourcesToRemove](const auto& track) {
const bool remove =
mOutputTrackSources.GetWeak(track->GetId());
if (remove) {
trackSourcesToRemove.RemoveElement(track->GetId());
}
return remove;
}));
// First remove stale track sources.
for (const auto& id : trackSourcesToRemove) {
RefPtr<MediaElementTrackSource> source = mOutputTrackSources.GetWeak(id);
LOG(LogLevel::Debug, ("Removing output track source %p for track %s",
source.get(), NS_ConvertUTF16toUTF8(id).get()));
if (mDecoder) {
mDecoder->RemoveOutputTrack(source->Track());
}
// The source of this track just ended. Force-notify that it ended.
// If we bounce it to the MediaTrackGraph it might not be picked up,
// for instance if the MediaInputPort was destroyed in the same
// iteration as it was added.
GetMainThreadSerialEventTarget()->Dispatch(
NewRunnableMethod("MediaElementTrackSource::OverrideEnded", source,
&MediaElementTrackSource::OverrideEnded));
// Remove the track from the MediaStream after it ended.
for (OutputMediaStream& ms : mOutputStreams) {
if (source->Track()->mType == MediaSegment::VIDEO &&
ms.mCapturingAudioOnly) {
continue;
}
DebugOnly<size_t> length = ms.mLiveTracks.Length();
ms.mLiveTracks.RemoveElementsBy(
[&](const RefPtr<MediaStreamTrack>& aTrack) {
if (&aTrack->GetSource() != source) {
return false;
}
GetMainThreadSerialEventTarget()->Dispatch(
NewRunnableMethod<RefPtr<MediaStreamTrack>>(
"DOMMediaStream::RemoveTrackInternal", ms.mStream,
&DOMMediaStream::RemoveTrackInternal, aTrack));
return true;
});
MOZ_ASSERT(ms.mLiveTracks.Length() == length - 1);
}
mOutputTrackSources.Remove(id);
}
// Then update finish-when-ended output streams as needed.
for (size_t i = mOutputStreams.Length(); i-- > 0;) {
if (!mOutputStreams[i].mFinishWhenEnded) {
continue;
}
if (!mOutputStreams[i].mFinishWhenEndedLoadingSrc &&
!mOutputStreams[i].mFinishWhenEndedAttrStream &&
!mOutputStreams[i].mFinishWhenEndedMediaSource) {
// This finish-when-ended stream has not seen any source loaded yet.
// Update the loading src if it's time.
if (!IsPlaybackEnded()) {
if (mLoadingSrc) {
mOutputStreams[i].mFinishWhenEndedLoadingSrc = mLoadingSrc;
} else if (mSrcAttrStream) {
mOutputStreams[i].mFinishWhenEndedAttrStream = mSrcAttrStream;
} else if (mSrcMediaSource) {
mOutputStreams[i].mFinishWhenEndedMediaSource = mSrcMediaSource;
}
}
continue;
}
// Discard finish-when-ended output streams with a loading src set as
// needed.
if (!IsPlaybackEnded() &&
mLoadingSrc == mOutputStreams[i].mFinishWhenEndedLoadingSrc) {
continue;
}
if (!IsPlaybackEnded() &&
mSrcAttrStream == mOutputStreams[i].mFinishWhenEndedAttrStream) {
continue;
}
if (!IsPlaybackEnded() &&
mSrcMediaSource == mOutputStreams[i].mFinishWhenEndedMediaSource) {
continue;
}
LOG(LogLevel::Debug,
("Playback ended or source changed. Discarding stream %p",
mOutputStreams[i].mStream.get()));
mOutputStreams.RemoveElementAt(i);
if (mOutputStreams.IsEmpty()) {
mTracksCaptured = nullptr;
// mTracksCaptured is one of the Watchables triggering this method.
// Unsetting it here means we'll run through this method again very soon.
return;
}
}
// Finally add new MediaTracks.
for (const auto& mediaTrack : mediaTracksToAdd) {
nsAutoString id;
mediaTrack->GetId(id);
MediaSegment::Type type;
if (mediaTrack->AsAudioTrack()) {
type = MediaSegment::AUDIO;
} else if (mediaTrack->AsVideoTrack()) {
type = MediaSegment::VIDEO;
} else {
MOZ_CRASH("Unknown track type");
}
RefPtr<ProcessedMediaTrack> track;
RefPtr<MediaElementTrackSource> source;
if (mDecoder) {
track = mTracksCaptured.Ref()->mTrack->Graph()->CreateForwardedInputTrack(
type);
RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
if (!principal || IsCORSSameOrigin()) {
principal = NodePrincipal();
}
source = MakeAndAddRef<MediaElementTrackSource>(
track, principal, OutputTracksMuted(),
type == MediaSegment::VIDEO
? HTMLVideoElement::FromNode(this)->HasAlpha()
: false);
mDecoder->AddOutputTrack(track);
} else if (mSrcStream) {
MediaStreamTrack* inputTrack;
if (AudioTrack* t = mediaTrack->AsAudioTrack()) {
inputTrack = t->GetAudioStreamTrack();
} else if (VideoTrack* t = mediaTrack->AsVideoTrack()) {
inputTrack = t->GetVideoStreamTrack();
} else {
MOZ_CRASH("Unknown track type");
}
MOZ_ASSERT(inputTrack);
if (!inputTrack) {
NS_ERROR("Input track not found in source stream");
return;
}
MOZ_DIAGNOSTIC_ASSERT(!inputTrack->Ended());
track = inputTrack->Graph()->CreateForwardedInputTrack(type);
RefPtr<MediaInputPort> port = inputTrack->ForwardTrackContentsTo(track);
source = MakeAndAddRef<MediaElementTrackSource>(
inputTrack, &inputTrack->GetSource(), track, port,
OutputTracksMuted());
// Track is muted initially, so we don't leak data if it's added while
// paused and an MTG iteration passes before the mute comes into effect.
source->SetEnabled(mSrcStreamIsPlaying);
} else {
MOZ_CRASH("Unknown source");
}
LOG(LogLevel::Debug, ("Adding output track source %p for track %s",
source.get(), NS_ConvertUTF16toUTF8(id).get()));
track->QueueSetAutoend(false);
MOZ_DIAGNOSTIC_ASSERT(!mOutputTrackSources.Contains(id));
mOutputTrackSources.InsertOrUpdate(id, RefPtr{source});
// Add the new track source to any existing output streams
for (OutputMediaStream& ms : mOutputStreams) {
if (source->Track()->mType == MediaSegment::VIDEO &&
ms.mCapturingAudioOnly) {
// If the output stream is for audio only we ignore video sources.
continue;
}
AddOutputTrackSourceToOutputStream(source, ms);
}
}
}
bool HTMLMediaElement::CanBeCaptured(StreamCaptureType aCaptureType) {
// Don't bother capturing when the document has gone away
nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
if (!window) {
return false;
}
// Prevent capturing restricted video
if (aCaptureType == StreamCaptureType::CAPTURE_ALL_TRACKS &&
ContainsRestrictedContent()) {
return false;
}
return true;
}
already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureStreamInternal(
StreamCaptureBehavior aFinishBehavior, StreamCaptureType aStreamCaptureType,
MediaTrackGraph* aGraph) {
MOZ_ASSERT(CanBeCaptured(aStreamCaptureType));
LogVisibility(CallerAPI::CAPTURE_STREAM);
MarkAsTainted();
if (mTracksCaptured.Ref()) {
// Already have an output stream. Check whether the graph rate matches if
// specified.
if (aGraph && aGraph != mTracksCaptured.Ref()->mTrack->Graph()) {
return nullptr;
}
} else {
// This is the first output stream, or there are no tracks. If the former,
// start capturing all tracks. If the latter, they will be added later.
MediaTrackGraph* graph = aGraph;
if (!graph) {
nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
if (!window) {
return nullptr;
}
MediaTrackGraph::GraphDriverType graphDriverType =
HasAudio() ? MediaTrackGraph::AUDIO_THREAD_DRIVER
: MediaTrackGraph::SYSTEM_THREAD_DRIVER;
graph = MediaTrackGraph::GetInstance(
graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
}
mTracksCaptured = MakeRefPtr<SharedDummyTrack>(
graph->CreateSourceTrack(MediaSegment::AUDIO));
UpdateOutputTrackSources();
}
nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
OutputMediaStream* out = mOutputStreams.EmplaceBack(
MakeRefPtr<DOMMediaStream>(window),
aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO,
aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED);
if (aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED &&
!mOutputTrackSources.IsEmpty()) {
// This output stream won't receive any more tracks when playback of the
// current src of this media element ends, or when the src of this media
// element changes. If we're currently playing something (i.e., if there are
// tracks currently captured), set the current src on the output stream so
// this can be tracked. If we're not playing anything,
// UpdateOutputTrackSources will set the current src when it becomes
// available later.
if (mLoadingSrc) {
out->mFinishWhenEndedLoadingSrc = mLoadingSrc;
}
if (mSrcAttrStream) {
out->mFinishWhenEndedAttrStream = mSrcAttrStream;
}
if (mSrcMediaSource) {
out->mFinishWhenEndedMediaSource = mSrcMediaSource;
}
MOZ_ASSERT(out->mFinishWhenEndedLoadingSrc ||
out->mFinishWhenEndedAttrStream ||
out->mFinishWhenEndedMediaSource);
}
if (aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO) {
if (mSrcStream) {
// We don't support applying volume and mute to the captured stream, when
// capturing a MediaStream.
ReportToConsole(nsIScriptError::errorFlag,
"MediaElementAudioCaptureOfMediaStreamError");
}
// mAudioCaptured tells the user that the audio played by this media element
// is being routed to the captureStreams *instead* of being played to
// speakers.
mAudioCaptured = true;
}
for (const RefPtr<MediaElementTrackSource>& source :
mOutputTrackSources.Values()) {
if (source->Track()->mType == MediaSegment::VIDEO) {
// Only add video tracks if we're a video element and the output stream
// wants video.
if (!IsVideo()) {
continue;
}
if (out->mCapturingAudioOnly) {
continue;
}
}
AddOutputTrackSourceToOutputStream(source, *out, AddTrackMode::SYNC);
}
return do_AddRef(out->mStream);
}
already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureAudio(
ErrorResult& aRv, MediaTrackGraph* aGraph) {
MOZ_RELEASE_ASSERT(aGraph);
if (!CanBeCaptured(StreamCaptureType::CAPTURE_AUDIO)) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
RefPtr<DOMMediaStream> stream =
CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
StreamCaptureType::CAPTURE_AUDIO, aGraph);
if (!stream) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
return stream.forget();
}
RefPtr<GenericNonExclusivePromise> HTMLMediaElement::GetAllowedToPlayPromise() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(!mOutputStreams.IsEmpty(),
"This method should only be called during stream capturing!");
if (AllowedToPlay()) {
AUTOPLAY_LOG("MediaElement %p has allowed to play, resolve promise", this);
return GenericNonExclusivePromise::CreateAndResolve(true, __func__);
}
AUTOPLAY_LOG("create allow-to-play promise for MediaElement %p", this);
return mAllowedToPlayPromise.Ensure(__func__);
}
already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStream(
ErrorResult& aRv) {
if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
RefPtr<DOMMediaStream> stream =
CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
StreamCaptureType::CAPTURE_ALL_TRACKS, nullptr);
if (!stream) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
return stream.forget();
}
already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStreamUntilEnded(
ErrorResult& aRv) {
if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
RefPtr<DOMMediaStream> stream =
CaptureStreamInternal(StreamCaptureBehavior::FINISH_WHEN_ENDED,
StreamCaptureType::CAPTURE_ALL_TRACKS, nullptr);
if (!stream) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
return stream.forget();
}
class MediaElementSetForURI : public nsURIHashKey {
public:
explicit MediaElementSetForURI(const nsIURI* aKey) : nsURIHashKey(aKey) {}
MediaElementSetForURI(MediaElementSetForURI&& aOther) noexcept
: nsURIHashKey(std::move(aOther)),
mElements(std::move(aOther.mElements)) {}
nsTArray<HTMLMediaElement*> mElements;
};
using MediaElementURITable = nsTHashtable<MediaElementSetForURI>;
// Elements in this table must have non-null mDecoder and mLoadingSrc, and those
// can't change while the element is in the table. The table is keyed by
// the element's mLoadingSrc. Each entry has a list of all elements with the
// same mLoadingSrc.
static MediaElementURITable* gElementTable;
#ifdef DEBUG
static bool URISafeEquals(nsIURI* a1, nsIURI* a2) {
if (!a1 || !a2) {
// Consider two empty URIs *not* equal!
return false;
}
bool equal = false;
nsresult rv = a1->Equals(a2, &equal);
return NS_SUCCEEDED(rv) && equal;
}
// Returns the number of times aElement appears in the media element table
// for aURI. If this returns other than 0 or 1, there's a bug somewhere!
static unsigned MediaElementTableCount(HTMLMediaElement* aElement,
nsIURI* aURI) {
if (!gElementTable || !aElement) {
return 0;
}
uint32_t uriCount = 0;
uint32_t otherCount = 0;
for (const auto& entry : *gElementTable) {
uint32_t count = 0;
for (const auto& elem : entry.mElements) {
if (elem == aElement) {
count++;
}
}
if (URISafeEquals(aURI, entry.GetKey())) {
uriCount = count;
} else {
otherCount += count;
}
}
NS_ASSERTION(otherCount == 0, "Should not have entries for unknown URIs");
return uriCount;
}
#endif
void HTMLMediaElement::AddMediaElementToURITable() {
NS_ASSERTION(mDecoder, "Call this only with decoder Load called");
NS_ASSERTION(
MediaElementTableCount(this, mLoadingSrc) == 0,
"Should not have entry for element in element table before addition");
if (!gElementTable) {
gElementTable = new MediaElementURITable();
}
MediaElementSetForURI* entry = gElementTable->PutEntry(mLoadingSrc);
entry->mElements.AppendElement(this);
NS_ASSERTION(
MediaElementTableCount(this, mLoadingSrc) == 1,
"Should have a single entry for element in element table after addition");
}
void HTMLMediaElement::RemoveMediaElementFromURITable() {
if (!mDecoder || !mLoadingSrc || !gElementTable) {
return;
}
MediaElementSetForURI* entry = gElementTable->GetEntry(mLoadingSrc);
if (!entry) {
return;
}
entry->mElements.RemoveElement(this);
if (entry->mElements.IsEmpty()) {
gElementTable->RemoveEntry(entry);
if (gElementTable->Count() == 0) {
delete gElementTable;
gElementTable = nullptr;
}
}
NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
"After remove, should no longer have an entry in element table");
}
HTMLMediaElement* HTMLMediaElement::LookupMediaElementURITable(nsIURI* aURI) {
if (!gElementTable) {
return nullptr;
}
MediaElementSetForURI* entry = gElementTable->GetEntry(aURI);
if (!entry) {
return nullptr;
}
for (uint32_t i = 0; i < entry->mElements.Length(); ++i) {
HTMLMediaElement* elem = entry->mElements[i];
bool equal;
// Look for elements that have the same principal and CORS mode.
// Ditto for anything else that could cause us to send different headers.
if (NS_SUCCEEDED(elem->NodePrincipal()->Equals(NodePrincipal(), &equal)) &&
equal && elem->mCORSMode == mCORSMode) {
// See SetupDecoder() below. We only add a element to the table when
// mDecoder is a ChannelMediaDecoder.
auto* decoder = static_cast<ChannelMediaDecoder*>(elem->mDecoder.get());
NS_ASSERTION(decoder, "Decoder gone");
if (decoder->CanClone()) {
return elem;
}
}
}
return nullptr;
}
class HTMLMediaElement::ShutdownObserver : public nsIObserver {
enum class Phase : int8_t { Init, Subscribed, Unsubscribed };
public:
NS_DECL_ISUPPORTS
NS_IMETHOD Observe(nsISupports*, const char* aTopic,
const char16_t*) override {
if (mPhase != Phase::Subscribed) {
// Bail out if we are not subscribed for this might be called even after
// |nsContentUtils::UnregisterShutdownObserver(this)|.
return NS_OK;
}
MOZ_DIAGNOSTIC_ASSERT(mWeak);
if (strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
mWeak->NotifyShutdownEvent();
}
return NS_OK;
}
void Subscribe(HTMLMediaElement* aPtr) {
MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Init);
MOZ_DIAGNOSTIC_ASSERT(!mWeak);
mWeak = aPtr;
nsContentUtils::RegisterShutdownObserver(this);
mPhase = Phase::Subscribed;
}
void Unsubscribe() {
MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Subscribed);
MOZ_DIAGNOSTIC_ASSERT(mWeak);
MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
"ReleaseMediaElement should have been called first");
mWeak = nullptr;
nsContentUtils::UnregisterShutdownObserver(this);
mPhase = Phase::Unsubscribed;
}
void AddRefMediaElement() {
MOZ_DIAGNOSTIC_ASSERT(mWeak);
MOZ_DIAGNOSTIC_ASSERT(!mAddRefed, "Should only ever AddRef once");
mWeak->AddRef();
mAddRefed = true;
}
void ReleaseMediaElement() {
MOZ_DIAGNOSTIC_ASSERT(mWeak);
MOZ_DIAGNOSTIC_ASSERT(mAddRefed, "Should only release after AddRef");
mWeak->Release();
mAddRefed = false;
}
private:
virtual ~ShutdownObserver() {
MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Unsubscribed);
MOZ_DIAGNOSTIC_ASSERT(!mWeak);
MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
"ReleaseMediaElement should have been called first");
}
// Guaranteed to be valid by HTMLMediaElement.
HTMLMediaElement* mWeak = nullptr;
Phase mPhase = Phase::Init;
bool mAddRefed = false;
};
NS_IMPL_ISUPPORTS(HTMLMediaElement::ShutdownObserver, nsIObserver)
class HTMLMediaElement::TitleChangeObserver final : public nsIObserver {
public:
NS_DECL_ISUPPORTS
explicit TitleChangeObserver(HTMLMediaElement* aElement)
: mElement(aElement) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aElement);
}
NS_IMETHOD Observe(nsISupports*, const char* aTopic,
const char16_t*) override {
if (mElement) {
mElement->UpdateStreamName();
}
return NS_OK;
}
void Subscribe() {
nsCOMPtr<nsIObserverService> observerService =
mozilla::services::GetObserverService();
if (observerService) {
observerService->AddObserver(this, "document-title-changed", false);
}
}
void Unsubscribe() {
nsCOMPtr<nsIObserverService> observerService =
mozilla::services::GetObserverService();
if (observerService) {
observerService->RemoveObserver(this, "document-title-changed");
}
}
private:
~TitleChangeObserver() = default;
WeakPtr<HTMLMediaElement> mElement;
};
NS_IMPL_ISUPPORTS(HTMLMediaElement::TitleChangeObserver, nsIObserver)
HTMLMediaElement::HTMLMediaElement(
already_AddRefed<mozilla::dom::NodeInfo>&& aNodeInfo)
: nsGenericHTMLElement(std::move(aNodeInfo)),
mWatchManager(this, AbstractThread::MainThread()),
mShutdownObserver(new ShutdownObserver),
mTitleChangeObserver(new TitleChangeObserver(this)),
mEventBlocker(new EventBlocker(this)),
mPlayed(new TimeRanges(ToSupports(OwnerDoc()))),
mTracksCaptured(nullptr, "HTMLMediaElement::mTracksCaptured"),
mErrorSink(new ErrorSink(this)),
mAudioChannelWrapper(new AudioChannelAgentCallback(this)),
mSink(std::pair(nsString(), RefPtr<AudioDeviceInfo>())),
mShowPoster(IsVideo()),
mMediaControlKeyListener(new MediaControlKeyListener(this)) {
MOZ_ASSERT(GetMainThreadSerialEventTarget());
// Please don't add anything to this constructor or the initialization
// list that can cause AddRef to be called. This prevents subclasses
// from overriding AddRef in a way that works with our refcount
// logging mechanisms. Put these things inside of the ::Init method
// instead.
}
void HTMLMediaElement::Init() {
MOZ_ASSERT(mRefCnt == 0 && !mRefCnt.IsPurple(),
"HTMLMediaElement::Init called when AddRef has been called "
"at least once already, probably in the constructor. Please "
"see the documentation in the HTMLMediaElement constructor.");
MOZ_ASSERT(!mRefCnt.IsPurple());
mAudioTrackList = new AudioTrackList(OwnerDoc()->GetParentObject(), this);
mVideoTrackList = new VideoTrackList(OwnerDoc()->GetParentObject(), this);
DecoderDoctorLogger::LogConstruction(this);
mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateWakeLock);
mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateOutputTracksMuting);
mWatchManager.Watch(
mPaused, &HTMLMediaElement::NotifyMediaControlPlaybackStateChanged);
mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTracksMuting);
mWatchManager.Watch(mTracksCaptured,
&HTMLMediaElement::UpdateOutputTrackSources);
mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTrackSources);
mWatchManager.Watch(mDownloadSuspendedByCache,
&HTMLMediaElement::UpdateReadyStateInternal);
mWatchManager.Watch(mFirstFrameLoaded,
&HTMLMediaElement::UpdateReadyStateInternal);
mWatchManager.Watch(mSrcStreamPlaybackEnded,
&HTMLMediaElement::UpdateReadyStateInternal);
ErrorResult rv;
double defaultVolume = Preferences::GetFloat("media.default_volume", 1.0);
SetVolume(defaultVolume, rv);
RegisterActivityObserver();
NotifyOwnerDocumentActivityChanged();
// We initialize the MediaShutdownManager as the HTMLMediaElement is always
// constructed on the main thread, and not during stable state.
// (MediaShutdownManager make use of nsIAsyncShutdownClient which is written
// in JS)
MediaShutdownManager::InitStatics();
#if defined(MOZ_WIDGET_ANDROID)
GVAutoplayPermissionRequestor::AskForPermissionIfNeeded(
OwnerDoc()->GetInnerWindow());
#endif
OwnerDoc()->SetDocTreeHadMedia();
mShutdownObserver->Subscribe(this);
mInitialized = true;
}
HTMLMediaElement::~HTMLMediaElement() {
MOZ_ASSERT(mInitialized,
"HTMLMediaElement must be initialized before it is destroyed.");
NS_ASSERTION(
!mHasSelfReference,
"How can we be destroyed if we're still holding a self reference?");
mWatchManager.Shutdown();
mShutdownObserver->Unsubscribe();
mTitleChangeObserver->Unsubscribe();
if (mVideoFrameContainer) {
mVideoFrameContainer->ForgetElement();
}
UnregisterActivityObserver();
mSetCDMRequest.DisconnectIfExists();
mAllowedToPlayPromise.RejectIfExists(NS_ERROR_FAILURE, __func__);
if (mDecoder) {
ShutdownDecoder();
}
if (mProgressTimer) {
StopProgress();
}
if (mSrcStream) {
EndSrcMediaStreamPlayback();
}
NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
"Destroyed media element should no longer be in element table");
if (mChannelLoader) {
mChannelLoader->Cancel();
}
if (mAudioChannelWrapper) {
mAudioChannelWrapper->Shutdown();
mAudioChannelWrapper = nullptr;
}
if (mResumeDelayedPlaybackAgent) {
mResumePlaybackRequest.DisconnectIfExists();
mResumeDelayedPlaybackAgent = nullptr;
}
mMediaControlKeyListener->StopIfNeeded();
mMediaControlKeyListener = nullptr;
WakeLockRelease();
DecoderDoctorLogger::LogDestruction(this);
}
void HTMLMediaElement::StopSuspendingAfterFirstFrame() {
mAllowSuspendAfterFirstFrame = false;
if (!mSuspendedAfterFirstFrame) return;
mSuspendedAfterFirstFrame = false;
if (mDecoder) {
mDecoder->Resume();
}
}
void HTMLMediaElement::SetPlayedOrSeeked(bool aValue) {
if (aValue == mHasPlayedOrSeeked) {
return;
}
mHasPlayedOrSeeked = aValue;
// Force a reflow so that the poster frame hides or shows immediately.
nsIFrame* frame = GetPrimaryFrame();
if (!frame) {
return;
}
frame->PresShell()->FrameNeedsReflow(frame, IntrinsicDirty::FrameAndAncestors,
NS_FRAME_IS_DIRTY);
}
void HTMLMediaElement::NotifyXPCOMShutdown() { ShutdownDecoder(); }
already_AddRefed<Promise> HTMLMediaElement::Play(ErrorResult& aRv) {
LOG(LogLevel::Debug,
("%p Play() called by JS readyState=%d", this, mReadyState.Ref()));
// 4.8.12.8
// When the play() method on a media element is invoked, the user agent must
// run the following steps.
RefPtr<PlayPromise> promise = CreatePlayPromise(aRv);
if (NS_WARN_IF(aRv.Failed())) {
return nullptr;
}
// 4.8.12.8 - Step 1:
// If the media element is not allowed to play, return a promise rejected
// with a "NotAllowedError" DOMException and abort these steps.
// NOTE: we may require requesting permission from the user, so we do the
// "not allowed" check below.
// 4.8.12.8 - Step 2:
// If the media element's error attribute is not null and its code
// attribute has the value MEDIA_ERR_SRC_NOT_SUPPORTED, return a promise
// rejected with a "NotSupportedError" DOMException and abort these steps.
if (GetError() && GetError()->Code() == MEDIA_ERR_SRC_NOT_SUPPORTED) {
LOG(LogLevel::Debug,
("%p Play() promise rejected because source not supported.", this));
promise->MaybeReject(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
return promise.forget();
}
// 4.8.12.8 - Step 3:
// Let promise be a new promise and append promise to the list of pending
// play promises.
// Note: Promise appended to list of pending promises as needed below.
if (ShouldBeSuspendedByInactiveDocShell()) {
LOG(LogLevel::Debug, ("%p no allow to play by the docShell for now", this));
mPendingPlayPromises.AppendElement(promise);
return promise.forget();
}
// We may delay starting playback of a media resource for an unvisited tab
// until it's going to foreground or being resumed by the play tab icon.
if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
CreateResumeDelayedMediaPlaybackAgentIfNeeded();
LOG(LogLevel::Debug, ("%p delay Play() call", this));
MaybeDoLoad();
// When play is delayed, save a reference to the promise, and return it.
// The promise will be resolved when we resume play by either the tab is
// brought to the foreground, or the audio tab indicator is clicked.
mPendingPlayPromises.AppendElement(promise);
return promise.forget();
}
const bool handlingUserInput = UserActivation::IsHandlingUserInput();
mPendingPlayPromises.AppendElement(promise);
if (AllowedToPlay()) {
AUTOPLAY_LOG("allow MediaElement %p to play", this);
mAllowedToPlayPromise.ResolveIfExists(true, __func__);
PlayInternal(handlingUserInput);
UpdateCustomPolicyAfterPlayed();
} else {
AUTOPLAY_LOG("reject MediaElement %p to play", this);
AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR);
}
return promise.forget();
}
void HTMLMediaElement::DispatchEventsWhenPlayWasNotAllowed() {
if (StaticPrefs::media_autoplay_block_event_enabled()) {
DispatchAsyncEvent(u"blocked"_ns);
}
DispatchBlockEventForVideoControl();
if (!mHasEverBeenBlockedForAutoplay) {
MaybeNotifyAutoplayBlocked();
ReportToConsole(nsIScriptError::warningFlag, "BlockAutoplayError");
mHasEverBeenBlockedForAutoplay = true;
}
}
void HTMLMediaElement::MaybeNotifyAutoplayBlocked() {
// This event is used to notify front-end side that we've blocked autoplay,
// so front-end side should show blocking icon as well.
RefPtr<AsyncEventDispatcher> asyncDispatcher =
new AsyncEventDispatcher(OwnerDoc(), u"GloballyAutoplayBlocked"_ns,
CanBubble::eYes, ChromeOnlyDispatch::eYes);
asyncDispatcher->PostDOMEvent();
}
void HTMLMediaElement::DispatchBlockEventForVideoControl() {
#if defined(MOZ_WIDGET_ANDROID)
nsVideoFrame* videoFrame = do_QueryFrame(GetPrimaryFrame());
if (!videoFrame || !videoFrame->GetVideoControls()) {
return;
}
RefPtr<AsyncEventDispatcher> asyncDispatcher = new AsyncEventDispatcher(
videoFrame->GetVideoControls(), u"MozNoControlsBlockedVideo"_ns,
CanBubble::eYes);
asyncDispatcher->PostDOMEvent();
#endif
}
void HTMLMediaElement::PlayInternal(bool aHandlingUserInput) {
if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE) {
// The media load algorithm will be initiated by a user interaction.
// We want to boost the channel priority for better responsiveness.
// Note this must be done before UpdatePreloadAction() which will
// update |mPreloadAction|.
mUseUrgentStartForChannel = true;
}
StopSuspendingAfterFirstFrame();
SetPlayedOrSeeked(true);
// 4.8.12.8 - Step 4:
// If the media element's networkState attribute has the value NETWORK_EMPTY,
// invoke the media element's resource selection algorithm.
MaybeDoLoad();
if (mSuspendedForPreloadNone) {
ResumeLoad(PRELOAD_ENOUGH);
}
// 4.8.12.8 - Step 5:
// If the playback has ended and the direction of playback is forwards,
// seek to the earliest possible position of the media resource.
// Even if we just did Load() or ResumeLoad(), we could already have a decoder
// here if we managed to clone an existing decoder.
if (mDecoder) {
if (mDecoder->IsEnded()) {
SetCurrentTime(0);
}
if (!mSuspendedByInactiveDocOrDocshell) {
mDecoder->Play();
}
}
if (mCurrentPlayRangeStart == -1.0) {
mCurrentPlayRangeStart = CurrentTime();
}
const bool oldPaused = mPaused;
mPaused = false;
// Step 5,
mCanAutoplayFlag = false;
// We changed mPaused and mCanAutoplayFlag which can affect
// AddRemoveSelfReference and our preload status.
AddRemoveSelfReference();
UpdatePreloadAction();
UpdateSrcMediaStreamPlaying();
StartMediaControlKeyListenerIfNeeded();
// Once play() has been called in a user generated event handler,
// it is allowed to autoplay. Note: we can reach here when not in
// a user generated event handler if our readyState has not yet
// reached HAVE_METADATA.
mIsBlessed |= aHandlingUserInput;
// TODO: If the playback has ended, then the user agent must set
// seek to the effective start.
// 4.8.12.8 - Step 6:
// If the media element's paused attribute is true, run the following steps:
if (oldPaused) {
// 6.1. Change the value of paused to false. (Already done.)
// This step is uplifted because the "block-media-playback" feature needs
// the mPaused to be false before UpdateAudioChannelPlayingState() being
// called.
// 6.2. If the show poster flag is true, set the element's show poster flag
// to false and run the time marches on steps.
if (mShowPoster) {
mShowPoster = false;
if (mTextTrackManager) {
mTextTrackManager->TimeMarchesOn();
}
}
// 6.3. Queue a task to fire a simple event named play at the element.
DispatchAsyncEvent(u"play"_ns);
// 6.4. If the media element's readyState attribute has the value
// HAVE_NOTHING, HAVE_METADATA, or HAVE_CURRENT_DATA, queue a task to
// fire a simple event named waiting at the element.
// Otherwise, the media element's readyState attribute has the value
// HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA: notify about playing for the
// element.
switch (mReadyState) {
case HAVE_NOTHING:
DispatchAsyncEvent(u"waiting"_ns);
break;
case HAVE_METADATA:
case HAVE_CURRENT_DATA:
DispatchAsyncEvent(u"waiting"_ns);
break;
case HAVE_FUTURE_DATA:
case HAVE_ENOUGH_DATA:
NotifyAboutPlaying();
break;
}
} else if (mReadyState >= HAVE_FUTURE_DATA) {
// 7. Otherwise, if the media element's readyState attribute has the value
// HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA, take pending play promises and
// queue a task to resolve pending play promises with the result.
AsyncResolvePendingPlayPromises();
}
// 8. Set the media element's autoplaying flag to false. (Already done.)
// 9. Return promise.
// (Done in caller.)
}
void HTMLMediaElement::MaybeDoLoad() {
if (mNetworkState == NETWORK_EMPTY) {
DoLoad();
}
}
void HTMLMediaElement::UpdateWakeLock() {
MOZ_ASSERT(NS_IsMainThread());
// Ensure we have a wake lock if we're playing audibly. This ensures the
// device doesn't sleep while playing.
bool playing = !mPaused;
bool isAudible = Volume() > 0.0 && !mMuted && mIsAudioTrackAudible;
// WakeLock when playing audible media.
if (playing && isAudible) {
CreateAudioWakeLockIfNeeded();
} else {
ReleaseAudioWakeLockIfExists();
}
}
void HTMLMediaElement::CreateAudioWakeLockIfNeeded() {
if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) {
return;
}
if (!mWakeLock) {
RefPtr<power::PowerManagerService> pmService =
power::PowerManagerService::GetInstance();
NS_ENSURE_TRUE_VOID(pmService);
ErrorResult rv;
mWakeLock = pmService->NewWakeLock(u"audio-playing"_ns,
OwnerDoc()->GetInnerWindow(), rv);
}
}
void HTMLMediaElement::ReleaseAudioWakeLockIfExists() {
if (mWakeLock) {
ErrorResult rv;
mWakeLock->Unlock(rv);
rv.SuppressException();
mWakeLock = nullptr;
}
}
void HTMLMediaElement::WakeLockRelease() { ReleaseAudioWakeLockIfExists(); }
void HTMLMediaElement::GetEventTargetParent(EventChainPreVisitor& aVisitor) {
if (!this->Controls() || !aVisitor.mEvent->mFlags.mIsTrusted) {
nsGenericHTMLElement::GetEventTargetParent(aVisitor);
return;
}
// We will need to trap pointer, touch, and mouse events within the media
// element, allowing media control exclusive consumption on these events,
// and preventing the content from handling them.
switch (aVisitor.mEvent->mMessage) {
case ePointerDown:
case ePointerUp:
case eTouchEnd:
// Always prevent touchmove captured in video element from being handled by
// content, since we always do that for touchstart.
case eTouchMove:
case eTouchStart:
case eMouseClick:
case eMouseDoubleClick:
case eMouseDown:
case eMouseUp:
aVisitor.mCanHandle = false;
return;
// The *move events however are only comsumed when the range input is being
// dragged.
case ePointerMove:
case eMouseMove: {
nsINode* node =
nsINode::FromEventTargetOrNull(aVisitor.mEvent->mOriginalTarget);
if (MOZ_UNLIKELY(!node)) {
return;
}
HTMLInputElement* el = nullptr;
if (node->ChromeOnlyAccess()) {
if (node->IsHTMLElement(nsGkAtoms::input)) {
// The node is a <input type="range">
el = static_cast<HTMLInputElement*>(node);
} else if (node->GetParentNode() &&
node->GetParentNode()->IsHTMLElement(nsGkAtoms::input)) {
// The node is a child of <input type="range">
el = static_cast<HTMLInputElement*>(node->GetParentNode());
}
}
if (el && el->IsDraggingRange()) {
aVisitor.mCanHandle = false;
return;
}
nsGenericHTMLElement::GetEventTargetParent(aVisitor);
return;
}
default:
nsGenericHTMLElement::GetEventTargetParent(aVisitor);
return;
}
}
bool HTMLMediaElement::ParseAttribute(int32_t aNamespaceID, nsAtom* aAttribute,
const nsAString& aValue,
nsIPrincipal* aMaybeScriptedPrincipal,
nsAttrValue& aResult) {
// Mappings from 'preload' attribute strings to an enumeration.
static const nsAttrValue::EnumTable kPreloadTable[] = {
{"", HTMLMediaElement::PRELOAD_ATTR_EMPTY},
{"none", HTMLMediaElement::PRELOAD_ATTR_NONE},
{"metadata", HTMLMediaElement::PRELOAD_ATTR_METADATA},
{"auto", HTMLMediaElement::PRELOAD_ATTR_AUTO},
{nullptr, 0}};
if (aNamespaceID == kNameSpaceID_None) {
if (aAttribute == nsGkAtoms::crossorigin) {
ParseCORSValue(aValue, aResult);
return true;
}
if (aAttribute == nsGkAtoms::preload) {
return aResult.ParseEnumValue(aValue, kPreloadTable, false);
}
}
return nsGenericHTMLElement::ParseAttribute(aNamespaceID, aAttribute, aValue,
aMaybeScriptedPrincipal, aResult);
}
void HTMLMediaElement::DoneCreatingElement() {
if (HasAttr(nsGkAtoms::muted)) {
mMuted |= MUTED_BY_CONTENT;
}
}
bool HTMLMediaElement::IsHTMLFocusable(bool aWithMouse, bool* aIsFocusable,
int32_t* aTabIndex) {
if (nsGenericHTMLElement::IsHTMLFocusable(aWithMouse, aIsFocusable,
aTabIndex)) {
return true;
}
*aIsFocusable = true;
return false;
}
int32_t HTMLMediaElement::TabIndexDefault() { return 0; }
void HTMLMediaElement::AfterSetAttr(int32_t aNameSpaceID, nsAtom* aName,
const nsAttrValue* aValue,
const nsAttrValue* aOldValue,
nsIPrincipal* aMaybeScriptedPrincipal,
bool aNotify) {
if (aNameSpaceID == kNameSpaceID_None) {
if (aName == nsGkAtoms::src) {
mSrcMediaSource = nullptr;
mSrcAttrTriggeringPrincipal = nsContentUtils::GetAttrTriggeringPrincipal(
this, aValue ? aValue->GetStringValue() : EmptyString(),
aMaybeScriptedPrincipal);
if (aValue) {
nsString srcStr = aValue->GetStringValue();
nsCOMPtr<nsIURI> uri;
NewURIFromString(srcStr, getter_AddRefs(uri));
if (uri && IsMediaSourceURI(uri)) {
nsresult rv = NS_GetSourceForMediaSourceURI(
uri, getter_AddRefs(mSrcMediaSource));
if (NS_FAILED(rv)) {
nsAutoString spec;
GetCurrentSrc(spec);
AutoTArray<nsString, 1> params = {spec};
ReportLoadError("MediaLoadInvalidURI", params);
}
}
}
} else if (aName == nsGkAtoms::autoplay) {
if (aNotify) {
if (aValue) {
StopSuspendingAfterFirstFrame();
CheckAutoplayDataReady();
}
// This attribute can affect AddRemoveSelfReference
AddRemoveSelfReference();
UpdatePreloadAction();
}
} else if (aName == nsGkAtoms::preload) {
UpdatePreloadAction();
} else if (aName == nsGkAtoms::loop) {
if (mDecoder) {
mDecoder->SetLooping(!!aValue);
}
} else if (aName == nsGkAtoms::controls && IsInComposedDoc()) {
NotifyUAWidgetSetupOrChange();
}
}
// Since AfterMaybeChangeAttr may call DoLoad, make sure that it is called
// *after* any possible changes to mSrcMediaSource.
if (aValue) {
AfterMaybeChangeAttr(aNameSpaceID, aName, aNotify);
}
return nsGenericHTMLElement::AfterSetAttr(
aNameSpaceID, aName, aValue, aOldValue, aMaybeScriptedPrincipal, aNotify);
}
void HTMLMediaElement::OnAttrSetButNotChanged(int32_t aNamespaceID,
nsAtom* aName,
const nsAttrValueOrString& aValue,
bool aNotify) {
AfterMaybeChangeAttr(aNamespaceID, aName, aNotify);
return nsGenericHTMLElement::OnAttrSetButNotChanged(aNamespaceID, aName,
aValue, aNotify);
}
void HTMLMediaElement::AfterMaybeChangeAttr(int32_t aNamespaceID, nsAtom* aName,
bool aNotify) {
if (aNamespaceID == kNameSpaceID_None) {
if (aName == nsGkAtoms::src) {
DoLoad();
}
}
}
nsresult HTMLMediaElement::BindToTree(BindContext& aContext, nsINode& aParent) {
nsresult rv = nsGenericHTMLElement::BindToTree(aContext, aParent);
if (IsInComposedDoc()) {
// Construct Shadow Root so web content can be hidden in the DOM.
AttachAndSetUAShadowRoot();
// The preload action depends on the value of the autoplay attribute.
// It's value may have changed, so update it.
UpdatePreloadAction();
}
NotifyDecoderActivityChanges();
mMediaControlKeyListener->UpdateOwnerBrowsingContextIfNeeded();
return rv;
}
void HTMLMediaElement::UnbindFromTree(UnbindContext& aContext) {
mVisibilityState = Visibility::Untracked;
if (IsInComposedDoc()) {
NotifyUAWidgetTeardown();
}
nsGenericHTMLElement::UnbindFromTree(aContext);
MOZ_ASSERT(IsActuallyInvisible());
NotifyDecoderActivityChanges();
//
// Dispatch a task to run once we're in a stable state which ensures we're
// paused if we're no longer in a document. Note that we need to dispatch this
// even if there are other tasks in flight for this because these can be
// cancelled if there's a new load.
//
// FIXME(emilio): Per that spec section, we should only do this if we used to
// be connected, though other browsers match our current behavior...
//
nsCOMPtr<nsIRunnable> task =
NS_NewRunnableFunction("dom::HTMLMediaElement::UnbindFromTree",
[self = RefPtr<HTMLMediaElement>(this)]() {
if (!self->IsInComposedDoc()) {
self->PauseInternal();
self->mMediaControlKeyListener->StopIfNeeded();
}
});
RunInStableState(task);
}
/* static */
CanPlayStatus HTMLMediaElement::GetCanPlay(
const nsAString& aType, DecoderDoctorDiagnostics* aDiagnostics) {
Maybe<MediaContainerType> containerType = MakeMediaContainerType(aType);
if (!containerType) {
return CANPLAY_NO;
}
CanPlayStatus status =
DecoderTraits::CanHandleContainerType(*containerType, aDiagnostics);
if (status == CANPLAY_YES &&
(*containerType).ExtendedType().Codecs().IsEmpty()) {
// Per spec: 'Generally, a user agent should never return "probably" for a
// type that allows the `codecs` parameter if that parameter is not
// present.' As all our currently-supported types allow for `codecs`, we can
// do this check here.
// TODO: Instead, missing `codecs` should be checked in each decoder's
// `IsSupportedType` call from `CanHandleCodecsType()`.
return CANPLAY_MAYBE;
}
return status;
}
void HTMLMediaElement::CanPlayType(const nsAString& aType, nsAString& aResult) {
DecoderDoctorDiagnostics diagnostics;
CanPlayStatus canPlay = GetCanPlay(aType, &diagnostics);
diagnostics.StoreFormatDiagnostics(OwnerDoc(), aType, canPlay != CANPLAY_NO,
__func__);
switch (canPlay) {
case CANPLAY_NO:
aResult.Truncate();
break;
case CANPLAY_YES:
aResult.AssignLiteral("probably");
break;
case CANPLAY_MAYBE:
aResult.AssignLiteral("maybe");
break;
default:
MOZ_ASSERT_UNREACHABLE("Unexpected case.");
break;
}
LOG(LogLevel::Debug,
("%p CanPlayType(%s) = \"%s\"", this, NS_ConvertUTF16toUTF8(aType).get(),
NS_ConvertUTF16toUTF8(aResult).get()));
}
void HTMLMediaElement::AssertReadyStateIsNothing() {
#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
if (mReadyState != HAVE_NOTHING) {
char buf[1024];
SprintfLiteral(buf,
"readyState=%d networkState=%d mLoadWaitStatus=%d "
"mSourceLoadCandidate=%d "
"mIsLoadingFromSourceChildren=%d mPreloadAction=%d "
"mSuspendedForPreloadNone=%d error=%d",
int(mReadyState), int(mNetworkState), int(mLoadWaitStatus),
!!mSourceLoadCandidate, mIsLoadingFromSourceChildren,
int(mPreloadAction), mSuspendedForPreloadNone,
GetError() ? GetError()->Code() : 0);
MOZ_CRASH_UNSAFE_PRINTF("ReadyState should be HAVE_NOTHING! %s", buf);
}
#endif
}
nsresult HTMLMediaElement::InitializeDecoderAsClone(
ChannelMediaDecoder* aOriginal) {
NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
NS_ASSERTION(mDecoder == nullptr, "Shouldn't have a decoder");
AssertReadyStateIsNothing();
MediaDecoderInit decoderInit(
this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
ClampPlaybackRate(mPlaybackRate),
mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
HasAttr(nsGkAtoms::loop), aOriginal->ContainerType());
RefPtr<ChannelMediaDecoder> decoder = aOriginal->Clone(decoderInit);
if (!decoder) return NS_ERROR_FAILURE;
LOG(LogLevel::Debug,
("%p Cloned decoder %p from %p", this, decoder.get(), aOriginal));
return FinishDecoderSetup(decoder);
}
template <typename DecoderType, typename... LoadArgs>
nsresult HTMLMediaElement::SetupDecoder(DecoderType* aDecoder,
LoadArgs&&... aArgs) {
LOG(LogLevel::Debug, ("%p Created decoder %p for type %s", this, aDecoder,
aDecoder->ContainerType().OriginalString().Data()));
nsresult rv = aDecoder->Load(std::forward<LoadArgs>(aArgs)...);
if (NS_FAILED(rv)) {
aDecoder->Shutdown();
LOG(LogLevel::Debug, ("%p Failed to load for decoder %p", this, aDecoder));
return rv;
}
rv = FinishDecoderSetup(aDecoder);
// Only ChannelMediaDecoder supports resource cloning.
if (std::is_same_v<DecoderType, ChannelMediaDecoder> && NS_SUCCEEDED(rv)) {
AddMediaElementToURITable();
NS_ASSERTION(
MediaElementTableCount(this, mLoadingSrc) == 1,
"Media element should have single table entry if decode initialized");
}
return rv;
}
nsresult HTMLMediaElement::InitializeDecoderForChannel(
nsIChannel* aChannel, nsIStreamListener** aListener) {
NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
AssertReadyStateIsNothing();
DecoderDoctorDiagnostics diagnostics;
nsAutoCString mimeType;
aChannel->GetContentType(mimeType);
NS_ASSERTION(!mimeType.IsEmpty(), "We should have the Content-Type.");
NS_ConvertUTF8toUTF16 mimeUTF16(mimeType);
RefPtr<HTMLMediaElement> self = this;
auto reportCanPlay = [&, self](bool aCanPlay) {
diagnostics.StoreFormatDiagnostics(self->OwnerDoc(), mimeUTF16, aCanPlay,
__func__);
if (!aCanPlay) {
nsAutoString src;
self->GetCurrentSrc(src);
AutoTArray<nsString, 2> params = {mimeUTF16, src};
self->ReportLoadError("MediaLoadUnsupportedMimeType", params);
}
};
auto onExit = MakeScopeExit([self] {
if (self->mChannelLoader) {
self->mChannelLoader->Done();
self->mChannelLoader = nullptr;
}
});
Maybe<MediaContainerType> containerType = MakeMediaContainerType(mimeType);
if (!containerType) {
reportCanPlay(false);
return NS_ERROR_FAILURE;
}
MediaDecoderInit decoderInit(
this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
ClampPlaybackRate(mPlaybackRate),
mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
HasAttr(nsGkAtoms::loop), *containerType);
#ifdef MOZ_ANDROID_HLS_SUPPORT
if (HLSDecoder::IsSupportedType(*containerType)) {
RefPtr<HLSDecoder> decoder = HLSDecoder::Create(decoderInit);
if (!decoder) {
reportCanPlay(false);
return NS_ERROR_OUT_OF_MEMORY;
}
reportCanPlay(true);
return SetupDecoder(decoder.get(), aChannel);
}
#endif
RefPtr<ChannelMediaDecoder> decoder =
ChannelMediaDecoder::Create(decoderInit, &diagnostics);
if (!decoder) {
reportCanPlay(false);
return NS_ERROR_FAILURE;
}
reportCanPlay(true);
bool isPrivateBrowsing = NodePrincipal()->GetPrivateBrowsingId() > 0;
return SetupDecoder(decoder.get(), aChannel, isPrivateBrowsing, aListener);
}
nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder) {
ChangeNetworkState(NETWORK_LOADING);
// Set mDecoder now so if methods like GetCurrentSrc get called between
// here and Load(), they work.
SetDecoder(aDecoder);
// Notify the decoder of the initial activity status.
NotifyDecoderActivityChanges();
// Update decoder principal before we start decoding, since it
// can affect how we feed data to MediaStreams
NotifyDecoderPrincipalChanged();
// Set sink device if we have one. Otherwise the default is used.
if (mSink.second) {
mDecoder->SetSink(mSink.second);
}
if (mMediaKeys) {
if (mMediaKeys->GetCDMProxy()) {
mDecoder->SetCDMProxy(mMediaKeys->GetCDMProxy());
} else {
// CDM must have crashed.
ShutdownDecoder();
return NS_ERROR_FAILURE;
}
}
if (mChannelLoader) {
mChannelLoader->Done();
mChannelLoader = nullptr;
}
// We may want to suspend the new stream now.
// This will also do an AddRemoveSelfReference.
NotifyOwnerDocumentActivityChanged();
if (!mDecoder) {
// NotifyOwnerDocumentActivityChanged may shutdown the decoder if the
// owning document is inactive and we're in the EME case. We could try and
// handle this, but at the time of writing it's a pretty niche case, so just
// bail.
return NS_ERROR_FAILURE;
}
if (mSuspendedByInactiveDocOrDocshell) {
mDecoder->Suspend();
}
if (!mPaused) {
SetPlayedOrSeeked(true);
if (!mSuspendedByInactiveDocOrDocshell) {
mDecoder->Play();
}
}
MaybeBeginCloningVisually();
return NS_OK;
}
void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) {
if (!mSrcStream) {
return;
}
bool shouldPlay = !(aFlags & REMOVING_SRC_STREAM) && !mPaused &&
!mSuspendedByInactiveDocOrDocshell;
if (shouldPlay == mSrcStreamIsPlaying) {
return;
}
mSrcStreamIsPlaying = shouldPlay;
LOG(LogLevel::Debug,
("MediaElement %p %s playback of DOMMediaStream %p", this,
shouldPlay ? "Setting up" : "Removing", mSrcStream.get()));
if (shouldPlay) {
mSrcStreamPlaybackEnded = false;
mSrcStreamReportPlaybackEnded = false;
if (mMediaStreamRenderer) {
mMediaStreamRenderer->Start();
}
if (mSecondaryMediaStreamRenderer) {
mSecondaryMediaStreamRenderer->Start();
}
SetCapturedOutputStreamsEnabled(true); // Unmute
// If the input is a media stream, we don't check its data and always regard
// it as audible when it's playing.
SetAudibleState(true);
} else {
if (mMediaStreamRenderer) {
mMediaStreamRenderer->Stop();
}
if (mSecondaryMediaStreamRenderer) {
mSecondaryMediaStreamRenderer->Stop();
}
SetCapturedOutputStreamsEnabled(false); // Mute
}
}
void HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying() {
if (!mMediaStreamRenderer) {
// Notifications are async, the renderer could have been cleared.
return;
}
mMediaStreamRenderer->SetProgressingCurrentTime(IsPotentiallyPlaying());
}
void HTMLMediaElement::UpdateSrcStreamTime() {
MOZ_ASSERT(NS_IsMainThread());
if (mSrcStreamPlaybackEnded) {
// We do a separate FireTimeUpdate() when this is set.
return;
}
FireTimeUpdate(TimeupdateType::ePeriodic);
}
void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) {
NS_ASSERTION(!mSrcStream, "Should have been ended already");
mLoadingSrc = nullptr;
mSrcStream = aStream;
VideoFrameContainer* container = GetVideoFrameContainer();
RefPtr<FirstFrameVideoOutput> firstFrameOutput =
container ? MakeAndAddRef<FirstFrameVideoOutput>(container,
AbstractMainThread())
: nullptr;
mMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
AbstractMainThread(), container, firstFrameOutput, this);
mWatchManager.Watch(mPaused,
&HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
mWatchManager.Watch(mReadyState,
&HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
mWatchManager.Watch(mSrcStreamPlaybackEnded,
&HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
mWatchManager.Watch(mSrcStreamPlaybackEnded,
&HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
mWatchManager.Watch(mMediaStreamRenderer->CurrentGraphTime(),
&HTMLMediaElement::UpdateSrcStreamTime);
SetVolumeInternal();
if (mSink.second) {
mMediaStreamRenderer->SetAudioOutputDevice(mSink.second);
}
UpdateSrcMediaStreamPlaying();
UpdateSrcStreamPotentiallyPlaying();
mSrcStreamVideoPrincipal = NodePrincipal();
// If we pause this media element, track changes in the underlying stream
// will continue to fire events at this element and alter its track list.
// That's simpler than delaying the events, but probably confusing...
nsTArray<RefPtr<MediaStreamTrack>> tracks;
mSrcStream->GetTracks(tracks);
for (const RefPtr<MediaStreamTrack>& track : tracks) {
NotifyMediaStreamTrackAdded(track);
}
mMediaStreamTrackListener = new MediaStreamTrackListener(this);
mSrcStream->RegisterTrackListener(mMediaStreamTrackListener.get());
ChangeNetworkState(NETWORK_IDLE);
ChangeDelayLoadStatus(false);
// FirstFrameLoaded() will be called when the stream has tracks.
}
void HTMLMediaElement::EndSrcMediaStreamPlayback() {
MOZ_ASSERT(mSrcStream);
UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
if (mSelectedVideoStreamTrack) {
mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
}
mSelectedVideoStreamTrack = nullptr;
MOZ_ASSERT_IF(mSecondaryMediaStreamRenderer,
!mMediaStreamRenderer == !mSecondaryMediaStreamRenderer);
if (mMediaStreamRenderer) {
mWatchManager.Unwatch(mPaused,
&HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
mWatchManager.Unwatch(mReadyState,
&HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
mWatchManager.Unwatch(mSrcStreamPlaybackEnded,
&HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
mWatchManager.Unwatch(
mSrcStreamPlaybackEnded,
&HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
mWatchManager.Unwatch(mMediaStreamRenderer->CurrentGraphTime(),
&HTMLMediaElement::UpdateSrcStreamTime);
mMediaStreamRenderer->Shutdown();
mMediaStreamRenderer = nullptr;
}
if (mSecondaryMediaStreamRenderer) {
mSecondaryMediaStreamRenderer->Shutdown();
mSecondaryMediaStreamRenderer = nullptr;
}
mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener.get());
mMediaStreamTrackListener = nullptr;
mSrcStreamPlaybackEnded = false;
mSrcStreamReportPlaybackEnded = false;
mSrcStreamVideoPrincipal = nullptr;
mSrcStream = nullptr;
}
static already_AddRefed<AudioTrack> CreateAudioTrack(
AudioStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
nsAutoString id;
nsAutoString label;
aStreamTrack->GetId(id);
aStreamTrack->GetLabel(label, CallerType::System);
return MediaTrackList::CreateAudioTrack(aOwnerGlobal, id, u"main"_ns, label,
u""_ns, true, aStreamTrack);
}
static already_AddRefed<VideoTrack> CreateVideoTrack(
VideoStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
nsAutoString id;
nsAutoString label;
aStreamTrack->GetId(id);
aStreamTrack->GetLabel(label, CallerType::System);
return MediaTrackList::CreateVideoTrack(aOwnerGlobal, id, u"main"_ns, label,
u""_ns, aStreamTrack);
}
void HTMLMediaElement::NotifyMediaStreamTrackAdded(
const RefPtr<MediaStreamTrack>& aTrack) {
MOZ_ASSERT(aTrack);
if (aTrack->Ended()) {
return;
}
#ifdef DEBUG
nsAutoString id;
aTrack->GetId(id);
LOG(LogLevel::Debug, ("%p, Adding %sTrack with id %s", this,
aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
NS_ConvertUTF16toUTF8(id).get()));
#endif
if (AudioStreamTrack* t = aTrack->AsAudioStreamTrack()) {
MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
RefPtr<AudioTrack> audioTrack =
CreateAudioTrack(t, AudioTracks()->GetOwnerGlobal());
AudioTracks()->AddTrack(audioTrack);
} else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
if (!IsVideo()) {
return;
}
MOZ_DIAGNOSTIC_ASSERT(VideoTracks(), "Element can't have been unlinked");
RefPtr<VideoTrack> videoTrack =
CreateVideoTrack(t, VideoTracks()->GetOwnerGlobal());
VideoTracks()->AddTrack(videoTrack);
// New MediaStreamTrack added, set the new added video track as selected
// video track when there is no selected track.
if (VideoTracks()->SelectedIndex() == -1) {
MOZ_ASSERT(!mSelectedVideoStreamTrack);
videoTrack->SetEnabledInternal(true, dom::MediaTrack::FIRE_NO_EVENTS);
}
}
// The set of enabled AudioTracks and selected video track might have changed.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
AbstractThread::DispatchDirectTask(
NewRunnableMethod("HTMLMediaElement::FirstFrameLoaded", this,
&HTMLMediaElement::FirstFrameLoaded));
}
void HTMLMediaElement::NotifyMediaStreamTrackRemoved(
const RefPtr<MediaStreamTrack>& aTrack) {
MOZ_ASSERT(aTrack);
nsAutoString id;
aTrack->GetId(id);
LOG(LogLevel::Debug, ("%p, Removing %sTrack with id %s", this,
aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
NS_ConvertUTF16toUTF8(id).get()));
MOZ_DIAGNOSTIC_ASSERT(AudioTracks() && VideoTracks(),
"Element can't have been unlinked");
if (dom::MediaTrack* t = AudioTracks()->GetTrackById(id)) {
AudioTracks()->RemoveTrack(t);
} else if (dom::MediaTrack* t = VideoTracks()->GetTrackById(id)) {
VideoTracks()->RemoveTrack(t);
} else {
NS_ASSERTION(aTrack->AsVideoStreamTrack() && !IsVideo(),
"MediaStreamTrack ended but did not exist in track lists. "
"This is only allowed if a video element ends and we are an "
"audio element.");
return;
}
}
void HTMLMediaElement::ProcessMediaFragmentURI() {
if (!mLoadingSrc) {
mFragmentStart = mFragmentEnd = -1.0;
return;
}
nsMediaFragmentURIParser parser(mLoadingSrc);
if (mDecoder && parser.HasEndTime()) {
mFragmentEnd = parser.GetEndTime();
}
if (parser.HasStartTime()) {
SetCurrentTime(parser.GetStartTime());
mFragmentStart = parser.GetStartTime();
}
}
void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo,
UniquePtr<const MetadataTags> aTags) {
MOZ_ASSERT(NS_IsMainThread());
if (mDecoder) {
ConstructMediaTracks(aInfo);
}
SetMediaInfo(*aInfo);
mIsEncrypted =
aInfo->IsEncrypted() || mPendingEncryptedInitData.IsEncrypted();
mTags = std::move(aTags);
mLoadedDataFired = false;
ChangeReadyState(HAVE_METADATA);
// Add output tracks synchronously now to be sure they're available in
// "loadedmetadata" event handlers.
UpdateOutputTrackSources();
DispatchAsyncEvent(u"durationchange"_ns);
if (IsVideo() && HasVideo()) {
DispatchAsyncEvent(u"resize"_ns);
Invalidate(ImageSizeChanged::No, Some(mMediaInfo.mVideo.mDisplay),
ForceInvalidate::No);
}
NS_ASSERTION(!HasVideo() || (mMediaInfo.mVideo.mDisplay.width > 0 &&
mMediaInfo.mVideo.mDisplay.height > 0),
"Video resolution must be known on 'loadedmetadata'");
DispatchAsyncEvent(u"loadedmetadata"_ns);
if (mDecoder && mDecoder->IsTransportSeekable() &&
mDecoder->IsMediaSeekable()) {
ProcessMediaFragmentURI();
mDecoder->SetFragmentEndTime(mFragmentEnd);
}
if (mIsEncrypted) {
// We only support playback of encrypted content via MSE by default.
if (!mMediaSource && Preferences::GetBool("media.eme.mse-only", true)) {
DecodeError(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
"Encrypted content not supported outside of MSE"));
return;
}
// Dispatch a distinct 'encrypted' event for each initData we have.
for (const auto& initData : mPendingEncryptedInitData.mInitDatas) {
DispatchEncrypted(initData.mInitData, initData.mType);
}
mPendingEncryptedInitData.Reset();
}
if (IsVideo() && aInfo->HasVideo()) {
// We are a video element playing video so update the screen wakelock
NotifyOwnerDocumentActivityChanged();
}
if (mDefaultPlaybackStartPosition != 0.0) {
SetCurrentTime(mDefaultPlaybackStartPosition);
mDefaultPlaybackStartPosition = 0.0;
}
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
}
void HTMLMediaElement::FirstFrameLoaded() {
LOG(LogLevel::Debug,
("%p, FirstFrameLoaded() mFirstFrameLoaded=%d mWaitingForKey=%d", this,
mFirstFrameLoaded.Ref(), mWaitingForKey));
NS_ASSERTION(!mSuspendedAfterFirstFrame, "Should not have already suspended");
if (!mFirstFrameLoaded) {
mFirstFrameLoaded = true;
}
ChangeDelayLoadStatus(false);
if (mDecoder && mAllowSuspendAfterFirstFrame && mPaused &&
!HasAttr(nsGkAtoms::autoplay) &&
mPreloadAction == HTMLMediaElement::PRELOAD_METADATA) {
mSuspendedAfterFirstFrame = true;
mDecoder->Suspend();
}
}
void HTMLMediaElement::NetworkError(const MediaResult& aError) {
if (mReadyState == HAVE_NOTHING) {
NoSupportedMediaSourceError(aError.Description());
} else {
Error(MEDIA_ERR_NETWORK);
}
}
void HTMLMediaElement::DecodeError(const MediaResult& aError) {
nsAutoString src;
GetCurrentSrc(src);
AutoTArray<nsString, 1> params = {src};
ReportLoadError("MediaLoadDecodeError", params);
DecoderDoctorDiagnostics diagnostics;
diagnostics.StoreDecodeError(OwnerDoc(), aError, src, __func__);
if (mIsLoadingFromSourceChildren) {
mErrorSink->ResetError();
if (mSourceLoadCandidate) {
DispatchAsyncSourceError(mSourceLoadCandidate);
QueueLoadFromSourceTask();
} else {
NS_WARNING("Should know the source we were loading from!");
}
} else if (mReadyState == HAVE_NOTHING) {
NoSupportedMediaSourceError(aError.Description());
} else if (IsCORSSameOrigin()) {
Error(MEDIA_ERR_DECODE, aError.Description());
} else {
Error(MEDIA_ERR_DECODE, "Failed to decode media"_ns);
}
}
void HTMLMediaElement::DecodeWarning(const MediaResult& aError) {
nsAutoString src;
GetCurrentSrc(src);
DecoderDoctorDiagnostics diagnostics;
diagnostics.StoreDecodeWarning(OwnerDoc(), aError, src, __func__);
}
bool HTMLMediaElement::HasError() const { return GetError(); }
void HTMLMediaElement::LoadAborted() { Error(MEDIA_ERR_ABORTED); }
void HTMLMediaElement::Error(uint16_t aErrorCode,
const nsACString& aErrorDetails) {
mErrorSink->SetError(aErrorCode, aErrorDetails);
ChangeDelayLoadStatus(false);
UpdateAudioChannelPlayingState();
}
void HTMLMediaElement::PlaybackEnded() {
// We changed state which can affect AddRemoveSelfReference
AddRemoveSelfReference();
NS_ASSERTION(!mDecoder || mDecoder->IsEnded(),
"Decoder fired ended, but not in ended state");
// IsPlaybackEnded() became true.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
if (mSrcStream) {
LOG(LogLevel::Debug,
("%p, got duration by reaching the end of the resource", this));
mSrcStreamPlaybackEnded = true;
DispatchAsyncEvent(u"durationchange"_ns);
} else {
// mediacapture-main:
// Setting the loop attribute has no effect since a MediaStream has no
// defined end and therefore cannot be looped.
if (HasAttr(nsGkAtoms::loop)) {
SetCurrentTime(0);
return;
}
}
FireTimeUpdate(TimeupdateType::eMandatory);
if (!mPaused) {
Pause();
}
if (mSrcStream) {
// A MediaStream that goes from inactive to active shall be eligible for
// autoplay again according to the mediacapture-main spec.
mCanAutoplayFlag = true;
}
if (StaticPrefs::media_mediacontrol_stopcontrol_aftermediaends()) {
mMediaControlKeyListener->StopIfNeeded();
}
DispatchAsyncEvent(u"ended"_ns);
}
void HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded() {
mSrcStreamReportPlaybackEnded = mSrcStreamPlaybackEnded;
}
void HTMLMediaElement::SeekStarted() { DispatchAsyncEvent(u"seeking"_ns); }
void HTMLMediaElement::SeekCompleted() {
mPlayingBeforeSeek = false;
SetPlayedOrSeeked(true);
if (mTextTrackManager) {
mTextTrackManager->DidSeek();
}
// (Step 16)
FireTimeUpdate(TimeupdateType::eMandatory);
DispatchAsyncEvent(u"seeked"_ns);
// We changed whether we're seeking so we need to AddRemoveSelfReference
AddRemoveSelfReference();
if (mCurrentPlayRangeStart == -1.0) {
mCurrentPlayRangeStart = CurrentTime();
}
if (mSeekDOMPromise) {
AbstractMainThread()->Dispatch(NS_NewRunnableFunction(
__func__, [promise = std::move(mSeekDOMPromise)] {
promise->MaybeResolveWithUndefined();
}));
}
MOZ_ASSERT(!mSeekDOMPromise);
}
void HTMLMediaElement::SeekAborted() {
if (mSeekDOMPromise) {
AbstractMainThread()->Dispatch(NS_NewRunnableFunction(
__func__, [promise = std::move(mSeekDOMPromise)] {
promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
}));
}
MOZ_ASSERT(!mSeekDOMPromise);
}
void HTMLMediaElement::NotifySuspendedByCache(bool aSuspendedByCache) {
LOG(LogLevel::Debug,
("%p, mDownloadSuspendedByCache=%d", this, aSuspendedByCache));
mDownloadSuspendedByCache = aSuspendedByCache;
}
void HTMLMediaElement::DownloadSuspended() {
if (mNetworkState == NETWORK_LOADING) {
DispatchAsyncEvent(u"progress"_ns);
}
ChangeNetworkState(NETWORK_IDLE);
}
void HTMLMediaElement::DownloadResumed() {
ChangeNetworkState(NETWORK_LOADING);
}
void HTMLMediaElement::CheckProgress(bool aHaveNewProgress) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
TimeStamp now = TimeStamp::NowLoRes();
if (aHaveNewProgress) {
mDataTime = now;
}
// If this is the first progress, or PROGRESS_MS has passed since the last
// progress event fired and more data has arrived since then, fire a
// progress event.
NS_ASSERTION(
(mProgressTime.IsNull() && !aHaveNewProgress) || !mDataTime.IsNull(),
"null TimeStamp mDataTime should not be used in comparison");
if (mProgressTime.IsNull()
? aHaveNewProgress
: (now - mProgressTime >=
TimeDuration::FromMilliseconds(PROGRESS_MS) &&
mDataTime > mProgressTime)) {
DispatchAsyncEvent(u"progress"_ns);
// Resolution() ensures that future data will have now > mProgressTime,
// and so will trigger another event. mDataTime is not reset because it
// is still required to detect stalled; it is similarly offset by
// resolution to indicate the new data has not yet arrived.
mProgressTime = now - TimeDuration::Resolution();
if (mDataTime > mProgressTime) {
mDataTime = mProgressTime;
}
if (!mProgressTimer) {
NS_ASSERTION(aHaveNewProgress,
"timer dispatched when there was no timer");
// Were stalled. Restart timer.
StartProgressTimer();
if (!mLoadedDataFired) {
ChangeDelayLoadStatus(true);
}
}
// Download statistics may have been updated, force a recheck of the
// readyState.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
}
if (now - mDataTime >= TimeDuration::FromMilliseconds(STALL_MS)) {
if (!mMediaSource) {
DispatchAsyncEvent(u"stalled"_ns);
} else {
ChangeDelayLoadStatus(false);
}
NS_ASSERTION(mProgressTimer, "detected stalled without timer");
// Stop timer events, which prevents repeated stalled events until there
// is more progress.
StopProgress();
}
AddRemoveSelfReference();
}
/* static */
void HTMLMediaElement::ProgressTimerCallback(nsITimer* aTimer, void* aClosure) {
auto* decoder = static_cast<HTMLMediaElement*>(aClosure);
decoder->CheckProgress(false);
}
void HTMLMediaElement::StartProgressTimer() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
NS_ASSERTION(!mProgressTimer, "Already started progress timer.");
NS_NewTimerWithFuncCallback(
getter_AddRefs(mProgressTimer), ProgressTimerCallback, this, PROGRESS_MS,
nsITimer::TYPE_REPEATING_SLACK, "HTMLMediaElement::ProgressTimerCallback",
GetMainThreadSerialEventTarget());
}
void HTMLMediaElement::StartProgress() {
// Record the time now for detecting stalled.
mDataTime = TimeStamp::NowLoRes();
// Reset mProgressTime so that mDataTime is not indicating bytes received
// after the last progress event.
mProgressTime = TimeStamp();
StartProgressTimer();
}
void HTMLMediaElement::StopProgress() {
MOZ_ASSERT(NS_IsMainThread());
if (!mProgressTimer) {
return;
}
mProgressTimer->Cancel();
mProgressTimer = nullptr;
}
void HTMLMediaElement::DownloadProgressed() {
if (mNetworkState != NETWORK_LOADING) {
return;
}
CheckProgress(true);
}
bool HTMLMediaElement::ShouldCheckAllowOrigin() {
return mCORSMode != CORS_NONE;
}
bool HTMLMediaElement::IsCORSSameOrigin() {
bool subsumes;
RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
return (NS_SUCCEEDED(NodePrincipal()->Subsumes(principal, &subsumes)) &&
subsumes) ||
ShouldCheckAllowOrigin();
}
void HTMLMediaElement::UpdateReadyStateInternal() {
if (!mDecoder && !mSrcStream) {
// Not initialized - bail out.
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Not initialized",
this));
return;
}
if (mDecoder && mReadyState < HAVE_METADATA) {
// aNextFrame might have a next frame because the decoder can advance
// on its own thread before MetadataLoaded gets a chance to run.
// The arrival of more data can't change us out of this readyState.
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Decoder ready state < HAVE_METADATA",
this));
return;
}
if (mDecoder) {
// IsPlaybackEnded() might have become false.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
}
if (mSrcStream && mReadyState < HAVE_METADATA) {
bool hasAudioTracks = AudioTracks() && !AudioTracks()->IsEmpty();
bool hasVideoTracks = VideoTracks() && !VideoTracks()->IsEmpty();
if (!hasAudioTracks && !hasVideoTracks) {
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Stream with no tracks",
this));
// Give it one last chance to remove the self reference if needed.
AddRemoveSelfReference();
return;
}
if (IsVideo() && hasVideoTracks && !HasVideo()) {
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Stream waiting for video",
this));
return;
}
LOG(LogLevel::Debug,
("MediaElement %p UpdateReadyStateInternal() Stream has "
"metadata; audioTracks=%d, videoTracks=%d, "
"hasVideoFrame=%d",
this, AudioTracks()->Length(), VideoTracks()->Length(), HasVideo()));
// We are playing a stream that has video and a video frame is now set.
// This means we have all metadata needed to change ready state.
MediaInfo mediaInfo = mMediaInfo;
if (hasAudioTracks) {
mediaInfo.EnableAudio();
}
if (hasVideoTracks) {
mediaInfo.EnableVideo();
if (mSelectedVideoStreamTrack) {
mediaInfo.mVideo.SetAlpha(mSelectedVideoStreamTrack->HasAlpha());
}
}
MetadataLoaded(&mediaInfo, nullptr);
}
if (mMediaSource) {
// readyState has changed, assuming it's following the pending mediasource
// operations. Notify the Mediasource that the operations have completed.
mMediaSource->CompletePendingTransactions();
}
enum NextFrameStatus nextFrameStatus = NextFrameStatus();
if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE && mDecoder &&
!mDecoder->IsEnded()) {
nextFrameStatus = mDecoder->NextFrameBufferedStatus();
}
} else if (mWaitingForKey == WAITING_FOR_KEY) {
if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE ||
nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
// Continuing 7.3.4 Queue a "waitingforkey" Event
// 4. Queue a task to fire a simple event named waitingforkey
// at the media element.
// 5. Set the readyState of media element to HAVE_METADATA.
// NOTE: We'll change to HAVE_CURRENT_DATA or HAVE_METADATA
// depending on whether we've loaded the first frame or not
// below.
// 6. Suspend playback.
// Note: Playback will already be stalled, as the next frame is
// unavailable.
mWaitingForKey = WAITING_FOR_KEY_DISPATCHED;
DispatchAsyncEvent(u"waitingforkey"_ns);
}
} else {
MOZ_ASSERT(mWaitingForKey == WAITING_FOR_KEY_DISPATCHED);
if (nextFrameStatus == NEXT_FRAME_AVAILABLE) {
// We have new frames after dispatching "waitingforkey".
// This means we've got the key and can reset mWaitingForKey now.
mWaitingForKey = NOT_WAITING_FOR_KEY;
}
}
if (nextFrameStatus == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING) {
LOG(LogLevel::Debug,
("MediaElement %p UpdateReadyStateInternal() "
"NEXT_FRAME_UNAVAILABLE_SEEKING; Forcing HAVE_METADATA",
this));
ChangeReadyState(HAVE_METADATA);
return;
}
if (IsVideo() && VideoTracks() && !VideoTracks()->IsEmpty() &&
!IsPlaybackEnded() && GetImageContainer() &&
!GetImageContainer()->HasCurrentImage()) {
// Don't advance if we are playing video, but don't have a video frame.
// Also, if video became available after advancing to HAVE_CURRENT_DATA
// while we are still playing, we need to revert to HAVE_METADATA until
// a video frame is available.
LOG(LogLevel::Debug,
("MediaElement %p UpdateReadyStateInternal() "
"Playing video but no video frame; Forcing HAVE_METADATA",
this));
ChangeReadyState(HAVE_METADATA);
return;
}
if (!mFirstFrameLoaded) {
// We haven't yet loaded the first frame, making us unable to determine
// if we have enough valid data at the present stage.
return;
}
if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
// Force HAVE_CURRENT_DATA when buffering.
ChangeReadyState(HAVE_CURRENT_DATA);
return;
}
// TextTracks must be loaded for the HAVE_ENOUGH_DATA and
// HAVE_FUTURE_DATA.
// So force HAVE_CURRENT_DATA if text tracks not loaded.
if (mTextTrackManager && !mTextTrackManager->IsLoaded()) {
ChangeReadyState(HAVE_CURRENT_DATA);
return;
}
if (mDownloadSuspendedByCache && mDecoder && !mDecoder->IsEnded()) {
// The decoder has signaled that the download has been suspended by the
// media cache. So move readyState into HAVE_ENOUGH_DATA, in case there's
// script waiting for a "canplaythrough" event; without this forced
// transition, we will never fire the "canplaythrough" event if the
// media cache is too small, and scripts are bound to fail. Don't force
// this transition if the decoder is in ended state; the readyState
// should remain at HAVE_CURRENT_DATA in this case.
// Note that this state transition includes the case where we finished
// downloaded the whole data stream.
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Decoder download suspended by cache",
this));
ChangeReadyState(HAVE_ENOUGH_DATA);
return;
}
if (nextFrameStatus != MediaDecoderOwner::NEXT_FRAME_AVAILABLE) {
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Next frame not available",
this));
ChangeReadyState(HAVE_CURRENT_DATA);
return;
}
if (mSrcStream) {
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Stream HAVE_ENOUGH_DATA",
this));
ChangeReadyState(HAVE_ENOUGH_DATA);
return;
}
// Now see if we should set HAVE_ENOUGH_DATA.
// If it's something we don't know the size of, then we can't
// make a real estimate, so we go straight to HAVE_ENOUGH_DATA once
// we've downloaded enough data that our download rate is considered
// reliable. We have to move to HAVE_ENOUGH_DATA at some point or
// autoplay elements for live streams will never play. Otherwise we
// move to HAVE_ENOUGH_DATA if we can play through the entire media
// without stopping to buffer.
if (mDecoder->CanPlayThrough()) {
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Decoder can play through",
this));
ChangeReadyState(HAVE_ENOUGH_DATA);
return;
}
LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
"Default; Decoder has future data",
this));
ChangeReadyState(HAVE_FUTURE_DATA);
}
static const char* const gReadyStateToString[] = {
"HAVE_NOTHING", "HAVE_METADATA", "HAVE_CURRENT_DATA", "HAVE_FUTURE_DATA",
"HAVE_ENOUGH_DATA"};
void HTMLMediaElement::ChangeReadyState(nsMediaReadyState aState) {
if (mReadyState == aState) {
return;
}
nsMediaReadyState oldState = mReadyState;
mReadyState = aState;
LOG(LogLevel::Debug,
("%p Ready state changed to %s", this, gReadyStateToString[aState]));
DDLOG(DDLogCategory::Property, "ready_state", gReadyStateToString[aState]);
// The user agent must synchronously unset cues' active flag whenever the
// media element's readyState is changed back to HAVE_NOTHING.
if (mReadyState == HAVE_NOTHING && mTextTrackManager) {
mTextTrackManager->NotifyReset();
}
if (mNetworkState == NETWORK_EMPTY) {
return;
}
UpdateAudioChannelPlayingState();
// Handle raising of "waiting" event during seek (see 4.8.10.9)
// or
// 4.8.12.7 Ready states:
// "If the previous ready state was HAVE_FUTURE_DATA or more, and the new
// ready state is HAVE_CURRENT_DATA or less
// If the media element was potentially playing before its readyState
// attribute changed to a value lower than HAVE_FUTURE_DATA, and the element
// has not ended playback, and playback has not stopped due to errors,
// paused for user interaction, or paused for in-band content, the user agent
// must queue a task to fire a simple event named timeupdate at the element,
// and queue a task to fire a simple event named waiting at the element."
if (mPlayingBeforeSeek && mReadyState < HAVE_FUTURE_DATA) {
DispatchAsyncEvent(u"waiting"_ns);
} else if (oldState >= HAVE_FUTURE_DATA && mReadyState < HAVE_FUTURE_DATA &&
!Paused() && !Ended() && !mErrorSink->mError) {
FireTimeUpdate(TimeupdateType::eMandatory);
DispatchAsyncEvent(u"waiting"_ns);
}
if (oldState < HAVE_CURRENT_DATA && mReadyState >= HAVE_CURRENT_DATA &&
!mLoadedDataFired) {
DispatchAsyncEvent(u"loadeddata"_ns);
mLoadedDataFired = true;
}
if (oldState < HAVE_FUTURE_DATA && mReadyState >= HAVE_FUTURE_DATA) {
DispatchAsyncEvent(u"canplay"_ns);
if (!mPaused) {
if (mDecoder && !mSuspendedByInactiveDocOrDocshell) {
MOZ_ASSERT(AllowedToPlay());
mDecoder->Play();
}
NotifyAboutPlaying();
}
}
CheckAutoplayDataReady();
if (oldState < HAVE_ENOUGH_DATA && mReadyState >= HAVE_ENOUGH_DATA) {
DispatchAsyncEvent(u"canplaythrough"_ns);
}
}
static const char* const gNetworkStateToString[] = {"EMPTY", "IDLE", "LOADING",
"NO_SOURCE"};
void HTMLMediaElement::ChangeNetworkState(nsMediaNetworkState aState) {
if (mNetworkState == aState) {
return;
}
nsMediaNetworkState oldState = mNetworkState;
mNetworkState = aState;
LOG(LogLevel::Debug,
("%p Network state changed to %s", this, gNetworkStateToString[aState]));
DDLOG(DDLogCategory::Property, "network_state",
gNetworkStateToString[aState]);
if (oldState == NETWORK_LOADING) {
// Stop progress notification when exiting NETWORK_LOADING.
StopProgress();
}
if (mNetworkState == NETWORK_LOADING) {
// Start progress notification when entering NETWORK_LOADING.
StartProgress();
} else if (mNetworkState == NETWORK_IDLE && !mErrorSink->mError) {
// Fire 'suspend' event when entering NETWORK_IDLE and no error presented.
DispatchAsyncEvent(u"suspend"_ns);
}
// According to the resource selection (step2, step9-18), dedicated media
// source failure step (step4) and aborting existing load (step4), set show
if (mNetworkState == NETWORK_NO_SOURCE || mNetworkState == NETWORK_EMPTY) {
mShowPoster = true;
}
// Changing mNetworkState affects AddRemoveSelfReference().
AddRemoveSelfReference();
}
bool HTMLMediaElement::IsEligibleForAutoplay() {
// We also activate autoplay when playing a media source since the data
// download is controlled by the script and there is no way to evaluate
// MediaDecoder::CanPlayThrough().
if (!HasAttr(nsGkAtoms::autoplay)) {
return false;
}
if (!mCanAutoplayFlag) {
return false;
}
if (IsEditable()) {
return false;
}
if (!mPaused) {
return false;
}
if (mSuspendedByInactiveDocOrDocshell) {
return false;
}
// Static document is used for print preview and printing, should not be
// autoplay
if (OwnerDoc()->IsStaticDocument()) {
return false;
}
if (ShouldBeSuspendedByInactiveDocShell()) {
LOG(LogLevel::Debug, ("%p prohibiting autoplay by the docShell", this));
return false;
}
if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
CreateResumeDelayedMediaPlaybackAgentIfNeeded();
LOG(LogLevel::Debug, ("%p delay playing from autoplay", this));
return false;
}
return mReadyState >= HAVE_ENOUGH_DATA;
}
void HTMLMediaElement::CheckAutoplayDataReady() {
if (!IsEligibleForAutoplay()) {
return;
}
if (!AllowedToPlay()) {
DispatchEventsWhenPlayWasNotAllowed();
return;
}
RunAutoplay();
}
void HTMLMediaElement::RunAutoplay() {
mAllowedToPlayPromise.ResolveIfExists(true, __func__);
mPaused = false;
// We changed mPaused which can affect AddRemoveSelfReference
AddRemoveSelfReference();
UpdateSrcMediaStreamPlaying();
UpdateAudioChannelPlayingState();
StartMediaControlKeyListenerIfNeeded();
if (mDecoder) {
SetPlayedOrSeeked(true);
if (mCurrentPlayRangeStart == -1.0) {
mCurrentPlayRangeStart = CurrentTime();
}
MOZ_ASSERT(!mSuspendedByInactiveDocOrDocshell);
mDecoder->Play();
} else if (mSrcStream) {
SetPlayedOrSeeked(true);
}
if (mShowPoster) {
mShowPoster = false;
if (mTextTrackManager) {
mTextTrackManager->TimeMarchesOn();
}
}
// For blocked media, the event would be pending until it is resumed.
DispatchAsyncEvent(u"play"_ns);
DispatchAsyncEvent(u"playing"_ns);
}
bool HTMLMediaElement::IsActuallyInvisible() const {
// That means an element is not connected. It probably hasn't connected to a
// document tree, or connects to a disconnected DOM tree.
if (!IsInComposedDoc()) {
return true;
}
// An element is not in user's view port, which means it's either existing in
// somewhere in the page where user hasn't seen yet, or is being set
// `display:none`.
if (!IsInViewPort()) {
return true;
}
// Element being used in picture-in-picture mode would be always visible.
if (IsBeingUsedInPictureInPictureMode()) {
return false;
}
// That check is the page is in the background.
return OwnerDoc()->Hidden();
}
bool HTMLMediaElement::IsInViewPort() const {
return mVisibilityState == Visibility::ApproximatelyVisible;
}
VideoFrameContainer* HTMLMediaElement::GetVideoFrameContainer() {
if (mShuttingDown) {
return nullptr;
}
if (mVideoFrameContainer) return mVideoFrameContainer;
// Only video frames need an image container.
if (!IsVideo()) {
return nullptr;
}
mVideoFrameContainer = new VideoFrameContainer(
this, MakeAndAddRef<ImageContainer>(ImageContainer::ASYNCHRONOUS));
return mVideoFrameContainer;
}
void HTMLMediaElement::PrincipalChanged(MediaStreamTrack* aTrack) {
if (aTrack != mSelectedVideoStreamTrack) {
return;
}
nsContentUtils::CombineResourcePrincipals(&mSrcStreamVideoPrincipal,
aTrack->GetPrincipal());
LOG(LogLevel::Debug,
("HTMLMediaElement %p video track principal changed to %p (combined "
"into %p). Waiting for it to reach VideoFrameContainer before setting.",
this, aTrack->GetPrincipal(), mSrcStreamVideoPrincipal.get()));
if (mVideoFrameContainer) {
UpdateSrcStreamVideoPrincipal(
mVideoFrameContainer->GetLastPrincipalHandle());
}
}
void HTMLMediaElement::UpdateSrcStreamVideoPrincipal(
const PrincipalHandle& aPrincipalHandle) {
nsTArray<RefPtr<VideoStreamTrack>> videoTracks;
mSrcStream->GetVideoTracks(videoTracks);
for (const RefPtr<VideoStreamTrack>& track : videoTracks) {
if (PrincipalHandleMatches(aPrincipalHandle, track->GetPrincipal()) &&
!track->Ended()) {
// When the PrincipalHandle for the VideoFrameContainer changes to that of
// a live track in mSrcStream we know that a removed track was displayed
// but is no longer so.
LOG(LogLevel::Debug, ("HTMLMediaElement %p VideoFrameContainer's "
"PrincipalHandle matches track %p. That's all we "
"need.",
this, track.get()));
mSrcStreamVideoPrincipal = track->GetPrincipal();
break;
}
}
}
void HTMLMediaElement::PrincipalHandleChangedForVideoFrameContainer(
VideoFrameContainer* aContainer,
const PrincipalHandle& aNewPrincipalHandle) {
MOZ_ASSERT(NS_IsMainThread());
if (!mSrcStream) {
return;
}
LOG(LogLevel::Debug, ("HTMLMediaElement %p PrincipalHandle changed in "
"VideoFrameContainer.",
this));
UpdateSrcStreamVideoPrincipal(aNewPrincipalHandle);
}
already_AddRefed<nsMediaEventRunner> HTMLMediaElement::GetEventRunner(
const nsAString& aName, EventFlag aFlag) {
RefPtr<nsMediaEventRunner> runner;
if (aName.EqualsLiteral("playing")) {
runner = new nsNotifyAboutPlayingRunner(this, TakePendingPlayPromises());
} else if (aName.EqualsLiteral("timeupdate")) {
runner = new nsTimeupdateRunner(this, aFlag == EventFlag::eMandatory);
} else {
runner = new nsAsyncEventRunner(aName, this);
}
return runner.forget();
}
nsresult HTMLMediaElement::DispatchEvent(const nsAString& aName) {
LOG_EVENT(LogLevel::Debug, ("%p Dispatching event %s", this,
NS_ConvertUTF16toUTF8(aName).get()));
if (mEventBlocker->ShouldBlockEventDelivery()) {
RefPtr<nsMediaEventRunner> runner = GetEventRunner(aName);
mEventBlocker->PostponeEvent(runner);
return NS_OK;
}
return nsContentUtils::DispatchTrustedEvent(OwnerDoc(), this, aName,
CanBubble::eNo, Cancelable::eNo);
}
void HTMLMediaElement::DispatchAsyncEvent(const nsAString& aName) {
RefPtr<nsMediaEventRunner> runner = GetEventRunner(aName);
DispatchAsyncEvent(std::move(runner));
}
void HTMLMediaElement::DispatchAsyncEvent(RefPtr<nsMediaEventRunner> aRunner) {
NS_ConvertUTF16toUTF8 eventName(aRunner->EventName());
LOG_EVENT(LogLevel::Debug, ("%p Queuing event %s", this, eventName.get()));
DDLOG(DDLogCategory::Event, "HTMLMediaElement", nsCString(eventName.get()));
if (mEventBlocker->ShouldBlockEventDelivery()) {
mEventBlocker->PostponeEvent(aRunner);
return;
}
GetMainThreadSerialEventTarget()->Dispatch(aRunner.forget());
}
bool HTMLMediaElement::IsPotentiallyPlaying() const {
// TODO:
// playback has not stopped due to errors,
// and the element has not paused for user interaction
return !mPaused &&
(mReadyState == HAVE_ENOUGH_DATA || mReadyState == HAVE_FUTURE_DATA) &&
!IsPlaybackEnded();
}
bool HTMLMediaElement::IsPlaybackEnded() const {
// TODO:
// the current playback position is equal to the effective end of the media
if (mDecoder) {
return mReadyState >= HAVE_METADATA && mDecoder->IsEnded();
}
if (mSrcStream) {
return mReadyState >= HAVE_METADATA && mSrcStreamPlaybackEnded;
}
return false;
}
already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentPrincipal() {
if (mDecoder) {
return mDecoder->GetCurrentPrincipal();
}
if (mSrcStream) {
nsTArray<RefPtr<MediaStreamTrack>> tracks;
mSrcStream->GetTracks(tracks);
nsCOMPtr<nsIPrincipal> principal = mSrcStream->GetPrincipal();
return principal.forget();
}
return nullptr;
}
bool HTMLMediaElement::HadCrossOriginRedirects() {
if (mDecoder) {
return mDecoder->HadCrossOriginRedirects();
}
return false;
}
bool HTMLMediaElement::ShouldResistFingerprinting(RFPTarget aTarget) const {
return OwnerDoc()->ShouldResistFingerprinting(aTarget);
}
already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentVideoPrincipal() {
if (mDecoder) {
return mDecoder->GetCurrentPrincipal();
}
if (mSrcStream) {
nsCOMPtr<nsIPrincipal> principal = mSrcStreamVideoPrincipal;
return principal.forget();
}
return nullptr;
}
void HTMLMediaElement::NotifyDecoderPrincipalChanged() {
RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
bool isSameOrigin = !principal || IsCORSSameOrigin();
mDecoder->UpdateSameOriginStatus(isSameOrigin);
if (isSameOrigin) {
principal = NodePrincipal();
}
for (const auto& entry : mOutputTrackSources.Values()) {
entry->SetPrincipal(principal);
}
mDecoder->SetOutputTracksPrincipal(principal);
}
void HTMLMediaElement::Invalidate(ImageSizeChanged aImageSizeChanged,
const Maybe<nsIntSize>& aNewIntrinsicSize,
ForceInvalidate aForceInvalidate) {
nsIFrame* frame = GetPrimaryFrame();
if (aNewIntrinsicSize) {
UpdateMediaSize(aNewIntrinsicSize.value());
if (frame) {
nsPresContext* presContext = frame->PresContext();
PresShell* presShell = presContext->PresShell();
presShell->FrameNeedsReflow(frame,
IntrinsicDirty::FrameAncestorsAndDescendants,
NS_FRAME_IS_DIRTY);
}
}
RefPtr<ImageContainer> imageContainer = GetImageContainer();
bool asyncInvalidate = imageContainer && imageContainer->IsAsync() &&
aForceInvalidate == ForceInvalidate::No;
if (frame) {
if (aImageSizeChanged == ImageSizeChanged::Yes) {
frame->InvalidateFrame();
} else {
frame->InvalidateLayer(DisplayItemType::TYPE_VIDEO, nullptr, nullptr,
asyncInvalidate ? nsIFrame::UPDATE_IS_ASYNC : 0);
}
}
SVGObserverUtils::InvalidateDirectRenderingObservers(this);
}
void HTMLMediaElement::UpdateMediaSize(const nsIntSize& aSize) {
MOZ_ASSERT(NS_IsMainThread());
if (IsVideo() && mReadyState != HAVE_NOTHING &&
mMediaInfo.mVideo.mDisplay != aSize) {
DispatchAsyncEvent(u"resize"_ns);
}
mMediaInfo.mVideo.mDisplay = aSize;
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
}
void HTMLMediaElement::SuspendOrResumeElement(bool aSuspendElement) {
LOG(LogLevel::Debug, ("%p SuspendOrResumeElement(suspend=%d) docHidden=%d",
this, aSuspendElement, OwnerDoc()->Hidden()));
if (aSuspendElement == mSuspendedByInactiveDocOrDocshell) {
return;
}
mSuspendedByInactiveDocOrDocshell = aSuspendElement;
UpdateSrcMediaStreamPlaying();
UpdateAudioChannelPlayingState();
if (aSuspendElement) {
if (mDecoder) {
mDecoder->Pause();
mDecoder->Suspend();
mDecoder->SetDelaySeekMode(true);
}
mEventBlocker->SetBlockEventDelivery(true);
// We won't want to resume media element from the bfcache.
ClearResumeDelayedMediaPlaybackAgentIfNeeded();
mMediaControlKeyListener->StopIfNeeded();
} else {
if (mDecoder) {
mDecoder->Resume();
if (!mPaused && !mDecoder->IsEnded()) {
mDecoder->Play();
}
mDecoder->SetDelaySeekMode(false);
}
mEventBlocker->SetBlockEventDelivery(false);
// If the media element has been blocked and isn't still allowed to play
// when it comes back from the bfcache, we would notify front end to show
// the blocking icon in order to inform user that the site is still being
// blocked.
if (mHasEverBeenBlockedForAutoplay && !AllowedToPlay()) {
MaybeNotifyAutoplayBlocked();
}
StartMediaControlKeyListenerIfNeeded();
}
if (StaticPrefs::media_testing_only_events()) {
auto dispatcher = MakeRefPtr<AsyncEventDispatcher>(
this, u"MozMediaSuspendChanged"_ns, CanBubble::eYes,
ChromeOnlyDispatch::eYes);
dispatcher->PostDOMEvent();
}
}
bool HTMLMediaElement::IsBeingDestroyed() {
nsIDocShell* docShell = OwnerDoc()->GetDocShell();
bool isBeingDestroyed = false;
if (docShell) {
docShell->IsBeingDestroyed(&isBeingDestroyed);
}
return isBeingDestroyed;
}
bool HTMLMediaElement::ShouldBeSuspendedByInactiveDocShell() const {
BrowsingContext* bc = OwnerDoc()->GetBrowsingContext();
return bc && !bc->IsActive() && bc->Top()->GetSuspendMediaWhenInactive();
}
void HTMLMediaElement::NotifyOwnerDocumentActivityChanged() {
if (mDecoder && !IsBeingDestroyed()) {
NotifyDecoderActivityChanges();
}
// We would suspend media when the document is inactive, or its docshell has
// been set to hidden and explicitly wants to suspend media. In those cases,
// the media would be not visible and we don't want them to continue playing.
bool shouldSuspend =
!OwnerDoc()->IsActive() || ShouldBeSuspendedByInactiveDocShell();
SuspendOrResumeElement(shouldSuspend);
// If the owning document has become inactive we should shutdown the CDM.
if (!OwnerDoc()->IsCurrentActiveDocument() && mMediaKeys) {
// We don't shutdown MediaKeys here because it also listens for document
// activity and will take care of shutting down itself.
DDUNLINKCHILD(mMediaKeys.get());
mMediaKeys = nullptr;
if (mDecoder) {
ShutdownDecoder();
}
}
AddRemoveSelfReference();
}
void HTMLMediaElement::NotifyFullScreenChanged() {
const bool isInFullScreen = IsInFullScreen();
if (isInFullScreen) {
StartMediaControlKeyListenerIfNeeded();
if (!mMediaControlKeyListener->IsStarted()) {
MEDIACONTROL_LOG("Failed to start the listener when entering fullscreen");
}
}
// Updating controller fullscreen state no matter the listener starts or not.
BrowsingContext* bc = OwnerDoc()->GetBrowsingContext();
if (RefPtr<IMediaInfoUpdater> updater = ContentMediaAgent::Get(bc)) {
updater->NotifyMediaFullScreenState(bc->Id(), isInFullScreen);
}
}
void HTMLMediaElement::AddRemoveSelfReference() {
// XXX we could release earlier here in many situations if we examined
// which event listeners are attached. Right now we assume there is a
// potential listener for every event. We would also have to keep the
// element alive if it was playing and producing audio output --- right now
// that's covered by the !mPaused check.
Document* ownerDoc = OwnerDoc();
// See the comment at the top of this file for the explanation of this
// boolean expression.
bool needSelfReference =
!mShuttingDown && ownerDoc->IsActive() &&
(mDelayingLoadEvent || (!mPaused && !Ended()) ||
(mDecoder && mDecoder->IsSeeking()) || IsEligibleForAutoplay() ||
(mMediaSource ? mProgressTimer : mNetworkState == NETWORK_LOADING));
if (needSelfReference != mHasSelfReference) {
mHasSelfReference = needSelfReference;
RefPtr<HTMLMediaElement> self = this;
if (needSelfReference) {
// The shutdown observer will hold a strong reference to us. This
// will do to keep us alive. We need to know about shutdown so that
// we can release our self-reference.
GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
"dom::HTMLMediaElement::AddSelfReference",
[self]() { self->mShutdownObserver->AddRefMediaElement(); }));
} else {
// Dispatch Release asynchronously so that we don't destroy this object
// inside a call stack of method calls on this object
GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
"dom::HTMLMediaElement::AddSelfReference",
[self]() { self->mShutdownObserver->ReleaseMediaElement(); }));
}
}
}
void HTMLMediaElement::NotifyShutdownEvent() {
mShuttingDown = true;
ResetState();
AddRemoveSelfReference();
}
void HTMLMediaElement::DispatchAsyncSourceError(nsIContent* aSourceElement) {
LOG_EVENT(LogLevel::Debug, ("%p Queuing simple source error event", this));
nsCOMPtr<nsIRunnable> event =
new nsSourceErrorEventRunner(this, aSourceElement);
GetMainThreadSerialEventTarget()->Dispatch(event.forget());
}
void HTMLMediaElement::NotifyAddedSource() {
// If a source element is inserted as a child of a media element
// that has no src attribute and whose networkState has the value
// NETWORK_EMPTY, the user agent must invoke the media element's
// resource selection algorithm.
if (!HasAttr(nsGkAtoms::src) && mNetworkState == NETWORK_EMPTY) {
AssertReadyStateIsNothing();
QueueSelectResourceTask();
}
// A load was paused in the resource selection algorithm, waiting for
// a new source child to be added, resume the resource selection algorithm.
if (mLoadWaitStatus == WAITING_FOR_SOURCE) {
// Rest the flag so we don't queue multiple LoadFromSourceTask() when
// multiple <source> are attached in an event loop.
mLoadWaitStatus = NOT_WAITING;
QueueLoadFromSourceTask();
}
}
HTMLSourceElement* HTMLMediaElement::GetNextSource() {
mSourceLoadCandidate = nullptr;
while (true) {
if (mSourcePointer == nsINode::GetLastChild()) {
return nullptr; // no more children
}
if (!mSourcePointer) {
mSourcePointer = nsINode::GetFirstChild();
} else {
mSourcePointer = mSourcePointer->GetNextSibling();
}
nsIContent* child = mSourcePointer;
// If child is a <source> element, it is the next candidate.
if (auto* source = HTMLSourceElement::FromNodeOrNull(child)) {
mSourceLoadCandidate = source;
return source;
}
}
MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
return nullptr;
}
void HTMLMediaElement::ChangeDelayLoadStatus(bool aDelay) {
if (mDelayingLoadEvent == aDelay) return;
mDelayingLoadEvent = aDelay;
LOG(LogLevel::Debug, ("%p ChangeDelayLoadStatus(%d) doc=0x%p", this, aDelay,
mLoadBlockedDoc.get()));
if (mDecoder) {
mDecoder->SetLoadInBackground(!aDelay);
}
if (aDelay) {
mLoadBlockedDoc = OwnerDoc();
mLoadBlockedDoc->BlockOnload();
} else {
// mLoadBlockedDoc might be null due to GC unlinking
if (mLoadBlockedDoc) {
mLoadBlockedDoc->UnblockOnload(false);
mLoadBlockedDoc = nullptr;
}
}
// We changed mDelayingLoadEvent which can affect AddRemoveSelfReference
AddRemoveSelfReference();
}
already_AddRefed<nsILoadGroup> HTMLMediaElement::GetDocumentLoadGroup() {
if (!OwnerDoc()->IsActive()) {
NS_WARNING("Load group requested for media element in inactive document.");
}
return OwnerDoc()->GetDocumentLoadGroup();
}
nsresult HTMLMediaElement::CopyInnerTo(Element* aDest) {
nsresult rv = nsGenericHTMLElement::CopyInnerTo(aDest);
NS_ENSURE_SUCCESS(rv, rv);
if (aDest->OwnerDoc()->IsStaticDocument()) {
HTMLMediaElement* dest = static_cast<HTMLMediaElement*>(aDest);
dest->SetMediaInfo(mMediaInfo);
}
return rv;
}
already_AddRefed<TimeRanges> HTMLMediaElement::Buffered() const {
media::TimeIntervals buffered =
mDecoder ? mDecoder->GetBuffered() : media::TimeIntervals();
RefPtr<TimeRanges> ranges = new TimeRanges(
ToSupports(OwnerDoc()), buffered.ToMicrosecondResolution());
return ranges.forget();
}
void HTMLMediaElement::SetRequestHeaders(nsIHttpChannel* aChannel) {
SetAcceptHeader(aChannel);
// Apache doesn't send Content-Length when gzip transfer encoding is used,
// which prevents us from estimating the video length (if explicit
// Content-Duration and a length spec in the container are not present either)
// and from seeking. So, disable the standard "Accept-Encoding: gzip,deflate"
DebugOnly<nsresult> rv =
aChannel->SetRequestHeader("Accept-Encoding"_ns, ""_ns, false);
MOZ_ASSERT(NS_SUCCEEDED(rv));
// Set the Referrer header
//
// FIXME: Shouldn't this use the Element constructor? Though I guess it
// doesn't matter as no HTMLMediaElement supports the referrerinfo attribute.
auto referrerInfo = MakeRefPtr<ReferrerInfo>(*OwnerDoc());
rv = aChannel->SetReferrerInfoWithoutClone(referrerInfo);
MOZ_ASSERT(NS_SUCCEEDED(rv));
}
const TimeStamp& HTMLMediaElement::LastTimeupdateDispatchTime() const {
MOZ_ASSERT(NS_IsMainThread());
return mLastTimeUpdateDispatchTime;
}
void HTMLMediaElement::UpdateLastTimeupdateDispatchTime() {
MOZ_ASSERT(NS_IsMainThread());
mLastTimeUpdateDispatchTime = TimeStamp::Now();
}
bool HTMLMediaElement::ShouldQueueTimeupdateAsyncTask(
TimeupdateType aType) const {
NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
// That means dispatching `timeupdate` is mandatorily required in the spec.
if (aType == TimeupdateType::eMandatory) {
return true;
}
// The timeupdate only occurs when the current playback position changes.
if (mLastCurrentTime == CurrentTime()) {
return false;
}
// Number of milliseconds between timeupdate events as defined by spec.
if (!mQueueTimeUpdateRunnerTime.IsNull() &&
TimeStamp::Now() - mQueueTimeUpdateRunnerTime <
TimeDuration::FromMilliseconds(TIMEUPDATE_MS)) {
return false;
}
return true;
}
void HTMLMediaElement::FireTimeUpdate(TimeupdateType aType) {
NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
if (ShouldQueueTimeupdateAsyncTask(aType)) {
RefPtr<nsMediaEventRunner> runner =
GetEventRunner(u"timeupdate"_ns, aType == TimeupdateType::eMandatory
? EventFlag::eMandatory
: EventFlag::eNone);
DispatchAsyncEvent(std::move(runner));
mQueueTimeUpdateRunnerTime = TimeStamp::Now();
mLastCurrentTime = CurrentTime();
}
if (mFragmentEnd >= 0.0 && CurrentTime() >= mFragmentEnd) {
Pause();
mFragmentEnd = -1.0;
mFragmentStart = -1.0;
mDecoder->SetFragmentEndTime(mFragmentEnd);
}
// Update the cues displaying on the video.
// Here mTextTrackManager can be null if the cycle collector has unlinked
// us before our parent. In that case UnbindFromTree will call us
// when our parent is unlinked.
if (mTextTrackManager) {
mTextTrackManager->TimeMarchesOn();
}
}
MediaError* HTMLMediaElement::GetError() const { return mErrorSink->mError; }
void HTMLMediaElement::GetCurrentSpec(nsCString& aString) {
// If playing a regular URL, an ObjectURL of a Blob/File, return that.
if (mLoadingSrc) {
mLoadingSrc->GetSpec(aString);
} else if (mSrcMediaSource) {
// If playing an ObjectURL, and it's a MediaSource, return the value of the
// `src` attribute.
nsAutoString src;
GetSrc(src);
CopyUTF16toUTF8(src, aString);
} else {
// Playing e.g. a MediaStream via an object URL - return an empty string
aString.Truncate();
}
}
double HTMLMediaElement::MozFragmentEnd() {
double duration = Duration();
// If there is no end fragment, or the fragment end is greater than the
// duration, return the duration.
return (mFragmentEnd < 0.0 || mFragmentEnd > duration) ? duration
: mFragmentEnd;
}
void HTMLMediaElement::SetDefaultPlaybackRate(double aDefaultPlaybackRate,
ErrorResult& aRv) {
if (mSrcAttrStream) {
return;
}
if (aDefaultPlaybackRate < 0) {
aRv.Throw(NS_ERROR_NOT_IMPLEMENTED);
return;
}
double defaultPlaybackRate = ClampPlaybackRate(aDefaultPlaybackRate);
if (mDefaultPlaybackRate == defaultPlaybackRate) {
return;
}
mDefaultPlaybackRate = defaultPlaybackRate;
DispatchAsyncEvent(u"ratechange"_ns);
}
void HTMLMediaElement::SetPlaybackRate(double aPlaybackRate, ErrorResult& aRv) {
if (mSrcAttrStream) {
return;
}
// Changing the playback rate of a media that has more than two channels is
// not supported.
if (aPlaybackRate < 0) {
aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
return;
}
if (mPlaybackRate == aPlaybackRate) {
return;
}
mPlaybackRate = aPlaybackRate;
// Playback rate threshold above which audio is muted.
uint32_t threshold = StaticPrefs::media_audio_playbackrate_muting_threshold();
if (mPlaybackRate != 0.0 &&
(mPlaybackRate > threshold || mPlaybackRate < 1. / threshold)) {
SetMutedInternal(mMuted | MUTED_BY_INVALID_PLAYBACK_RATE);
} else {
SetMutedInternal(mMuted & ~MUTED_BY_INVALID_PLAYBACK_RATE);
}
if (mDecoder) {
mDecoder->SetPlaybackRate(ClampPlaybackRate(mPlaybackRate));
}
DispatchAsyncEvent(u"ratechange"_ns);
}
void HTMLMediaElement::SetPreservesPitch(bool aPreservesPitch) {
mPreservesPitch = aPreservesPitch;
if (mDecoder) {
mDecoder->SetPreservesPitch(mPreservesPitch);
}
}
ImageContainer* HTMLMediaElement::GetImageContainer() {
VideoFrameContainer* container = GetVideoFrameContainer();
return container ? container->GetImageContainer() : nullptr;
}
void HTMLMediaElement::UpdateAudioChannelPlayingState() {
if (mAudioChannelWrapper) {
mAudioChannelWrapper->UpdateAudioChannelPlayingState();
}
}
static const char* VisibilityString(Visibility aVisibility) {
switch (aVisibility) {
case Visibility::Untracked: {
return "Untracked";
}
case Visibility::ApproximatelyNonVisible: {
return "ApproximatelyNonVisible";
}
case Visibility::ApproximatelyVisible: {
return "ApproximatelyVisible";
}
}
return "NAN";
}
void HTMLMediaElement::OnVisibilityChange(Visibility aNewVisibility) {
LOG(LogLevel::Debug,
("OnVisibilityChange(): %s\n", VisibilityString(aNewVisibility)));
mVisibilityState = aNewVisibility;
if (StaticPrefs::media_test_video_suspend()) {
DispatchAsyncEvent(u"visibilitychanged"_ns);
}
if (!mDecoder) {
return;
}
NotifyDecoderActivityChanges();
}
MediaKeys* HTMLMediaElement::GetMediaKeys() const { return mMediaKeys; }
bool HTMLMediaElement::ContainsRestrictedContent() const {
return GetMediaKeys() != nullptr;
}
void HTMLMediaElement::SetCDMProxyFailure(const MediaResult& aResult) {
LOG(LogLevel::Debug, ("%s", __func__));
MOZ_ASSERT(mSetMediaKeysDOMPromise);
ResetSetMediaKeysTempVariables();
mSetMediaKeysDOMPromise->MaybeReject(aResult.Code(), aResult.Message());
}
void HTMLMediaElement::RemoveMediaKeys() {
LOG(LogLevel::Debug, ("%s", __func__));
// 5.2.3 Stop using the CDM instance represented by the mediaKeys attribute
// to decrypt media data and remove the association with the media element.
if (mMediaKeys) {
mMediaKeys->Unbind();
}
mMediaKeys = nullptr;
}
bool HTMLMediaElement::TryRemoveMediaKeysAssociation() {
MOZ_ASSERT(mMediaKeys);
LOG(LogLevel::Debug, ("%s", __func__));
// 5.2.1 If the user agent or CDM do not support removing the association,
// let this object's attaching media keys value be false and reject promise
// with a new DOMException whose name is NotSupportedError.
// 5.2.2 If the association cannot currently be removed, let this object's
// attaching media keys value be false and reject promise with a new
// DOMException whose name is InvalidStateError.
if (mDecoder) {
RefPtr<HTMLMediaElement> self = this;
mDecoder->SetCDMProxy(nullptr)
->Then(
AbstractMainThread(), __func__,
[self]() {
self->mSetCDMRequest.Complete();
self->RemoveMediaKeys();
if (self->AttachNewMediaKeys()) {
// No incoming MediaKeys object or MediaDecoder is not
// created yet.
self->MakeAssociationWithCDMResolved();
}
},
[self](const MediaResult& aResult) {
self->mSetCDMRequest.Complete();
// 5.2.4 If the preceding step failed, let this object's
// attaching media keys value be false and reject promise with
// a new DOMException whose name is the appropriate error name.
self->SetCDMProxyFailure(aResult);
})
->Track(mSetCDMRequest);
return false;
}
RemoveMediaKeys();
return true;
}
bool HTMLMediaElement::DetachExistingMediaKeys() {
LOG(LogLevel::Debug, ("%s", __func__));
MOZ_ASSERT(mSetMediaKeysDOMPromise);
// 5.1 If mediaKeys is not null, CDM instance represented by mediaKeys is
// already in use by another media element, and the user agent is unable
// to use it with this element, let this object's attaching media keys
// value be false and reject promise with a new DOMException whose name
// is QuotaExceededError.
if (mIncomingMediaKeys && mIncomingMediaKeys->IsBoundToMediaElement()) {
SetCDMProxyFailure(MediaResult(
NS_ERROR_DOM_MEDIA_KEY_QUOTA_EXCEEDED_ERR,
"MediaKeys object is already bound to another HTMLMediaElement"));
return false;
}
// 5.2 If the mediaKeys attribute is not null, run the following steps:
if (mMediaKeys) {
return TryRemoveMediaKeysAssociation();
}
return true;
}
void HTMLMediaElement::MakeAssociationWithCDMResolved() {
LOG(LogLevel::Debug, ("%s", __func__));
MOZ_ASSERT(mSetMediaKeysDOMPromise);
// 5.4 Set the mediaKeys attribute to mediaKeys.
mMediaKeys = mIncomingMediaKeys;
#ifdef MOZ_WMF_CDM
if (mMediaKeys && mMediaKeys->GetCDMProxy()) {
mIsUsingWMFCDM = !!mMediaKeys->GetCDMProxy()->AsWMFCDMProxy();
}
#endif
// 5.5 Let this object's attaching media keys value be false.
ResetSetMediaKeysTempVariables();
// 5.6 Resolve promise.
mSetMediaKeysDOMPromise->MaybeResolveWithUndefined();
mSetMediaKeysDOMPromise = nullptr;
}
bool HTMLMediaElement::TryMakeAssociationWithCDM(CDMProxy* aProxy) {
LOG(LogLevel::Debug, ("%s", __func__));
MOZ_ASSERT(aProxy);
// 5.3.3 Queue a task to run the "Attempt to Resume Playback If Necessary"
// algorithm on the media element.
// Note: Setting the CDMProxy on the MediaDecoder will unblock playback.
if (mDecoder) {
// CDMProxy is set asynchronously in MediaFormatReader, once it's done,
// HTMLMediaElement should resolve or reject the DOM promise.
RefPtr<HTMLMediaElement> self = this;
mDecoder->SetCDMProxy(aProxy)
->Then(
AbstractMainThread(), __func__,
[self]() {
self->mSetCDMRequest.Complete();
self->MakeAssociationWithCDMResolved();
},
[self](const MediaResult& aResult) {
self->mSetCDMRequest.Complete();
self->SetCDMProxyFailure(aResult);
})
->Track(mSetCDMRequest);
return false;
}
return true;
}
bool HTMLMediaElement::AttachNewMediaKeys() {
LOG(LogLevel::Debug,
("%s incoming MediaKeys(%p)", __func__, mIncomingMediaKeys.get()));
MOZ_ASSERT(mSetMediaKeysDOMPromise);
// 5.3. If mediaKeys is not null, run the following steps:
if (mIncomingMediaKeys) {
auto* cdmProxy = mIncomingMediaKeys->GetCDMProxy();
if (!cdmProxy) {
SetCDMProxyFailure(MediaResult(
NS_ERROR_DOM_INVALID_STATE_ERR,
"CDM crashed before binding MediaKeys object to HTMLMediaElement"));
return false;
}
// 5.3.1 Associate the CDM instance represented by mediaKeys with the
// media element for decrypting media data.
if (NS_FAILED(mIncomingMediaKeys->Bind(this))) {
// 5.3.2 If the preceding step failed, run the following steps:
// 5.3.2.1 Set the mediaKeys attribute to null.
mMediaKeys = nullptr;
// 5.3.2.2 Let this object's attaching media keys value be false.
// 5.3.2.3 Reject promise with a new DOMException whose name is
// the appropriate error name.
SetCDMProxyFailure(
MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
"Failed to bind MediaKeys object to HTMLMediaElement"));
return false;
}
return TryMakeAssociationWithCDM(cdmProxy);
}
return true;
}
void HTMLMediaElement::ResetSetMediaKeysTempVariables() {
mAttachingMediaKey = false;
mIncomingMediaKeys = nullptr;
}
already_AddRefed<Promise> HTMLMediaElement::SetMediaKeys(
mozilla::dom::MediaKeys* aMediaKeys, ErrorResult& aRv) {
LOG(LogLevel::Debug, ("%p SetMediaKeys(%p) mMediaKeys=%p mDecoder=%p", this,
aMediaKeys, mMediaKeys.get(), mDecoder.get()));
if (MozAudioCaptured()) {
aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
return nullptr;
}
nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
if (!win) {
aRv.Throw(NS_ERROR_UNEXPECTED);
return nullptr;
}
RefPtr<DetailedPromise> promise = DetailedPromise::Create(
win->AsGlobal(), aRv, "HTMLMediaElement.setMediaKeys"_ns);
if (aRv.Failed()) {
return nullptr;
}
// 1. If mediaKeys and the mediaKeys attribute are the same object,
// return a resolved promise.
if (mMediaKeys == aMediaKeys) {
promise->MaybeResolveWithUndefined();
return promise.forget();
}
// 2. If this object's attaching media keys value is true, return a
// promise rejected with a new DOMException whose name is InvalidStateError.
if (mAttachingMediaKey) {
promise->MaybeRejectWithInvalidStateError(
"A MediaKeys object is in attaching operation.");
return promise.forget();
}
// 3. Let this object's attaching media keys value be true.
mAttachingMediaKey = true;
mIncomingMediaKeys = aMediaKeys;
// 4. Let promise be a new promise.
mSetMediaKeysDOMPromise = promise;
// 5. Run the following steps in parallel:
// 5.1 & 5.2 & 5.3
if (!DetachExistingMediaKeys() || !AttachNewMediaKeys()) {
return promise.forget();
}
// 5.4, 5.5, 5.6
MakeAssociationWithCDMResolved();
// 6. Return promise.
return promise.forget();
}
EventHandlerNonNull* HTMLMediaElement::GetOnencrypted() {
return EventTarget::GetEventHandler(nsGkAtoms::onencrypted);
}
void HTMLMediaElement::SetOnencrypted(EventHandlerNonNull* aCallback) {
EventTarget::SetEventHandler(nsGkAtoms::onencrypted, aCallback);
}
EventHandlerNonNull* HTMLMediaElement::GetOnwaitingforkey() {
return EventTarget::GetEventHandler(nsGkAtoms::onwaitingforkey);
}
void HTMLMediaElement::SetOnwaitingforkey(EventHandlerNonNull* aCallback) {
EventTarget::SetEventHandler(nsGkAtoms::onwaitingforkey, aCallback);
}
void HTMLMediaElement::DispatchEncrypted(const nsTArray<uint8_t>& aInitData,
const nsAString& aInitDataType) {
LOG(LogLevel::Debug, ("%p DispatchEncrypted initDataType='%s'", this,
NS_ConvertUTF16toUTF8(aInitDataType).get()));
if (mReadyState == HAVE_NOTHING) {
// Ready state not HAVE_METADATA (yet), don't dispatch encrypted now.
// Queueing for later dispatch in MetadataLoaded.
mPendingEncryptedInitData.AddInitData(aInitDataType, aInitData);
return;
}
RefPtr<MediaEncryptedEvent> event;
if (IsCORSSameOrigin()) {
event = MediaEncryptedEvent::Constructor(this, aInitDataType, aInitData);
} else {
event = MediaEncryptedEvent::Constructor(this);
}
RefPtr<AsyncEventDispatcher> asyncDispatcher =
new AsyncEventDispatcher(this, event.forget());
asyncDispatcher->PostDOMEvent();
}
bool HTMLMediaElement::IsEventAttributeNameInternal(nsAtom* aName) {
return aName == nsGkAtoms::onencrypted ||
nsGenericHTMLElement::IsEventAttributeNameInternal(aName);
}
void HTMLMediaElement::NotifyWaitingForKey() {
LOG(LogLevel::Debug, ("%p, NotifyWaitingForKey()", this));
// 7.3.4 Queue a "waitingforkey" Event
// 1. Let the media element be the specified HTMLMediaElement object.
// 2. If the media element's waiting for key value is true, abort these steps.
if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
// 3. Set the media element's waiting for key value to true.
// Note: algorithm continues in UpdateReadyStateInternal() when all decoded
// data enqueued in the MDSM is consumed.
mWaitingForKey = WAITING_FOR_KEY;
// mWaitingForKey changed outside of UpdateReadyStateInternal. This may
// affect mReadyState.
mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
}
}
AudioTrackList* HTMLMediaElement::AudioTracks() { return mAudioTrackList; }
VideoTrackList* HTMLMediaElement::VideoTracks() { return mVideoTrackList; }
TextTrackList* HTMLMediaElement::GetTextTracks() {
return GetOrCreateTextTrackManager()->GetTextTracks();
}
already_AddRefed<TextTrack> HTMLMediaElement::AddTextTrack(
TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage) {
return GetOrCreateTextTrackManager()->AddTextTrack(
aKind, aLabel, aLanguage, TextTrackMode::Hidden,
TextTrackReadyState::Loaded, TextTrackSource::AddTextTrack);
}
void HTMLMediaElement::PopulatePendingTextTrackList() {
if (mTextTrackManager) {
mTextTrackManager->PopulatePendingList();
}
}
TextTrackManager* HTMLMediaElement::GetOrCreateTextTrackManager() {
if (!mTextTrackManager) {
mTextTrackManager = new TextTrackManager(this);
mTextTrackManager->AddListeners();
}
return mTextTrackManager;
}
MediaDecoderOwner::NextFrameStatus HTMLMediaElement::NextFrameStatus() {
if (mDecoder) {
return mDecoder->NextFrameStatus();
}
if (mSrcStream) {
AutoTArray<RefPtr<MediaTrack>, 4> tracks;
GetAllEnabledMediaTracks(tracks);
if (!tracks.IsEmpty() && !mSrcStreamPlaybackEnded) {
return NEXT_FRAME_AVAILABLE;
}
return NEXT_FRAME_UNAVAILABLE;
}
return NEXT_FRAME_UNINITIALIZED;
}
void HTMLMediaElement::SetDecoder(MediaDecoder* aDecoder) {
MOZ_ASSERT(aDecoder); // Use ShutdownDecoder() to clear.
if (mDecoder) {
ShutdownDecoder();
}
mDecoder = aDecoder;
DDLINKCHILD("decoder", mDecoder.get());
if (mDecoder && mForcedHidden) {
mDecoder->SetForcedHidden(mForcedHidden);
}
}
float HTMLMediaElement::ComputedVolume() const {
return mMuted ? 0.0f
: mAudioChannelWrapper ? mAudioChannelWrapper->GetEffectiveVolume()
: static_cast<float>(mVolume);
}
bool HTMLMediaElement::ComputedMuted() const {
return (mMuted & MUTED_BY_AUDIO_CHANNEL);
}
bool HTMLMediaElement::IsSuspendedByInactiveDocOrDocShell() const {
return mSuspendedByInactiveDocOrDocshell;
}
bool HTMLMediaElement::IsCurrentlyPlaying() const {
// We have playable data, but we still need to check whether data is "real"
// current data.
return mReadyState >= HAVE_CURRENT_DATA && !IsPlaybackEnded();
}
void HTMLMediaElement::SetAudibleState(bool aAudible) {
if (mIsAudioTrackAudible != aAudible) {
mIsAudioTrackAudible = aAudible;
NotifyAudioPlaybackChanged(
AudioChannelService::AudibleChangedReasons::eDataAudibleChanged);
}
}
void HTMLMediaElement::NotifyAudioPlaybackChanged(
AudibleChangedReasons aReason) {
if (mAudioChannelWrapper) {
mAudioChannelWrapper->NotifyAudioPlaybackChanged(aReason);
}
// We would start the listener after media becomes audible.
const bool isAudible = IsAudible();
if (isAudible && !mMediaControlKeyListener->IsStarted()) {
StartMediaControlKeyListenerIfNeeded();
}
mMediaControlKeyListener->UpdateMediaAudibleState(isAudible);
// only request wake lock for audible media.
UpdateWakeLock();
}
void HTMLMediaElement::SetMediaInfo(const MediaInfo& aInfo) {
const bool oldHasAudio = mMediaInfo.HasAudio();
mMediaInfo = aInfo;
if ((aInfo.HasAudio() != oldHasAudio) && mResumeDelayedPlaybackAgent) {
mResumeDelayedPlaybackAgent->UpdateAudibleState(this, IsAudible());
}
nsILoadContext* loadContext = OwnerDoc()->GetLoadContext();
if (HasAudio() && loadContext && !loadContext->UsePrivateBrowsing()) {
mTitleChangeObserver->Subscribe();
UpdateStreamName();
} else {
mTitleChangeObserver->Unsubscribe();
}
if (mAudioChannelWrapper) {
mAudioChannelWrapper->AudioCaptureTrackChangeIfNeeded();
}
UpdateWakeLock();
}
MediaInfo HTMLMediaElement::GetMediaInfo() const { return mMediaInfo; }
FrameStatistics* HTMLMediaElement::GetFrameStatistics() const {
return mDecoder ? &(mDecoder->GetFrameStatistics()) : nullptr;
}
void HTMLMediaElement::DispatchAsyncTestingEvent(const nsAString& aName) {
if (!StaticPrefs::media_testing_only_events()) {
return;
}
DispatchAsyncEvent(aName);
}
void HTMLMediaElement::AudioCaptureTrackChange(bool aCapture) {
// No need to capture a silent media element.
if (!HasAudio()) {
return;
}
if (aCapture && !mStreamWindowCapturer) {
nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
if (!window) {
return;
}
MediaTrackGraph* mtg = MediaTrackGraph::GetInstance(
MediaTrackGraph::AUDIO_THREAD_DRIVER, window,
MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
RefPtr<DOMMediaStream> stream =
CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
StreamCaptureType::CAPTURE_AUDIO, mtg);
mStreamWindowCapturer =
new MediaStreamWindowCapturer(stream, window->WindowID());
mStreamWindowCapturer->mStream->RegisterTrackListener(
mStreamWindowCapturer);
} else if (!aCapture && mStreamWindowCapturer) {
for (size_t i = 0; i < mOutputStreams.Length(); i++) {
if (mOutputStreams[i].mStream == mStreamWindowCapturer->mStream) {
// We own this MediaStream, it is not exposed to JS.
AutoTArray<RefPtr<MediaStreamTrack>, 2> tracks;
mStreamWindowCapturer->mStream->GetTracks(tracks);
for (auto& track : tracks) {
track->Stop();
}
mOutputStreams.RemoveElementAt(i);
break;
}
}
mStreamWindowCapturer->mStream->UnregisterTrackListener(
mStreamWindowCapturer);
mStreamWindowCapturer = nullptr;
if (mOutputStreams.IsEmpty()) {
mTracksCaptured = nullptr;
}
}
}
void HTMLMediaElement::NotifyCueDisplayStatesChanged() {
if (!mTextTrackManager) {
return;
}
mTextTrackManager->DispatchUpdateCueDisplay();
}
void HTMLMediaElement::LogVisibility(CallerAPI aAPI) {
const bool isVisible = mVisibilityState == Visibility::ApproximatelyVisible;
LOG(LogLevel::Debug, ("%p visibility = %u, API: '%d' and 'All'", this,
isVisible, static_cast<int>(aAPI)));
if (!isVisible) {
LOG(LogLevel::Debug, ("%p inTree = %u, API: '%d' and 'All'", this,
IsInComposedDoc(), static_cast<int>(aAPI)));
}
}
void HTMLMediaElement::UpdateCustomPolicyAfterPlayed() {
if (mAudioChannelWrapper) {
mAudioChannelWrapper->NotifyPlayStateChanged();
}
}
AbstractThread* HTMLMediaElement::AbstractMainThread() const {
return AbstractThread::MainThread();
}
nsTArray<RefPtr<PlayPromise>> HTMLMediaElement::TakePendingPlayPromises() {
return std::move(mPendingPlayPromises);
}
void HTMLMediaElement::NotifyAboutPlaying() {
// Stick to the DispatchAsyncEvent() call path for now because we want to
// trigger some telemetry-related codes in the DispatchAsyncEvent() method.
DispatchAsyncEvent(u"playing"_ns);
}
already_AddRefed<PlayPromise> HTMLMediaElement::CreatePlayPromise(
ErrorResult& aRv) const {
nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
if (!win) {
aRv.Throw(NS_ERROR_UNEXPECTED);
return nullptr;
}
RefPtr<PlayPromise> promise = PlayPromise::Create(win->AsGlobal(), aRv);
LOG(LogLevel::Debug, ("%p created PlayPromise %p", this, promise.get()));
return promise.forget();
}
already_AddRefed<Promise> HTMLMediaElement::CreateDOMPromise(
ErrorResult& aRv) const {
nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
if (!win) {
aRv.Throw(NS_ERROR_UNEXPECTED);
return nullptr;
}
return Promise::Create(win->AsGlobal(), aRv);
}
void HTMLMediaElement::AsyncResolvePendingPlayPromises() {
if (mShuttingDown) {
return;
}
nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
this, TakePendingPlayPromises());
GetMainThreadSerialEventTarget()->Dispatch(event.forget());
}
void HTMLMediaElement::AsyncRejectPendingPlayPromises(nsresult aError) {
if (!mPaused) {
mPaused = true;
DispatchAsyncEvent(u"pause"_ns);
}
if (mShuttingDown) {
return;
}
if (aError == NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR) {
DispatchEventsWhenPlayWasNotAllowed();
}
nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
this, TakePendingPlayPromises(), aError);
GetMainThreadSerialEventTarget()->Dispatch(event.forget());
}
void HTMLMediaElement::GetEMEInfo(dom::EMEDebugInfo& aInfo) {
MOZ_ASSERT(NS_IsMainThread(),
"MediaKeys expects to be interacted with on main thread!");
if (!mMediaKeys) {
return;
}
mMediaKeys->GetKeySystem(aInfo.mKeySystem);
mMediaKeys->GetSessionsInfo(aInfo.mSessionsInfo);
}
void HTMLMediaElement::NotifyDecoderActivityChanges() const {
if (mDecoder) {
mDecoder->NotifyOwnerActivityChanged(IsActuallyInvisible(),
IsInComposedDoc());
}
}
Document* HTMLMediaElement::GetDocument() const { return OwnerDoc(); }
bool HTMLMediaElement::IsAudible() const {
// No audio track.
if (!HasAudio()) {
return false;
}
// Muted or the volume should not be ~0
if (mMuted || (std::fabs(Volume()) <= 1e-7)) {
return false;
}
return mIsAudioTrackAudible;
}
Maybe<nsAutoString> HTMLMediaElement::GetKeySystem() const {
if (!mMediaKeys) {
return Nothing();
}
nsAutoString keySystem;
mMediaKeys->GetKeySystem(keySystem);
return Some(keySystem);
}
void HTMLMediaElement::ConstructMediaTracks(const MediaInfo* aInfo) {
if (!aInfo) {
return;
}
AudioTrackList* audioList = AudioTracks();
if (audioList && aInfo->HasAudio()) {
const TrackInfo& info = aInfo->mAudio;
RefPtr<AudioTrack> track = MediaTrackList::CreateAudioTrack(
audioList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
info.mLanguage, info.mEnabled);
audioList->AddTrack(track);
}
VideoTrackList* videoList = VideoTracks();
if (videoList && aInfo->HasVideo()) {
const TrackInfo& info = aInfo->mVideo;
RefPtr<VideoTrack> track = MediaTrackList::CreateVideoTrack(
videoList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
info.mLanguage);
videoList->AddTrack(track);
track->SetEnabledInternal(info.mEnabled, MediaTrack::FIRE_NO_EVENTS);
}
}
void HTMLMediaElement::RemoveMediaTracks() {
if (mAudioTrackList) {
mAudioTrackList->RemoveTracks();
}
if (mVideoTrackList) {
mVideoTrackList->RemoveTracks();
}
}
class MediaElementGMPCrashHelper : public GMPCrashHelper {
public:
explicit MediaElementGMPCrashHelper(HTMLMediaElement* aElement)
: mElement(aElement) {
MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
}
already_AddRefed<nsPIDOMWindowInner> GetPluginCrashedEventTarget() override {
MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
if (!mElement) {
return nullptr;
}
return do_AddRef(mElement->OwnerDoc()->GetInnerWindow());
}
private:
WeakPtr<HTMLMediaElement> mElement;
};
already_AddRefed<GMPCrashHelper> HTMLMediaElement::CreateGMPCrashHelper() {
return MakeAndAddRef<MediaElementGMPCrashHelper>(this);
}
void HTMLMediaElement::MarkAsTainted() {
mHasSuspendTaint = true;
if (mDecoder) {
mDecoder->SetSuspendTaint(true);
}
}
bool HasDebuggerOrTabsPrivilege(JSContext* aCx, JSObject* aObj) {
return nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::debugger) ||
nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::tabs);
}
already_AddRefed<Promise> HTMLMediaElement::SetSinkId(const nsAString& aSinkId,
ErrorResult& aRv) {
LOG(LogLevel::Info,
("%p, setSinkId(%s)", this, NS_ConvertUTF16toUTF8(aSinkId).get()));
nsCOMPtr<nsPIDOMWindowInner> win = OwnerDoc()->GetInnerWindow();
if (!win) {
aRv.Throw(NS_ERROR_UNEXPECTED);
return nullptr;
}
RefPtr<Promise> promise = Promise::Create(win->AsGlobal(), aRv);
if (aRv.Failed()) {
return nullptr;
}
if (!FeaturePolicyUtils::IsFeatureAllowed(win->GetExtantDoc(),
u"speaker-selection"_ns)) {
promise->MaybeRejectWithNotAllowedError(
"Document's Permissions Policy does not allow setSinkId()");
}
if (mSink.first.Equals(aSinkId)) {
promise->MaybeResolveWithUndefined();
return promise.forget();
}
RefPtr<MediaDevices> mediaDevices = win->Navigator()->GetMediaDevices(aRv);
if (aRv.Failed()) {
return nullptr;
}
nsString sinkId(aSinkId);
mediaDevices->GetSinkDevice(sinkId)
->Then(
AbstractMainThread(), __func__,
[self = RefPtr<HTMLMediaElement>(this),
this](RefPtr<AudioDeviceInfo>&& aInfo) {
// Sink found switch output device.
MOZ_ASSERT(aInfo);
if (mDecoder) {
RefPtr<SinkInfoPromise> p = mDecoder->SetSink(aInfo)->Then(
AbstractMainThread(), __func__,
[aInfo](const GenericPromise::ResolveOrRejectValue& aValue) {
if (aValue.IsResolve()) {
return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
}
return SinkInfoPromise::CreateAndReject(
aValue.RejectValue(), __func__);
});
return p;
}
if (mSrcStream) {
MOZ_ASSERT(mMediaStreamRenderer);
RefPtr<SinkInfoPromise> p =
mMediaStreamRenderer->SetAudioOutputDevice(aInfo)->Then(
AbstractMainThread(), __func__,
[aInfo](
const GenericPromise::ResolveOrRejectValue& aValue) {
if (aValue.IsResolve()) {
return SinkInfoPromise::CreateAndResolve(aInfo,
__func__);
}
return SinkInfoPromise::CreateAndReject(
aValue.RejectValue(), __func__);
});
return p;
}
// No media attached to the element save it for later.
return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
},
[](nsresult res) {
// Promise is rejected, sink not found.
return SinkInfoPromise::CreateAndReject(res, __func__);
})
->Then(AbstractMainThread(), __func__,
[promise, self = RefPtr<HTMLMediaElement>(this), this,
sinkId](const SinkInfoPromise::ResolveOrRejectValue& aValue) {
if (aValue.IsResolve()) {
LOG(LogLevel::Info, ("%p, set sinkid=%s", this,
NS_ConvertUTF16toUTF8(sinkId).get()));
mSink = std::pair(sinkId, aValue.ResolveValue());
promise->MaybeResolveWithUndefined();
} else {
switch (aValue.RejectValue()) {
case NS_ERROR_ABORT:
promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
break;
case NS_ERROR_NOT_AVAILABLE: {
promise->MaybeRejectWithNotFoundError(
"The object can not be found here.");
break;
}
default:
MOZ_ASSERT_UNREACHABLE("Invalid error.");
}
}
});
aRv = NS_OK;
return promise.forget();
}
void HTMLMediaElement::NotifyTextTrackModeChanged() {
if (mPendingTextTrackChanged) {
return;
}
mPendingTextTrackChanged = true;
AbstractMainThread()->Dispatch(
NS_NewRunnableFunction("HTMLMediaElement::NotifyTextTrackModeChanged",
[this, self = RefPtr<HTMLMediaElement>(this)]() {
mPendingTextTrackChanged = false;
if (!mTextTrackManager) {
return;
}
GetTextTracks()->CreateAndDispatchChangeEvent();
if (!mShowPoster) {
mTextTrackManager->TimeMarchesOn();
}
}));
}
void HTMLMediaElement::CreateResumeDelayedMediaPlaybackAgentIfNeeded() {
if (mResumeDelayedPlaybackAgent) {
return;
}
mResumeDelayedPlaybackAgent =
MediaPlaybackDelayPolicy::CreateResumeDelayedPlaybackAgent(this,
IsAudible());
if (!mResumeDelayedPlaybackAgent) {
LOG(LogLevel::Debug,
("%p Failed to create a delayed playback agant", this));
return;
}
mResumeDelayedPlaybackAgent->GetResumePromise()
->Then(
AbstractMainThread(), __func__,
[self = RefPtr<HTMLMediaElement>(this)]() {
LOG(LogLevel::Debug, ("%p Resume delayed Play() call", self.get()));
self->mResumePlaybackRequest.Complete();
self->mResumeDelayedPlaybackAgent = nullptr;
IgnoredErrorResult dummy;
RefPtr<Promise> toBeIgnored = self->Play(dummy);
},
[self = RefPtr<HTMLMediaElement>(this)]() {
LOG(LogLevel::Debug,
("%p Can not resume delayed Play() call", self.get()));
self->mResumePlaybackRequest.Complete();
self->mResumeDelayedPlaybackAgent = nullptr;
})
->Track(mResumePlaybackRequest);
}
void HTMLMediaElement::ClearResumeDelayedMediaPlaybackAgentIfNeeded() {
if (mResumeDelayedPlaybackAgent) {
mResumePlaybackRequest.DisconnectIfExists();
mResumeDelayedPlaybackAgent = nullptr;
}
}
void HTMLMediaElement::NotifyMediaControlPlaybackStateChanged() {
if (!mMediaControlKeyListener->IsStarted()) {
return;
}
if (mPaused) {
mMediaControlKeyListener->NotifyMediaStoppedPlaying();
} else {
mMediaControlKeyListener->NotifyMediaStartedPlaying();
}
}
bool HTMLMediaElement::IsInFullScreen() const {
return State().HasState(ElementState::FULLSCREEN);
}
bool HTMLMediaElement::IsPlayable() const {
return (mDecoder || mSrcStream) && !HasError();
}
bool HTMLMediaElement::ShouldStartMediaControlKeyListener() const {
if (!IsPlayable()) {
MEDIACONTROL_LOG("Not start listener because media is not playable");
return false;
}
if (mSrcStream) {
MEDIACONTROL_LOG("Not listening because media is real-time");
return false;
}
if (IsBeingUsedInPictureInPictureMode()) {
MEDIACONTROL_LOG("Start listener because of being used in PiP mode");
return true;
}
if (IsInFullScreen()) {
MEDIACONTROL_LOG("Start listener because of being used in fullscreen");
return true;
}
// In order to filter out notification-ish sound, we use this pref to set the
// eligible media duration to prevent showing media control for those short
// sound.
if (Duration() <
StaticPrefs::media_mediacontrol_eligible_media_duration_s()) {
MEDIACONTROL_LOG("Not listening because media's duration %f is too short.",
Duration());
return false;
}
// This includes cases such like `video is muted`, `video has zero volume`,
// `video's audio track is still inaudible` and `tab is muted by audio channel
// (tab sound indicator)`, all these cases would make media inaudible.
// `ComputedVolume()` would return the final volume applied the affection made
// by audio channel, which is used to detect if the tab is muted by audio
// channel.
if (!IsAudible() || ComputedVolume() == 0.0f) {
MEDIACONTROL_LOG("Not listening because media is inaudible");
return false;
}
return true;
}
void HTMLMediaElement::StartMediaControlKeyListenerIfNeeded() {
if (!ShouldStartMediaControlKeyListener()) {
return;
}
mMediaControlKeyListener->Start();
}
void HTMLMediaElement::UpdateStreamName() {
MOZ_ASSERT(NS_IsMainThread());
nsAutoString aTitle;
OwnerDoc()->GetTitle(aTitle);
if (mDecoder) {
mDecoder->SetStreamName(aTitle);
}
}
void HTMLMediaElement::SetSecondaryMediaStreamRenderer(
VideoFrameContainer* aContainer,
FirstFrameVideoOutput* aFirstFrameOutput /* = nullptr */) {
MOZ_ASSERT(mSrcStream);
MOZ_ASSERT(mMediaStreamRenderer);
if (mSecondaryMediaStreamRenderer) {
mSecondaryMediaStreamRenderer->Shutdown();
mSecondaryMediaStreamRenderer = nullptr;
}
if (aContainer) {
mSecondaryMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
AbstractMainThread(), aContainer, aFirstFrameOutput, this);
if (mSrcStreamIsPlaying) {
mSecondaryMediaStreamRenderer->Start();
}
if (mSelectedVideoStreamTrack) {
mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
}
}
}
void HTMLMediaElement::UpdateMediaControlAfterPictureInPictureModeChanged() {
if (IsBeingUsedInPictureInPictureMode()) {
// When media enters PIP mode, we should ensure that the listener has been
// started because we always want to control PIP video.
StartMediaControlKeyListenerIfNeeded();
if (!mMediaControlKeyListener->IsStarted()) {
MEDIACONTROL_LOG("Failed to start listener when entering PIP mode");
}
// Updating controller PIP state no matter the listener starts or not.
mMediaControlKeyListener->SetPictureInPictureModeEnabled(true);
} else {
mMediaControlKeyListener->SetPictureInPictureModeEnabled(false);
}
}
bool HTMLMediaElement::IsBeingUsedInPictureInPictureMode() const {
if (!IsVideo()) {
return false;
}
return static_cast<const HTMLVideoElement*>(this)->IsCloningElementVisually();
}
void HTMLMediaElement::NodeInfoChanged(Document* aOldDoc) {
if (mMediaSource) {
OwnerDoc()->AddMediaElementWithMSE();
aOldDoc->RemoveMediaElementWithMSE();
}
nsGenericHTMLElement::NodeInfoChanged(aOldDoc);
}
#ifdef MOZ_WMF_CDM
bool HTMLMediaElement::IsUsingWMFCDM() const { return mIsUsingWMFCDM; };
#endif
} // namespace mozilla::dom
#undef LOG
#undef LOG_EVENT