Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#if !defined(PlatformDecoderModule_h_)
# define PlatformDecoderModule_h_
# include "DecoderDoctorLogger.h"
# include "GMPCrashHelper.h"
# include "MediaCodecsSupport.h"
# include "MediaEventSource.h"
# include "MediaInfo.h"
# include "MediaResult.h"
# include "mozilla/EnumSet.h"
# include "mozilla/EnumTypeTraits.h"
# include "mozilla/MozPromise.h"
# include "mozilla/RefPtr.h"
# include "mozilla/TaskQueue.h"
# include "mozilla/layers/KnowsCompositor.h"
# include "mozilla/layers/LayersTypes.h"
# include "mozilla/ipc/UtilityAudioDecoder.h"
# include "nsTArray.h"
# include "PerformanceRecorder.h"
namespace mozilla {
class TrackInfo;
class AudioInfo;
class VideoInfo;
class MediaRawData;
class DecoderDoctorDiagnostics;
namespace layers {
class ImageContainer;
} // namespace layers
class MediaDataDecoder;
class RemoteDecoderModule;
class CDMProxy;
static LazyLogModule sPDMLog("PlatformDecoderModule");
namespace media {
enum class Option {
Default,
LowLatency,
HardwareDecoderNotAllowed,
FullH264Parsing,
ErrorIfNoInitializationData, // By default frames delivered before
// initialization data are dropped. Pass this
// option to raise an error if frames are
// delivered before initialization data.
DefaultPlaybackDeviceMono, // Currently only used by Opus on RDD to avoid
// initialization of audio backends on RDD
SENTINEL // one past the last valid value
};
using OptionSet = EnumSet<Option>;
struct UseNullDecoder {
UseNullDecoder() = default;
explicit UseNullDecoder(bool aUseNullDecoder) : mUse(aUseNullDecoder) {}
bool mUse = false;
};
// Do not wrap H264 decoder in a H264Converter.
struct NoWrapper {
NoWrapper() = default;
explicit NoWrapper(bool aDontUseWrapper) : mDontUseWrapper(aDontUseWrapper) {}
bool mDontUseWrapper = false;
};
struct VideoFrameRate {
VideoFrameRate() = default;
explicit VideoFrameRate(float aFramerate) : mValue(aFramerate) {}
float mValue = 0.0f;
};
} // namespace media
struct CreateDecoderParams;
struct CreateDecoderParamsForAsync {
using Option = media::Option;
using OptionSet = media::OptionSet;
explicit CreateDecoderParamsForAsync(const CreateDecoderParams& aParams);
CreateDecoderParamsForAsync(CreateDecoderParamsForAsync&& aParams);
const VideoInfo& VideoConfig() const {
MOZ_ASSERT(mConfig->IsVideo());
return *mConfig->GetAsVideoInfo();
}
const AudioInfo& AudioConfig() const {
MOZ_ASSERT(mConfig->IsAudio());
return *mConfig->GetAsAudioInfo();
}
UniquePtr<TrackInfo> mConfig;
const RefPtr<layers::ImageContainer> mImageContainer;
const RefPtr<layers::KnowsCompositor> mKnowsCompositor;
const RefPtr<GMPCrashHelper> mCrashHelper;
const media::UseNullDecoder mUseNullDecoder;
const media::NoWrapper mNoWrapper;
const TrackInfo::TrackType mType = TrackInfo::kUndefinedTrack;
std::function<MediaEventProducer<TrackInfo::TrackType>*()>
mOnWaitingForKeyEvent;
const OptionSet mOptions = OptionSet(Option::Default);
const media::VideoFrameRate mRate;
const Maybe<uint64_t> mMediaEngineId;
const Maybe<TrackingId> mTrackingId;
};
struct MOZ_STACK_CLASS CreateDecoderParams final {
using Option = media::Option;
using OptionSet = media::OptionSet;
using UseNullDecoder = media::UseNullDecoder;
using NoWrapper = media::NoWrapper;
using VideoFrameRate = media::VideoFrameRate;
explicit CreateDecoderParams(const TrackInfo& aConfig) : mConfig(aConfig) {}
CreateDecoderParams(const CreateDecoderParams& aParams) = default;
MOZ_IMPLICIT CreateDecoderParams(const CreateDecoderParamsForAsync& aParams)
: mConfig(*aParams.mConfig),
mImageContainer(aParams.mImageContainer),
mKnowsCompositor(aParams.mKnowsCompositor),
mCrashHelper(aParams.mCrashHelper),
mUseNullDecoder(aParams.mUseNullDecoder),
mNoWrapper(aParams.mNoWrapper),
mType(aParams.mType),
mOnWaitingForKeyEvent(aParams.mOnWaitingForKeyEvent),
mOptions(aParams.mOptions),
mRate(aParams.mRate),
mMediaEngineId(aParams.mMediaEngineId),
mTrackingId(aParams.mTrackingId) {}
template <typename T1, typename... Ts>
CreateDecoderParams(const TrackInfo& aConfig, T1&& a1, Ts&&... args)
: mConfig(aConfig) {
Set(std::forward<T1>(a1), std::forward<Ts>(args)...);
}
template <typename T1, typename... Ts>
CreateDecoderParams(const CreateDecoderParams& aParams, T1&& a1, Ts&&... args)
: CreateDecoderParams(aParams) {
Set(std::forward<T1>(a1), std::forward<Ts>(args)...);
}
const VideoInfo& VideoConfig() const {
MOZ_ASSERT(mConfig.IsVideo());
return *mConfig.GetAsVideoInfo();
}
const AudioInfo& AudioConfig() const {
MOZ_ASSERT(mConfig.IsAudio());
return *mConfig.GetAsAudioInfo();
}
layers::LayersBackend GetLayersBackend() const {
if (mKnowsCompositor) {
return mKnowsCompositor->GetCompositorBackendType();
}
return layers::LayersBackend::LAYERS_NONE;
}
// CreateDecoderParams is a MOZ_STACK_CLASS, it is only used to
// simplify the passing of arguments to Create*Decoder.
// It is safe to use references and raw pointers.
const TrackInfo& mConfig;
layers::ImageContainer* mImageContainer = nullptr;
MediaResult* mError = nullptr;
layers::KnowsCompositor* mKnowsCompositor = nullptr;
GMPCrashHelper* mCrashHelper = nullptr;
media::UseNullDecoder mUseNullDecoder;
media::NoWrapper mNoWrapper;
TrackInfo::TrackType mType = TrackInfo::kUndefinedTrack;
std::function<MediaEventProducer<TrackInfo::TrackType>*()>
mOnWaitingForKeyEvent;
OptionSet mOptions = OptionSet(Option::Default);
media::VideoFrameRate mRate;
// Used on Windows when the MF media engine playback is enabled.
Maybe<uint64_t> mMediaEngineId;
Maybe<TrackingId> mTrackingId;
private:
void Set(layers::ImageContainer* aImageContainer) {
mImageContainer = aImageContainer;
}
void Set(MediaResult* aError) { mError = aError; }
void Set(GMPCrashHelper* aCrashHelper) { mCrashHelper = aCrashHelper; }
void Set(UseNullDecoder aUseNullDecoder) {
mUseNullDecoder = aUseNullDecoder;
}
void Set(NoWrapper aNoWrapper) { mNoWrapper = aNoWrapper; }
void Set(const OptionSet& aOptions) { mOptions = aOptions; }
void Set(VideoFrameRate aRate) { mRate = aRate; }
void Set(layers::KnowsCompositor* aKnowsCompositor) {
if (aKnowsCompositor) {
mKnowsCompositor = aKnowsCompositor;
MOZ_ASSERT(aKnowsCompositor->IsThreadSafe());
}
}
void Set(TrackInfo::TrackType aType) { mType = aType; }
void Set(std::function<MediaEventProducer<TrackInfo::TrackType>*()>&&
aOnWaitingForKey) {
mOnWaitingForKeyEvent = std::move(aOnWaitingForKey);
}
void Set(const std::function<MediaEventProducer<TrackInfo::TrackType>*()>&
aOnWaitingForKey) {
mOnWaitingForKeyEvent = aOnWaitingForKey;
}
void Set(const Maybe<uint64_t>& aMediaEngineId) {
mMediaEngineId = aMediaEngineId;
}
void Set(const Maybe<TrackingId>& aTrackingId) { mTrackingId = aTrackingId; }
void Set(const CreateDecoderParams& aParams) {
// Set all but mTrackInfo;
mImageContainer = aParams.mImageContainer;
mError = aParams.mError;
mKnowsCompositor = aParams.mKnowsCompositor;
mCrashHelper = aParams.mCrashHelper;
mUseNullDecoder = aParams.mUseNullDecoder;
mNoWrapper = aParams.mNoWrapper;
mType = aParams.mType;
mOnWaitingForKeyEvent = aParams.mOnWaitingForKeyEvent;
mOptions = aParams.mOptions;
mRate = aParams.mRate;
mMediaEngineId = aParams.mMediaEngineId;
mTrackingId = aParams.mTrackingId;
}
template <typename T1, typename T2, typename... Ts>
void Set(T1&& a1, T2&& a2, Ts&&... args) {
Set(std::forward<T1>(a1));
Set(std::forward<T2>(a2), std::forward<Ts>(args)...);
}
};
struct MOZ_STACK_CLASS SupportDecoderParams final {
using Option = media::Option;
using OptionSet = media::OptionSet;
using UseNullDecoder = media::UseNullDecoder;
using NoWrapper = media::NoWrapper;
using VideoFrameRate = media::VideoFrameRate;
explicit SupportDecoderParams(const TrackInfo& aConfig) : mConfig(aConfig) {}
explicit SupportDecoderParams(const CreateDecoderParams& aParams)
: mConfig(aParams.mConfig),
mError(aParams.mError),
mKnowsCompositor(aParams.mKnowsCompositor),
mUseNullDecoder(aParams.mUseNullDecoder),
mNoWrapper(aParams.mNoWrapper),
mOptions(aParams.mOptions),
mRate(aParams.mRate),
mMediaEngineId(aParams.mMediaEngineId) {}
template <typename T1, typename... Ts>
SupportDecoderParams(const TrackInfo& aConfig, T1&& a1, Ts&&... args)
: mConfig(aConfig) {
Set(std::forward<T1>(a1), std::forward<Ts>(args)...);
}
const nsCString& MimeType() const { return mConfig.mMimeType; }
const TrackInfo& mConfig;
DecoderDoctorDiagnostics* mDiagnostics = nullptr;
MediaResult* mError = nullptr;
RefPtr<layers::KnowsCompositor> mKnowsCompositor;
UseNullDecoder mUseNullDecoder;
NoWrapper mNoWrapper;
OptionSet mOptions = OptionSet(Option::Default);
VideoFrameRate mRate;
Maybe<uint64_t> mMediaEngineId;
private:
void Set(DecoderDoctorDiagnostics* aDiagnostics) {
mDiagnostics = aDiagnostics;
}
void Set(MediaResult* aError) { mError = aError; }
void Set(media::UseNullDecoder aUseNullDecoder) {
mUseNullDecoder = aUseNullDecoder;
}
void Set(media::NoWrapper aNoWrapper) { mNoWrapper = aNoWrapper; }
void Set(const media::OptionSet& aOptions) { mOptions = aOptions; }
void Set(media::VideoFrameRate aRate) { mRate = aRate; }
void Set(layers::KnowsCompositor* aKnowsCompositor) {
if (aKnowsCompositor) {
mKnowsCompositor = aKnowsCompositor;
MOZ_ASSERT(aKnowsCompositor->IsThreadSafe());
}
}
void Set(const Maybe<uint64_t>& aMediaEngineId) {
mMediaEngineId = aMediaEngineId;
}
template <typename T1, typename T2, typename... Ts>
void Set(T1&& a1, T2&& a2, Ts&&... args) {
Set(std::forward<T1>(a1));
Set(std::forward<T2>(a2), std::forward<Ts>(args)...);
}
};
// Used for IPDL serialization.
// The 'value' have to be the biggest enum from CreateDecoderParams::Option.
template <>
struct MaxEnumValue<::mozilla::CreateDecoderParams::Option> {
static constexpr unsigned int value =
static_cast<unsigned int>(CreateDecoderParams::Option::SENTINEL);
};
// The PlatformDecoderModule interface is used by the MediaFormatReader to
// abstract access to decoders provided by various
// platforms.
// Each platform (Windows, MacOSX, Linux, B2G etc) must implement a
// PlatformDecoderModule to provide access to its decoders in order to get
// decompressed H.264/AAC from the MediaFormatReader.
//
// Decoding is asynchronous, and should be performed on the task queue
// provided if the underlying platform isn't already exposing an async API.
//
// A cross-platform decoder module that discards input and produces "blank"
// output samples exists for testing, and is created when the pref
// "media.use-blank-decoder" is true.
class PlatformDecoderModule {
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(PlatformDecoderModule)
// Perform any per-instance initialization.
// This is called on the decode task queue.
virtual nsresult Startup() { return NS_OK; }
// Indicates if the PlatformDecoderModule supports decoding of aMimeType,
// and whether or not hardware-accelerated decoding is supported.
// The answer to both SupportsMimeType and Supports doesn't guarantee that
// creation of a decoder will actually succeed.
virtual media::DecodeSupportSet SupportsMimeType(
const nsACString& aMimeType,
DecoderDoctorDiagnostics* aDiagnostics) const = 0;
virtual media::DecodeSupportSet Supports(
const SupportDecoderParams& aParams,
DecoderDoctorDiagnostics* aDiagnostics) const {
const TrackInfo& trackInfo = aParams.mConfig;
const media::DecodeSupportSet support =
SupportsMimeType(trackInfo.mMimeType, aDiagnostics);
// Bail early if we don't support this format at all
if (support.isEmpty()) {
return support;
}
const auto* videoInfo = trackInfo.GetAsVideoInfo();
if (!videoInfo) {
// No video stream = software decode only
return media::DecodeSupport::SoftwareDecode;
}
// Check whether we support the desired color depth
if (!SupportsColorDepth(videoInfo->mColorDepth, aDiagnostics)) {
return media::DecodeSupportSet{};
}
return support;
}
using CreateDecoderPromise = MozPromise<RefPtr<MediaDataDecoder>, MediaResult,
/* IsExclusive = */ true>;
protected:
PlatformDecoderModule() = default;
virtual ~PlatformDecoderModule() = default;
friend class MediaChangeMonitor;
friend class PDMFactory;
friend class EMEDecoderModule;
friend class RemoteDecoderModule;
// Indicates if the PlatformDecoderModule supports decoding of aColorDepth.
// Should override this method when the platform can support color depth != 8.
virtual bool SupportsColorDepth(
gfx::ColorDepth aColorDepth,
DecoderDoctorDiagnostics* aDiagnostics) const {
return aColorDepth == gfx::ColorDepth::COLOR_8;
}
// Creates a Video decoder. The layers backend is passed in so that
// decoders can determine whether hardware accelerated decoding can be used.
// On Windows the task queue's threads in have MSCOM initialized with
// COINIT_MULTITHREADED.
// Returns nullptr if the decoder can't be created.
// It is not safe to store a reference to aParams or aParams.mConfig as the
// object isn't guaranteed to live after the call.
// CreateVideoDecoder may need to make additional checks if the
// CreateDecoderParams argument is actually supported and return nullptr if
// not to allow for fallback PDMs to be tried.
virtual already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
const CreateDecoderParams& aParams) = 0;
// Creates an Audio decoder with the specified properties.
// Returns nullptr if the decoder can't be created.
// On Windows the task queue's threads in have MSCOM initialized with
// COINIT_MULTITHREADED.
// It is not safe to store a reference to aParams or aParams.mConfig as the
// object isn't guaranteed to live after the call.
// CreateAudioDecoder may need to make additional checks if the
// CreateDecoderParams argument is actually supported and return nullptr if
// not to allow for fallback PDMs to be tried.
virtual already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
const CreateDecoderParams& aParams) = 0;
// Asychronously create a decoder.
virtual RefPtr<CreateDecoderPromise> AsyncCreateDecoder(
const CreateDecoderParams& aParams);
};
DDLoggedTypeDeclName(MediaDataDecoder);
// MediaDataDecoder is the interface exposed by decoders created by the
// PlatformDecoderModule's Create*Decoder() functions. The type of
// media data that the decoder accepts as valid input and produces as
// output is determined when the MediaDataDecoder is created.
//
// Unless otherwise noted, all functions are only called on the decode task
// queue. An exception is the MediaDataDecoder in
// MediaFormatReader::IsVideoAccelerated() for which all calls (Init(),
// IsHardwareAccelerated(), and Shutdown()) are from the main thread.
//
// Don't block inside these functions, unless it's explicitly noted that you
// should (like in Flush()).
//
// Decoding is done asynchronously.
class MediaDataDecoder : public DecoderDoctorLifeLogger<MediaDataDecoder> {
protected:
virtual ~MediaDataDecoder() = default;
public:
using TrackType = TrackInfo::TrackType;
using DecodedData = nsTArray<RefPtr<MediaData>>;
using InitPromise = MozPromise<TrackType, MediaResult, true>;
using DecodePromise = MozPromise<DecodedData, MediaResult, true>;
using FlushPromise = MozPromise<bool, MediaResult, true>;
NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING
// Initialize the decoder. The decoder should be ready to decode once
// the promise resolves. The decoder should do any initialization here, rather
// than in its constructor or PlatformDecoderModule::Create*Decoder(),
// so that if the MediaFormatReader needs to shutdown during initialization,
// it can call Shutdown() to cancel this operation. Any initialization
// that requires blocking the calling thread in this function *must*
// be done here so that it can be canceled by calling Shutdown()!
// Methods Decode, DecodeBatch, Drain, Flush, Shutdown are guaranteed to be
// called on the thread where Init() first ran.
virtual RefPtr<InitPromise> Init() = 0;
// Inserts a sample into the decoder's decode pipeline. The DecodePromise will
// be resolved with the decoded MediaData. In case the decoder needs more
// input, the DecodePromise may be resolved with an empty array of samples to
// indicate that Decode should be called again before a MediaData is returned.
virtual RefPtr<DecodePromise> Decode(MediaRawData* aSample) = 0;
// This could probably be implemented as a wrapper that takes a
// generic MediaDataDecoder and manages batching as needed. For now
// only AudioTrimmer with RemoteMediaDataDecoder supports batch
// decoding.
// Inserts an array of samples into the decoder's decode pipeline. The
// DecodePromise will be resolved with the decoded MediaData. In case
// the decoder needs more input, the DecodePromise may be resolved
// with an empty array of samples to indicate that Decode should be
// called again before a MediaData is returned.
virtual bool CanDecodeBatch() const { return false; }
virtual RefPtr<DecodePromise> DecodeBatch(
nsTArray<RefPtr<MediaRawData>>&& aSamples) {
MOZ_CRASH("DecodeBatch not implemented yet");
return MediaDataDecoder::DecodePromise::CreateAndReject(
NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__);
}
// Causes all complete samples in the pipeline that can be decoded to be
// output. If the decoder can't produce samples from the current output,
// it drops the input samples. The decoder may be holding onto samples
// that are required to decode samples that it expects to get in future.
// This is called when the demuxer reaches end of stream.
// This function is asynchronous.
// The MediaDataDecoder shall resolve the pending DecodePromise with drained
// samples. Drain will be called multiple times until the resolved
// DecodePromise is empty which indicates that there are no more samples to
// drain.
virtual RefPtr<DecodePromise> Drain() = 0;
// Causes all samples in the decoding pipeline to be discarded. When this
// promise resolves, the decoder must be ready to accept new data for
// decoding. This function is called when the demuxer seeks, before decoding
// resumes after the seek. The current DecodePromise if any shall be rejected
// with NS_ERROR_DOM_MEDIA_CANCELED
virtual RefPtr<FlushPromise> Flush() = 0;
// Cancels all init/decode/drain operations, and shuts down the decoder. The
// platform decoder should clean up any resources it's using and release
// memory etc. The shutdown promise will be resolved once the decoder has
// completed shutdown. The reader calls Flush() before calling Shutdown(). The
// reader will delete the decoder once the promise is resolved.
// The ShutdownPromise must only ever be resolved.
// Shutdown() may not be called if init hasn't been called first. It is
// possible under some circumstances for the decoder to be deleted without
// Init having been called first.
virtual RefPtr<ShutdownPromise> Shutdown() = 0;
// Called from the state machine task queue or main thread. Decoder needs to
// decide whether or not hardware acceleration is supported after creating.
// It doesn't need to call Init() before calling this function.
virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const {
return false;
}
// Return the name of the MediaDataDecoder, only used for decoding.
// May be accessed in a non thread-safe fashion.
virtual nsCString GetDescriptionName() const = 0;
virtual nsCString GetProcessName() const {
nsCString rv = nsCString(XRE_GetProcessTypeString());
if (XRE_IsUtilityProcess()) {
rv += "+"_ns + mozilla::ipc::GetChildAudioActorName();
}
return rv;
};
virtual nsCString GetCodecName() const = 0;
// Set a hint of seek target time to decoder. Decoder will drop any decoded
// data which pts is smaller than this value. This threshold needs to be clear
// after reset decoder. To clear it explicitly, call this method with
// TimeUnit::Invalid().
// Decoder may not honor this value. However, it'd be better that
// video decoder implements this API to improve seek performance.
// Note: it should be called before Input() or after Flush().
virtual void SetSeekThreshold(const media::TimeUnit& aTime) {}
// When playing adaptive playback, recreating an Android video decoder will
// cause the transition not smooth during resolution change.
// Reuse the decoder if the decoder support recycling.
// Currently, only Android video decoder will return true.
virtual bool SupportDecoderRecycling() const { return false; }
enum class ConversionRequired {
kNeedNone = 0,
kNeedAVCC = 1,
kNeedAnnexB = 2,
};
// Indicates that the decoder requires a specific format.
// The demuxed data will be converted accordingly before feeding it to
// Decode().
virtual ConversionRequired NeedsConversion() const {
return ConversionRequired::kNeedNone;
}
};
} // namespace mozilla
#endif