DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Mercurial (9084006e2fcb)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef VideoOutput_h
#define VideoOutput_h

#include "MediaStreamListener.h"
#include "VideoFrameContainer.h"

namespace mozilla {

using layers::Image;
using layers::ImageContainer;
using layers::PlanarYCbCrData;
using layers::PlanarYCbCrImage;

static bool SetImageToBlackPixel(PlanarYCbCrImage* aImage) {
  uint8_t blackPixel[] = {0x10, 0x80, 0x80};

  PlanarYCbCrData data;
  data.mYChannel = blackPixel;
  data.mCbChannel = blackPixel + 1;
  data.mCrChannel = blackPixel + 2;
  data.mYStride = data.mCbCrStride = 1;
  data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1);
  return aImage->CopyData(data);
}

class VideoOutput : public DirectMediaStreamTrackListener {
 protected:
  virtual ~VideoOutput() = default;

  void DropPastFrames() {
    TimeStamp now = TimeStamp::Now();
    size_t nrChunksInPast = 0;
    for (const auto& idChunkPair : mFrames) {
      const VideoChunk& chunk = idChunkPair.second();
      if (chunk.mTimeStamp > now) {
        break;
      }
      ++nrChunksInPast;
    }
    if (nrChunksInPast > 1) {
      // We need to keep one frame that starts in the past, because it only ends
      // when the next frame starts (which also needs to be in the past for it
      // to drop).
      mFrames.RemoveElementsAt(0, nrChunksInPast - 1);
    }
  }

  void SendFramesEnsureLocked() {
    mMutex.AssertCurrentThreadOwns();
    SendFrames();
  }

  void SendFrames() {
    DropPastFrames();

    if (mFrames.IsEmpty()) {
      return;
    }

    // Collect any new frames produced in this iteration.
    AutoTArray<ImageContainer::NonOwningImage, 16> images;
    PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;

    for (const auto& idChunkPair : mFrames) {
      ImageContainer::FrameID frameId = idChunkPair.first();
      const VideoChunk& chunk = idChunkPair.second();
      const VideoFrame& frame = chunk.mFrame;
      Image* image = frame.GetImage();
      if (frame.GetForceBlack() || !mEnabled) {
        if (!mBlackImage) {
          RefPtr<Image> blackImage = mVideoFrameContainer->GetImageContainer()
                                         ->CreatePlanarYCbCrImage();
          if (blackImage) {
            // Sets the image to a single black pixel, which will be scaled to
            // fill the rendered size.
            if (SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage())) {
              mBlackImage = blackImage;
            }
          }
        }
        if (mBlackImage) {
          image = mBlackImage;
        }
      }
      if (!image) {
        // We ignore null images.
        continue;
      }
      images.AppendElement(ImageContainer::NonOwningImage(
          image, chunk.mTimeStamp, frameId, mProducerID));

      lastPrincipalHandle = chunk.GetPrincipalHandle();
    }

    if (images.IsEmpty()) {
      // This could happen if the only images in mFrames are null. We leave the
      // container at the current frame in this case.
      mVideoFrameContainer->ClearFutureFrames();
      return;
    }

    bool principalHandleChanged =
        lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
        lastPrincipalHandle != mVideoFrameContainer->GetLastPrincipalHandle();

    if (principalHandleChanged) {
      mVideoFrameContainer->UpdatePrincipalHandleForFrameID(
          lastPrincipalHandle, images.LastElement().mFrameID);
    }

    mVideoFrameContainer->SetCurrentFrames(
        mFrames[0].second().mFrame.GetIntrinsicSize(), images);
    mMainThread->Dispatch(NewRunnableMethod("VideoFrameContainer::Invalidate",
                                            mVideoFrameContainer,
                                            &VideoFrameContainer::Invalidate));
  }

 public:
  VideoOutput(VideoFrameContainer* aContainer, AbstractThread* aMainThread)
      : mMutex("VideoOutput::mMutex"),
        mVideoFrameContainer(aContainer),
        mMainThread(aMainThread) {}
  void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
                               StreamTime aTrackOffset,
                               const MediaSegment& aMedia) override {
    MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
    const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
    MutexAutoLock lock(mMutex);
    for (VideoSegment::ConstChunkIterator i(video); !i.IsEnded(); i.Next()) {
      if (!mLastFrameTime.IsNull() && i->mTimeStamp < mLastFrameTime) {
        // Time can go backwards if the source is a captured MediaDecoder and
        // it seeks, as the previously buffered frames would stretch into the
        // future. If this happens, we clear the buffered frames and start over.
        mFrames.ClearAndRetainStorage();
      }
      mFrames.AppendElement(MakePair(mVideoFrameContainer->NewFrameID(), *i));
      mLastFrameTime = i->mTimeStamp;
    }

    SendFramesEnsureLocked();
  }
  void NotifyRemoved() override {
    // Doesn't need locking by mMutex, since the direct listener is removed from
    // the track before we get notified.
    if (mFrames.Length() <= 1) {
      // The compositor has already received the last frame.
      mFrames.ClearAndRetainStorage();
      mVideoFrameContainer->ClearFutureFrames();
      return;
    }

    // The compositor has multiple frames. ClearFutureFrames() would only retain
    // the first as that's normally the current one. We however stop doing
    // SetCurrentFrames() once we've received the last frame in a track, so
    // there might be old frames lingering. We'll find the current one and
    // re-send that.
    DropPastFrames();
    mFrames.RemoveElementsAt(1, mFrames.Length() - 1);
    SendFrames();
    mFrames.ClearAndRetainStorage();
  }
  void NotifyEnded() override {
    // Doesn't need locking by mMutex, since for the track to end, it must have
    // been ended by the source, meaning that the source won't append more data.
    if (mFrames.IsEmpty()) {
      return;
    }

    // Re-send only the last one to the compositor.
    mFrames.RemoveElementsAt(0, mFrames.Length() - 1);
    SendFrames();
    mFrames.ClearAndRetainStorage();
  }
  void NotifyEnabledStateChanged(bool aEnabled) override {
    MutexAutoLock lock(mMutex);
    mEnabled = aEnabled;
    // Since mEnabled will affect whether frames are real, or black, we assign
    // new FrameIDs whenever this changes.
    for (auto& idChunkPair : mFrames) {
      idChunkPair.first() = mVideoFrameContainer->NewFrameID();
    }
    SendFramesEnsureLocked();
  }

  Mutex mMutex;
  TimeStamp mLastFrameTime;
  // Once the frame is forced to black, we initialize mBlackImage for use in any
  // following forced-black frames.
  RefPtr<Image> mBlackImage;
  bool mEnabled = true;
  // This array is accessed from both the direct video thread, and the graph
  // thread. Protected by mMutex.
  nsTArray<Pair<ImageContainer::FrameID, VideoChunk>> mFrames;
  const RefPtr<VideoFrameContainer> mVideoFrameContainer;
  const RefPtr<AbstractThread> mMainThread;
  const layers::ImageContainer::ProducerID mProducerID =
      layers::ImageContainer::AllocateProducerID();
};

}  // namespace mozilla

#endif  // VideoOutput_h