DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Implementation

Mercurial (cdf352f02ac4)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef MODULES_VIDEO_CODING_JITTER_BUFFER_H_
#define MODULES_VIDEO_CODING_JITTER_BUFFER_H_

#include <list>
#include <map>
#include <memory>
#include <set>
#include <vector>

#include "modules/include/module_common_types.h"
#include "modules/utility/include/process_thread.h"
#include "modules/video_coding/decoding_state.h"
#include "modules/video_coding/include/video_coding.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "modules/video_coding/inter_frame_delay.h"
#include "modules/video_coding/jitter_buffer_common.h"
#include "modules/video_coding/jitter_estimator.h"
#include "modules/video_coding/nack_module.h"
#include "rtc_base/constructormagic.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/thread_annotations.h"
#include "typedefs.h"  // NOLINT(build/include)

namespace webrtc {

enum VCMNackMode { kNack, kNoNack };

// forward declarations
class Clock;
class EventFactory;
class EventWrapper;
class VCMFrameBuffer;
class VCMPacket;
class VCMEncodedFrame;

typedef std::list<VCMFrameBuffer*> UnorderedFrameList;

struct VCMJitterSample {
  VCMJitterSample() : timestamp(0), frame_size(0), latest_packet_time(-1) {}
  uint32_t timestamp;
  uint32_t frame_size;
  int64_t latest_packet_time;
};

class TimestampLessThan {
 public:
  bool operator()(uint32_t timestamp1, uint32_t timestamp2) const {
    return IsNewerTimestamp(timestamp2, timestamp1);
  }
};

class FrameList
    : public std::map<uint32_t, VCMFrameBuffer*, TimestampLessThan> {
 public:
  void InsertFrame(VCMFrameBuffer* frame);
  VCMFrameBuffer* FindFrame(uint16_t seq_num, uint32_t timestamp);
  VCMFrameBuffer* PopFrame(uint32_t timestamp);
  VCMFrameBuffer* Front() const;
  VCMFrameBuffer* Back() const;
  int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
                                 UnorderedFrameList* free_frames);
  void CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
                               UnorderedFrameList* free_frames);
  void Reset(UnorderedFrameList* free_frames);
};

class Vp9SsMap {
 public:
  typedef std::map<uint32_t, GofInfoVP9, TimestampLessThan> SsMap;
  bool Insert(const VCMPacket& packet);
  void Reset();

  // Removes SS data that are older than |timestamp|.
  // The |timestamp| should be an old timestamp, i.e. packets with older
  // timestamps should no longer be inserted.
  void RemoveOld(uint32_t timestamp);

  bool UpdatePacket(VCMPacket* packet);
  void UpdateFrames(FrameList* frames);

  // Public for testing.
  // Returns an iterator to the corresponding SS data for the input |timestamp|.
  bool Find(uint32_t timestamp, SsMap::iterator* it);

 private:
  // These two functions are called by RemoveOld.
  // Checks if it is time to do a clean up (done each kSsCleanupIntervalSec).
  bool TimeForCleanup(uint32_t timestamp) const;

  // Advances the oldest SS data to handle timestamp wrap in cases where SS data
  // are received very seldom (e.g. only once in beginning, second when
  // IsNewerTimestamp is not true).
  void AdvanceFront(uint32_t timestamp);

  SsMap ss_map_;
};

class VCMJitterBuffer {
 public:
  VCMJitterBuffer(Clock* clock,
                  std::unique_ptr<EventWrapper> event,
                  NackSender* nack_sender = nullptr,
                  KeyFrameRequestSender* keyframe_request_sender = nullptr);

  ~VCMJitterBuffer();

  // Initializes and starts jitter buffer.
  void Start();

  // Signals all internal events and stops the jitter buffer.
  void Stop();

  // Returns true if the jitter buffer is running.
  bool Running() const;

  // Empty the jitter buffer of all its data.
  void Flush();

  // Get the number of received frames, by type, since the jitter buffer
  // was started.
  FrameCounts FrameStatistics() const;

  // Gets number of packets received.
  int num_packets() const;

  // Gets number of duplicated packets received.
  int num_duplicated_packets() const;

  // Gets number of packets discarded by the jitter buffer.
  int num_discarded_packets() const;

  // Statistics, Calculate frame and bit rates.
  void IncomingRateStatistics(unsigned int* framerate, unsigned int* bitrate);

  // Wait |max_wait_time_ms| for a complete frame to arrive.
  // If found, a pointer to the frame is returned. Returns nullptr otherwise.
  VCMEncodedFrame* NextCompleteFrame(uint32_t max_wait_time_ms);

  // Locates a frame for decoding (even an incomplete) without delay.
  // The function returns true once such a frame is found, its corresponding
  // timestamp is returned. Otherwise, returns false.
  bool NextMaybeIncompleteTimestamp(uint32_t* timestamp);

  // Extract frame corresponding to input timestamp.
  // Frame will be set to a decoding state.
  VCMEncodedFrame* ExtractAndSetDecode(uint32_t timestamp);

  // Releases a frame returned from the jitter buffer, should be called when
  // done with decoding.
  void ReleaseFrame(VCMEncodedFrame* frame);

  // Returns the time in ms when the latest packet was inserted into the frame.
  // Retransmitted is set to true if any of the packets belonging to the frame
  // has been retransmitted.
  int64_t LastPacketTime(const VCMEncodedFrame* frame,
                         bool* retransmitted) const;

  // Inserts a packet into a frame returned from GetFrame().
  // If the return value is <= 0, |frame| is invalidated and the pointer must
  // be dropped after this function returns.
  VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted);

  // Returns the estimated jitter in milliseconds.
  uint32_t EstimatedJitterMs();

  // Updates the round-trip time estimate.
  void UpdateRtt(int64_t rtt_ms);

  // Set the NACK mode. |high_rtt_nack_threshold_ms| is an RTT threshold in ms
  // above which NACK will be disabled if the NACK mode is |kNack|, -1 meaning
  // that NACK is always enabled in the |kNack| mode.
  // |low_rtt_nack_threshold_ms| is an RTT threshold in ms below which we expect
  // to rely on NACK only, and therefore are using larger buffers to have time
  // to wait for retransmissions.
  void SetNackMode(VCMNackMode mode,
                   int64_t low_rtt_nack_threshold_ms,
                   int64_t high_rtt_nack_threshold_ms);

  void SetNackSettings(size_t max_nack_list_size,
                       int max_packet_age_to_nack,
                       int max_incomplete_time_ms);

  // Returns the current NACK mode.
  VCMNackMode nack_mode() const;

  // Returns a list of the sequence numbers currently missing.
  std::vector<uint16_t> GetNackList(bool* request_key_frame);

  // Set decode error mode - Should not be changed in the middle of the
  // session. Changes will not influence frames already in the buffer.
  void SetDecodeErrorMode(VCMDecodeErrorMode error_mode);
  VCMDecodeErrorMode decode_error_mode() const { return decode_error_mode_; }

  void RegisterStatsCallback(VCMReceiveStatisticsCallback* callback);

 private:
  class SequenceNumberLessThan {
   public:
    bool operator()(const uint16_t& sequence_number1,
                    const uint16_t& sequence_number2) const {
      return IsNewerSequenceNumber(sequence_number2, sequence_number1);
    }
  };
  typedef std::set<uint16_t, SequenceNumberLessThan> SequenceNumberSet;

  // Gets the frame assigned to the timestamp of the packet. May recycle
  // existing frames if no free frames are available. Returns an error code if
  // failing, or kNoError on success. |frame_list| contains which list the
  // packet was in, or NULL if it was not in a FrameList (a new frame).
  VCMFrameBufferEnum GetFrame(const VCMPacket& packet,
                              VCMFrameBuffer** frame,
                              FrameList** frame_list)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  // Returns true if |frame| is continuous in |decoding_state|, not taking
  // decodable frames into account.
  bool IsContinuousInState(const VCMFrameBuffer& frame,
                           const VCMDecodingState& decoding_state) const
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  // Returns true if |frame| is continuous in the |last_decoded_state_|, taking
  // all decodable frames into account.
  bool IsContinuous(const VCMFrameBuffer& frame) const
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  // Looks for frames in |incomplete_frames_| which are continuous in the
  // provided |decoded_state|. Starts the search from the timestamp of
  // |decoded_state|.
  void FindAndInsertContinuousFramesWithState(
      const VCMDecodingState& decoded_state)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  // Looks for frames in |incomplete_frames_| which are continuous in
  // |last_decoded_state_| taking all decodable frames into account. Starts
  // the search from |new_frame|.
  void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  VCMFrameBuffer* NextFrame() const RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  // Returns true if the NACK list was updated to cover sequence numbers up to
  // |sequence_number|. If false a key frame is needed to get into a state where
  // we can continue decoding.
  bool UpdateNackList(uint16_t sequence_number)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  bool TooLargeNackList() const;
  // Returns true if the NACK list was reduced without problem. If false a key
  // frame is needed to get into a state where we can continue decoding.
  bool HandleTooLargeNackList() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  bool MissingTooOldPacket(uint16_t latest_sequence_number) const
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  // Returns true if the too old packets was successfully removed from the NACK
  // list. If false, a key frame is needed to get into a state where we can
  // continue decoding.
  bool HandleTooOldPackets(uint16_t latest_sequence_number)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
  // Drops all packets in the NACK list up until |last_decoded_sequence_number|.
  void DropPacketsFromNackList(uint16_t last_decoded_sequence_number);

  // Gets an empty frame, creating a new frame if necessary (i.e. increases
  // jitter buffer size).
  VCMFrameBuffer* GetEmptyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  // Attempts to increase the size of the jitter buffer. Returns true on
  // success, false otherwise.
  bool TryToIncreaseJitterBufferSize() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  // Recycles oldest frames until a key frame is found. Used if jitter buffer is
  // completely full. Returns true if a key frame was found.
  bool RecycleFramesUntilKeyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  // Updates the frame statistics.
  // Counts only complete frames, so decodable incomplete frames will not be
  // counted.
  void CountFrame(const VCMFrameBuffer& frame)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  // Update rolling average of packets per frame.
  void UpdateAveragePacketsPerFrame(int current_number_packets_);

  // Cleans the frame list in the JB from old/empty frames.
  // Should only be called prior to actual use.
  void CleanUpOldOrEmptyFrames() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  // Returns true if |packet| is likely to have been retransmitted.
  bool IsPacketRetransmitted(const VCMPacket& packet) const;

  // The following three functions update the jitter estimate with the
  // payload size, receive time and RTP timestamp of a frame.
  void UpdateJitterEstimate(const VCMJitterSample& sample,
                            bool incomplete_frame);
  void UpdateJitterEstimate(const VCMFrameBuffer& frame, bool incomplete_frame);
  void UpdateJitterEstimate(int64_t latest_packet_time_ms,
                            uint32_t timestamp,
                            unsigned int frame_size,
                            bool incomplete_frame);

  // Returns true if we should wait for retransmissions, false otherwise.
  bool WaitForRetransmissions();

  int NonContinuousOrIncompleteDuration()
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const;

  void UpdateHistograms() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  // Reset frame buffer and return it to free_frames_.
  void RecycleFrameBuffer(VCMFrameBuffer* frame)
      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);

  Clock* clock_;
  // If we are running (have started) or not.
  bool running_;
  rtc::CriticalSection crit_sect_;
  // Event to signal when we have a frame ready for decoder.
  std::unique_ptr<EventWrapper> frame_event_;
  // Number of allocated frames.
  int max_number_of_frames_;
  UnorderedFrameList free_frames_ RTC_GUARDED_BY(crit_sect_);
  FrameList decodable_frames_ RTC_GUARDED_BY(crit_sect_);
  FrameList incomplete_frames_ RTC_GUARDED_BY(crit_sect_);
  VCMDecodingState last_decoded_state_ RTC_GUARDED_BY(crit_sect_);
  bool first_packet_since_reset_;

  // Statistics.
  VCMReceiveStatisticsCallback* stats_callback_ RTC_GUARDED_BY(crit_sect_);
  // Frame counts for each type (key, delta, ...)
  FrameCounts receive_statistics_;
  // Latest calculated frame rates of incoming stream.
  unsigned int incoming_frame_rate_;
  unsigned int incoming_frame_count_;
  int64_t time_last_incoming_frame_count_;
  unsigned int incoming_bit_count_;
  unsigned int incoming_bit_rate_;
  // Number of packets in a row that have been too old.
  int num_consecutive_old_packets_;
  // Number of packets received.
  int num_packets_ RTC_GUARDED_BY(crit_sect_);
  // Number of duplicated packets received.
  int num_duplicated_packets_ RTC_GUARDED_BY(crit_sect_);
  // Number of packets discarded by the jitter buffer.
  int num_discarded_packets_ RTC_GUARDED_BY(crit_sect_);
  // Time when first packet is received.
  int64_t time_first_packet_ms_ RTC_GUARDED_BY(crit_sect_);

  // Jitter estimation.
  // Filter for estimating jitter.
  VCMJitterEstimator jitter_estimate_;
  // Calculates network delays used for jitter calculations.
  VCMInterFrameDelay inter_frame_delay_;
  VCMJitterSample waiting_for_completion_;
  int64_t rtt_ms_;

  // NACK and retransmissions.
  VCMNackMode nack_mode_;
  int64_t low_rtt_nack_threshold_ms_;
  int64_t high_rtt_nack_threshold_ms_;
  // Holds the internal NACK list (the missing sequence numbers).
  SequenceNumberSet missing_sequence_numbers_;
  uint16_t latest_received_sequence_number_;
  size_t max_nack_list_size_;
  int max_packet_age_to_nack_;  // Measured in sequence numbers.
  int max_incomplete_time_ms_;

  VCMDecodeErrorMode decode_error_mode_;
  // Estimated rolling average of packets per frame
  float average_packets_per_frame_;
  // average_packets_per_frame converges fast if we have fewer than this many
  // frames.
  int frame_counter_;

  RTC_DISALLOW_COPY_AND_ASSIGN(VCMJitterBuffer);
};
}  // namespace webrtc

#endif  // MODULES_VIDEO_CODING_JITTER_BUFFER_H_