Android ExoPlayer 音画同步代码分析开发者社区

Android ExoPlayer 音画同步代码分析

@Override
@Nullable
public MediaClock getMediaClock() {
  return this;
}

public void onRendererEnabled(Renderer renderer) throws ExoPlaybackException {
  @Nullable MediaClock rendererMediaClock = renderer.getMediaClock(); //只有音频的不为空
  if (rendererMediaClock != null && rendererMediaClock != rendererClock) { 
    if (rendererClock != null) {
      throw ExoPlaybackException.createForUnexpected(
          new IllegalStateException("Multiple renderer media clocks enabled."));
    this.rendererClock = rendererMediaClock;
    this.rendererClockSource = renderer;
    rendererClock.setPlaybackParameters(standaloneClock.getPlaybackParameters());
}

public interface MediaClock {
   * Returns the current media position in microseconds.
  long getPositionUs();
   * Attempts to set the playback parameters. The media clock may override the speed if changing the
   * playback parameters is not supported.
   * @param playbackParameters The playback parameters to attempt to set.
  void setPlaybackParameters(PlaybackParameters playbackParameters);
  /** Returns the active playback parameters. */
  PlaybackParameters getPlaybackParameters();
}

private void updatePlaybackPositions() throws ExoPlaybackException {
  MediaPeriodHolder playingPeriodHolder = queue.getPlayingPeriod();
  if (playingPeriodHolder == null) {
    return;
  // Update the playback position.
  long discontinuityPositionUs =
      playingPeriodHolder.prepared
          ? playingPeriodHolder.mediaPeriod.readDiscontinuity()
          : C.TIME_UNSET;
  if (discontinuityPositionUs != C.TIME_UNSET) {
    resetRendererPosition(discontinuityPositionUs);
    // A MediaPeriod may report a discontinuity at the current playback position to ensure the
    // renderers are flushed. Only report the discontinuity externally if the position changed.
    if (discontinuityPositionUs != playbackInfo.positionUs) {
      playbackInfo =
          handlePositionDiscontinuity(
              playbackInfo.periodId,
              discontinuityPositionUs,
              playbackInfo.requestedContentPositionUs);
      playbackInfoUpdate.setPositionDiscontinuity(Player.DISCONTINUITY_REASON_INTERNAL);
  } else {
    rendererPositionUs =
        mediaClock.syncAndGetPositionUs(  
            /* isReadingAhead= */ playingPeriodHolder != queue.getReadingPeriod());
    long periodPositionUs = playingPeriodHolder.toPeriodTime(rendererPositionUs);
    maybeTriggerPendingMessages(playbackInfo.positionUs, periodPositionUs);
    playbackInfo.positionUs = periodPositionUs;
  // Update the buffered position and total buffered duration.
  MediaPeriodHolder loadingPeriod = queue.getLoadingPeriod();
  playbackInfo.bufferedPositionUs = loadingPeriod.getBufferedPositionUs();
  playbackInfo.totalBufferedDurationUs = getTotalBufferedDurationUs();

private void doSomeWork() throws ExoPlaybackException, IOException {
  updatePlaybackPositions();  //更新位置
//...................//
   boolean renderersEnded = true;
   boolean renderersReadyOrEnded = true;
  for(Render render : enabledRenderers){    //便利所有当前正在使用的渲染器
      // TODO: Each renderer should return the maximum delay before which it wishes to be called
      // again. The minimum of these values should then be used as the delay before the next
      // invocation of this method.
       renderer.render(rendererPositionUs, rendererPositionElapsedRealtimeUs);  //同步位置
       renderersEnded = renderersEnded && renderer.isEnded();  //判断是否渲染结束
       boolean rendererReadyOrEnded = renderer.isReady() || readerer.isEnded() || rendererWaitingForNextStream(rendered);
       if(!rendererReadyOrEnded){  //如果不处于Ready状态或者结束状态，说明Renderer解码的数据可能存在问题
          renderer.maybeThrowStreamError();
       renderersReadyOrEnded = renderersReadyOrEnded && rendererReadyOrEnded;
}

 protected boolean processOutputBuffer(
      long positionUs,
      long elapsedRealtimeUs,
      @Nullable MediaCodec codec,
      @Nullable ByteBuffer buffer,
      int bufferIndex,
      int bufferFlags,
      int sampleCount,
      long bufferPresentationTimeUs,
      boolean isDecodeOnlyBuffer,
      boolean isLastBuffer,
      Format format)
      throws ExoPlaybackException {
    Assertions.checkNotNull(codec); // Can not render video without codec
    if (initialPositionUs == C.TIME_UNSET) {
      initialPositionUs = positionUs;
   //获取数据流偏移时间
    long outputStreamOffsetUs = getOutputStreamOffsetUs();
    //计算出当前要送显的pts
    long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
    if (isDecodeOnlyBuffer && !isLastBuffer) {
    //如果仅仅参与解码，且不是最后提个buffer，所有数据均不送显示， 最终调用codec.releaseOutputBuffer(index, false) ，false 表示不送显
      skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
      return true;
     // 计算出与同步时钟的时间偏移
    long earlyUs = bufferPresentationTimeUs - positionUs;
    if (surface == dummySurface) {
      // Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
      //如果是默认兜底的Surface，不进行同步，如果buffer晚于播放时间，直接丢弃SKIP，否则buffer依然保留，知道时间到达后再次skip
      if (isBufferLate(earlyUs)) {
        skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
        updateVideoFrameProcessingOffsetCounters(earlyUs);
        return true;
      return false;
    long elapsedRealtimeNowUs = SystemClock.elapsedRealtime() * 1000;
    long elapsedSinceLastRenderUs = elapsedRealtimeNowUs - lastRenderTimeUs;
    //判断是否处于播放状态
    boolean isStarted = getState() == STATE_STARTED;
    //判断是否该渲染首帧
    boolean shouldRenderFirstFrame =
        !renderedFirstFrameAfterEnable
            ? (isStarted || mayRenderFirstFrameAfterEnableIfNotStarted)
            : !renderedFirstFrameAfterReset;
    // Don't force output until we joined and the position reached the current stream.
    //判断是否强制渲染，基本上如果是首帧，或者上一帧显示超过100ms且early小于30ms才会强制渲染，其他情况不需要强制渲染，具体看shouldForceRenderOutputBuffer() 源码
    boolean forceRenderOutputBuffer =
        joiningDeadlineMs == C.TIME_UNSET
            && positionUs >= outputStreamOffsetUs
            && (shouldRenderFirstFrame
                || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs)));
    if (forceRenderOutputBuffer) {
      long releaseTimeNs = System.nanoTime();
      notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
      if (Util.SDK_INT >= 21) {
        renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
      } else {
        renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
      updateVideoFrameProcessingOffsetCounters(earlyUs);
      return true;
     //如果视频还没正式播放，直接返回
    if (!isStarted || positionUs == initialPositionUs) {
      return false;
    //下面是early大于 0的情况
    // Fine-grained adjustment of earlyUs based on the elapsed time since the start of the current
    // iteration of the rendering loop.
    long elapsedSinceStartOfLoopUs = elapsedRealtimeNowUs - elapsedRealtimeUs;
    earlyUs -= elapsedSinceStartOfLoopUs;
    // Compute the buffer's desired release time in nanoseconds.
    long systemTimeNs = System.nanoTime();
    long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000);
    // Apply a timestamp adjustment, if there is one.
    long adjustedReleaseTimeNs = frameReleaseTimeHelper.adjustReleaseTime(
        bufferPresentationTimeUs, unadjustedFrameReleaseTimeNs);
  //调整送显时间
    earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000;
//判断是否跳帧还是丢帧，跳帧和丢帧最大的区别是，后者会通知到播放器外部
    boolean treatDroppedBuffersAsSkipped = joiningDeadlineMs != C.TIME_UNSET;
    if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer)
        && maybeDropBuffersToKeyframe(
            codec, bufferIndex, presentationTimeUs, positionUs, treatDroppedBuffersAsSkipped)) {
      //是否丢帧到关键帧的位置，这里尤其需要注意，如果逻辑走到这里，解码器可能会重启，可能出现黑屏问题
      return false;
    } else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastBuffer)) {
      if (treatDroppedBuffersAsSkipped) {
        skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
      } else {
        dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
      updateVideoFrameProcessingOffsetCounters(earlyUs);
      return true;
    if (Util.SDK_INT >= 21) {
      // Let the underlying framework time the release.
     // android 5.0+ 版本只要送显时间小于50ms则送显
      if (earlyUs < 50000) {
        notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format);
        renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs);
        updateVideoFrameProcessingOffsetCounters(earlyUs);
        return true;
    } else {
      // We need to time the release ourselves.
      if (earlyUs < 30000) {
      //android 4.4 之前 版本 小于30ms送显，但是如果时间大于11ms，则适当降低延迟
        if (earlyUs > 11000) {
          // We're a little too early to render the frame. Sleep until the frame can be rendered.
          // Note: The 11ms threshold was chosen fairly arbitrarily.
          try {
            // Subtracting 10000 rather than 11000 ensures the sleep time will be at least 1ms.
           //这里实际上exoplayer希望的是尽可能保证10ms的节奏，如果超过至少sleep 1ms
            Thread.sleep((earlyUs - 10000) / 1000);
          } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            return false;
        notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format);
        renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
        updateVideoFrameProcessingOffsetCounters(earlyUs);
        return true;
     //返回false，一般来说主要还是送显时间未到，那么未送显的buffer仍然保留着
    // We're either not playing, or it's not time to render the frame yet.
    return false;
  }

public class MusicMediaCodecVideoRenderer extends MediaCodecVideoRenderer {
    private KtvMediaClock mediaClock;
    public MusicMediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector) {
        super(context, mediaCodecSelector);
    public MusicMediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector, long allowedJoiningTimeMs) {
        super(context, mediaCodecSelector, allowedJoiningTimeMs);
    public MusicMediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector, long allowedJoiningTimeMs, @Nullable  Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify) {
        super(context, mediaCodecSelector, allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify);
    public MusicMediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector, long allowedJoiningTimeMs, boolean enableDecoderFallback, @Nullable Handler eventHandler, @Nullable  VideoRendererEventListener eventListener, int maxDroppedFramesToNotify) {
        super(context, mediaCodecSelector, allowedJoiningTimeMs, enableDecoderFallback, eventHandler, eventListener, maxDroppedFramesToNotify);
    public void setMediaClock(KtvMediaClock mediaClock) {
        this.mediaClock = mediaClock;
    @Override
    protected void onStarted() {
        super.onStarted();
        if(this.mediaClock != null) {
            this.mediaClock.start();
    @Override
    protected void onStopped() {
        super.onStopped();
        if (this.mediaClock != null) {
            this.mediaClock.stop();
    @Override
    protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
        super.onPositionReset(positionUs, joining);
        if (this.mediaClock != null) {
            this.mediaClock.resetPosition(positionUs);
    @Override
    public MediaClock getMediaClock() {
        return mediaClock;
}

public class AudioClockOutputDevice extends AudioOutput{
    private final AudioParams params;
    private boolean isResumed = false;
    int playState = 0;
    int writeFrameSize = 0;
    public AudioClockOutputDevice(AudioParams params) {
        this.params = params;
        playState = AudioPlayState.PLAYSTATE_NEW;
    @Override
    public void start() throws IOException {
        isResumed = true;
        playState = AudioPlayState.PLAYSTATE_PLAYING;
    @Override
    public void stop() throws IOException {
        isResumed = false;
        playState = AudioPlayState.PLAYSTATE_STOPPED;
    @Override
    public void resume() throws IOException {
        isResumed = true;
        playState = AudioPlayState.PLAYSTATE_PLAYING;
    @Override
    public void pause() throws IOException {
        isResumed = false;
        playState = AudioPlayState.PLAYSTATE_PAUSED;
    @Override
    public void flush() throws IOException {
    @Override
    public void release() throws IOException {
        playState = AudioPlayState.PLAYSTATE_STOPPED;
    @Override
    public int write(AudioFrame audioFrame) throws IOException {
        int toTimeMillis = AudioUtils.byteSizeToTimeMillis(audioFrame.size, (int) this.params.sampleRate, this.params.channelCount, this.params.bitDepth);
        int totalWrittenFrameSize = writeFrameSize + audioFrame.size;
        if (toTimeMillis > 0) {
            long targetTimeMillis = SystemClock.uptimeMillis() + toTimeMillis;
            int bytePerMilliseconds = audioFrame.size / toTimeMillis;
            final long STEP = 10; 
            while (targetTimeMillis >= (SystemClock.uptimeMillis() + STEP)) {
                try {
                    TimeUnit.MILLISECONDS.sleep(STEP);
                } catch (Throwable e) {
                    e.printStackTrace();
                int result  = (int) (writeFrameSize + STEP * bytePerMilliseconds);
                if(result > totalWrittenFrameSize){
                    result = totalWrittenFrameSize;
                writeFrameSize = result;
        writeFrameSize = totalWrittenFrameSize;
        return audioFrame.size;
    @Override
    public void setVolume(float v) throws IOException {
    @Override
    public void setMicVolume(float v) throws IOException {
    @Override
    public int getPlaybackHeadPosition() throws IOException {
        return AudioUtils.byteSizeToSamplePosition(writeFrameSize,this.params.channelCount,this.params.bitDepth);
    @Override
    public int getPlaybackBufferSize() throws IOException {
        return 4096;
    @Override
    public int getAudioSessionId() throws IOException {
        return 0;

Android ExoPlayer 音画同步代码分析

Android ExoPlayer 音画同步代码分析

一、音画同步

回到本文主题，我们来分析一下ExoPlayer的音画同步方式，以便利用这种机制实现一些场景下的多播放器同步。和主流播放器一样，ExoPlayer也是以音频为准的同步方式，本文将一步一步解释说明。

四、ExoPlayer 音画同步流程总结

五、如何在业务中使用自定义的MediaClock呢 ？

六、附：关于AudioTrack的缺陷