diff --git a/media/base/decoder.cc b/media/base/decoder.cc index 4a2d5c372326aea839dd3fe7fed58e20026d559e..ab0534a17e94ab1c90fe2be134f26a5b77217b30 100644 --- a/media/base/decoder.cc +++ b/media/base/decoder.cc @@ -81,6 +81,8 @@ std::string GetDecoderName(AudioDecoderType type) { return "AudioToolboxAudioDecoder"; case AudioDecoderType::kMediaFoundation: return "MediaFoundationAudioDecoder"; + case AudioDecoderType::kOhos: + return "OHOSAudioDecoder"; } } diff --git a/media/base/decoder.h b/media/base/decoder.h index 874910b633b38a6dc0aa1b73ab18c9674539da49..d7c2fa7dad8764f3ec50de2ef7348bb796e6ae5b 100644 --- a/media/base/decoder.h +++ b/media/base/decoder.h @@ -26,9 +26,9 @@ enum class AudioDecoderType : int { kAudioToolbox = 7, // AudioToolbox (macOS) kMediaFoundation = 8, // MediaFoundationAudioDecoder kPassthroughDTS = 9, // Passthrough DTS audio - + kOhos = 10, // Keep this at the end and equal to the last entry. - kMaxValue = kPassthroughDTS, + kMaxValue = kOhos, }; // List of known VideoDecoder implementations; recorded to UKM, always add new diff --git a/media/base/media_content_type.cc b/media/base/media_content_type.cc index 64d916f8009514edada570855f1455f0ac2c313e..bee1b3ef32d5f71b29f87f397fece21a87677bf9 100644 --- a/media/base/media_content_type.cc +++ b/media/base/media_content_type.cc @@ -15,7 +15,7 @@ const int kSnippetContentDurationSecs = 60; MediaContentType DurationToMediaContentType(base::TimeDelta duration) { #if defined(OHOS_MEDIA_POLICY) - if (duration > base::Seconds(kMinimumContentDurationSecs) && + if (duration > base::Seconds(kMinimumContentDurationSecs) && duration <= base::Seconds(kSnippetContentDurationSecs)) { return MediaContentType::Snippet; } diff --git a/media/filters/BUILD.gn b/media/filters/BUILD.gn index 599fa50787715e249ed846c37ad5cd90f74ee5d5..130148853918413074185b1bfe8e7f1486a03ddc 100644 --- a/media/filters/BUILD.gn +++ b/media/filters/BUILD.gn @@ -215,6 +215,15 @@ source_set("filters") { "mac/audio_toolbox_audio_encoder.h", ] } + if (is_ohos) { + sources += [ + "ohos/ohos_audio_decoder.cc", + "ohos/ohos_audio_decoder.h", + "ohos/ohos_audio_decoder_loop.cc", + "ohos/ohos_audio_decoder_loop.h", + ] + } + if (is_win && enable_platform_dts_audio) { sources += [ "passthrough_dts_audio_decoder.cc", diff --git a/media/filters/ohos/ohos_audio_decoder.cc b/media/filters/ohos/ohos_audio_decoder.cc new file mode 100644 index 0000000000000000000000000000000000000000..989ab408febdcc17adb7b3b3ed13713dda982933 --- /dev/null +++ b/media/filters/ohos/ohos_audio_decoder.cc @@ -0,0 +1,486 @@ +#include "ohos_audio_decoder.h" +#include +#include +#include +#include "base/android/build_info.h" +#include "base/functional/bind.h" +#include "base/functional/callback.h" +#include "base/functional/callback_helpers.h" +#include "base/logging.h" +#include "base/task/bind_post_task.h" +#include "base/task/sequenced_task_runner.h" +#include "base/task/single_thread_task_runner.h" +#include "media/base/audio_timestamp_helper.h" +#include "media/base/status.h" +#include "media/base/timestamp_constants.h" +#include "media/formats/ac3/ac3_util.h" +#include "media/formats/dts/dts_util.h" +#include "media/media_buildflags.h" +#include "ohos_adapter_helper.h" + +namespace media { + int32_t INVALID_ +int32_t OHOSAudioDecoderFormat::GetSampleRate() { + return sample_rate_; +} + +int32_t OHOSAudioDecoderFormat::GetChannelCount() { + return channel_count_; +} + +int32_t OHOSAudioDecoderFormat::GetBitRate() { + return bit_rate_; +} + +int32_t OHOSAudioDecoderFormat::GetMaxInputSize() { + return max_input_size_; +} + +void OHOSAudioDecoderFormat::SetSampleRate(int32_t sampleRate) { + sample_rate_ = sampleRate; +} + +void OHOSAudioDecoderFormat::SetChannelCount(int32_t channelCount) { + channel_count_ = channelCount; +} + +void OHOSAudioDecoderFormat::SetBitRate(int32_t bitRate) { + bit_rate_ = bitRate; +} + +void OHOSAudioDecoderFormat::SetMaxInputSize(int32_t maxInputSize) { + max_input_size_ = maxInputSize; +} + +void AudioDecoderCallback::OnError(int32_t errorCode) { + LOG(ERROR) << "OHOSAudioDecoder::AudioDecoderCallback::OnError, errorCode: " << errorCode << endl; + SetState(ERROR); +} + +void AudioDecoderCallback::OnOutputFormatChanged(const std::shared_ptr format) { + // TODO 这里生成一个新的OHOSAudioDecoderFormat, onDecodeFrame 的时候判断，是否存在回调的OutputFormat，如果存在则用回调的 OutputFormat。 +} + +void AudioDecoderCallback::OnInputBufferAvailable(uint32_t index) { + input_buffer_queue_.push_back(index); +} + +void AudioDecoderCallback::OnOutputBufferAvailable(uint32_t index, uint8_t *bufferData, uint32_t size) { + OutputBufferData data(index, data, size, 0, 0); + output_buffer_queue.push_back(data); +} + +AudioDecoderType OHOSAudioDecoder::GetDecoderType() const { + return AudioDecoderType::kOhos; +} + +void OHOSAudioDecoder::Initialize(const AudioDecoderConfig& config, + CdmContext* cdm_context, + InitCB init_cb, + const OutputCB& output_cb, + const WaitingCB& waiting_cb) { + DCHECK_NE(state_, WAITING_FOR_MEDIA_CRYPTO); + DCHECK(output_cb); + DCHECK(waiting_cb); + + // 清空输入buffer，设置回调结果为DecoderStatus::Codes::kAborted + DCHECK(input_queue_.empty()); + ClearInputQueue(DecoderStatus::Codes::kAborted); + + // 异常状态处理 + if (state_ == ERROR) { + // 通知 DecoderStream initialization 失敗 + base::BindPostTaskToCurrentDefault(std::move(init_cb)) + .Run(DecoderStatus::Codes::kFailed); + return; + } + + // 检查音频格式是否为 MediaCodec 支持 + std::string mime_type; + is_passthrough_ = false; + sample_format_ = config.target_output_sample_format(); + switch (config.codec()) { + case AudioCodec::kAAC: + mime_type = "OH_AVCODEC_MIMETYPE_AUDIO_AAC"; + break; + case AudioCodec::kFLAC: + mime_type = "OH_AVCODEC_MIMETYPE_AUDIO_FLAC"; + break; + case AudioCodec::kVorbis: + mime_type = "OH_AVCODEC_MIMETYPE_AUDIO_VORBIS"; + break; + case AudioCodec::kMP3: + mime_type = "OH_AVCODEC_MIMETYPE_AUDIO_MPEG"; + break; + case AudioCodec::kAMR_NB: + mime_type = "OH_AVCODEC_MIMETYPE_AUDIO_AMR_NB"; + break; + case AudioCodec::kAMR_WB: + mime_type = "OH_AVCODEC_MIMETYPE_AUDIO_AMR_WB"; + break; + case AudioCodec::kUnknown: + case AudioCodec::kOpus: + case AudioCodec::kPCM: + case AudioCodec::kPCM_MULAW: + case AudioCodec::kGSM_MS: + case AudioCodec::kPCM_S16BE: + case AudioCodec::kPCM_S24BE: + case AudioCodec::kPCM_ALAW: + case AudioCodec::kALAC: + break; + case AudioCodec::kAC3: + case AudioCodec::kEAC3: + case AudioCodec::kDTS: + case AudioCodec::kDTSXP2: + case AudioCodec::kDTSE: + case AudioCodec::kMpegHAudio: + is_passthrough_ = sample_format_ != kUnknownSampleFormat; + break; + } + + if (!is_passthrough_) + sample_format_ = kSampleFormatS16; + + const bool is_codec_supported = !mime_type.empty() || is_passthrough_; + if (!is_codec_supported) { + LOG(ERROR) << "OHOSAudioDecoder::Initialize, Unsupported codec " << GetCodecName(config.codec()); + base::BindPostTaskToCurrentDefault(std::move(init_cb)) + .Run(DecoderStatus::Codes::kUnsupportedCodec); + return; + } + + // 保存 AudioDecoderConfig + config_ = config; + channel_layout_ = config_.channel_layout(); + //src/media/base/channel_layout.cc + channel_count_ = ChannelLayoutToChannelCount(channel_layout_); + sample_rate_ = config_.samples_per_second(); + + timestamp_helper_ = std::make_unique(sample_rate_); + + output_cb_ = base::BindPostTaskToCurrentDefault(output_cb); + waiting_cb_ = base::BindPostTaskToCurrentDefault(waiting_cb); + + // TODO 解密判断，如果是加密的流，并且没有解密配置时，直接退出 + if (config_.is_encrypted()) { + // 没有 cdm_context 或者 cdm_context->GetMediaCryptoContext()，初始化失败，回调 + //if (!cdm_context || !cdm_context->GetMediaCryptoContext()) + if (!cdm_context) { + LOG(ERROR) << "The stream is encrypted but there is no CdmContext or " + "MediaCryptoContext is not supported"; + SetState(ERROR); + base::BindPostTaskToCurrentDefault(std::move(init_cb)) + .Run(DecoderStatus::Codes::kUnsupportedEncryptionMode); + return; + } + } + // TODO 处理 cdm context + SetState(WAITING_FOR_MEDIA_CRYPTO); + SetCdm(cdm_context, std::move(init_cb)); + if(!InitAudioDecoder(mime_type)) { + base::BindPostTaskToCurrentDefault(std::move(init_cb)) + .Run(DecoderStatus::Codes::kFailed); + return; + } + + // 创建 MediaCodecLoop + CreateOhosDecoderLoop(); + + // 设置状态为 ready + SetState(READY); + base::BindPostTaskToCurrentDefault(std::move(init_cb)) + .Run(DecoderStatus::Codes::kOk); +} + +void OHOSAudioDecoder::SetCdm(CdmContext* cdm_context, InitCB init_cb) { + if (!cdm_context) { + LOG(DEBUG) << "OHOSAudioDecoder::SetCdm No CDM provided"; + return; + } + // TODO oh 需要在 src/media/base/cdm_context.cc 定义一个 OhosCdmContext 对象，然后设置到 audioDecoder 对象中 +} + +bool OHOSAudioDecoder::InitAudioDecoder(std::string mime_type) { + audio_decoder_ = + OhosAdapterHelper::GetInstance().CreateAudioCodecDecoderAdapter(); + std::shared_ptr audioDecoderFormat = + std::make_shared(); + // 采样率和声道数量为必填，比特率和最大输出长度可选 + audioDecoderFormat->SetSampleRate(sample_rate_); + audioDecoderFormat->SetChannelCount(channel_count_); + + audio_decoder_->CreateAudioDecoderByMime(mime_type); + + // TODO callback 需要用什么对象，需要在这里传进去 + decoder_callback_ = std::make_shared(); + if (!decoder_callback_) { + LOG(ERROR) + << "OHOSAudioDecoder::InitAudioDecoder Get decoderCallback failed."; + return false; + } + + audio_decoder_->SetCallbackDec(decoder_callback_); + // TODO 这里需要设置解密密钥 + audio_decoder_->SetDecryptionConfig(true); + audio_decoder_->PrepareDecoder(); + audio_decoder_->StartDecoder(); + + return true; +} + +bool OHOSAudioDecoder::CreateOhosDecoderLoop() { + decoder_loop_.reset(); + decoder_loop_ = std::make_unique(this, + auido_decoder_, scoped_refptr()); + return true; +} + +// 设置 decoder 的状态 +void OHOSAudioDecoder::SetState(State new_state) { + state_ = new_state; +} + +// 为每个Decoder Buffer 设置 状态，清空 input_queue +void OHOSAudioDecoder::ClearInputQueue(DecoderStatus decode_status) { + for (auto& entry : input_queue_) + std::move(entry.second).Run(decode_status); + + input_queue_.clear(); +} + +void OHOSAudioDecoder::Decode(scoped_refptr buffer, DecodeCB decode_cb) { + DecodeCB cb = + base::BindPostTaskToCurrentDefault(std::move(decode_cb)); + + // src/media/base/decoder_buffer.cc + /** + *这个函数的目的是确保加密的媒体数据中的子样本与解密配置中定义的子样本匹配 + */ + if (!DecoderBuffer::DoSubsamplesMatch(*buffer)) { + std::move(cb).Run(DecoderStatus::Codes::kFailed); + return; + } + /** + * buffer 不是stream的末尾，并且没有时间，直接跳过 + */ + if (!buffer->end_of_stream() && buffer->timestamp() == kNoTimestamp) { + LOG(ERROR)<<"OHOSAudioDecoder::Decode "<< buffer->AsHumanReadableString() + << ": no timestamp, skipping this buffer"; + std::move(cb).Run(DecoderStatus::Codes::kFailed); + return; + } + + /** + * 失败状态处理，清理InputQueue，回调失败状态 + */ + // Note that we transition to STATE_ERROR if |codec_loop_| does. + if (state_ == ERROR) { + // We get here if an error happens in DequeueOutput() or Reset(). + LOG(ERROR)<<"OHOSAudioDecoder::Decode "<< buffer->AsHumanReadableString() + << ": Error state, returning decode error for all buffers"; + ClearInputQueue(DecoderStatus::Codes::kFailed); + std::move(cb).Run(DecoderStatus::Codes::kFailed); + return; + } + + DCHECK(decoder_loop_); + DCHECK_EQ(state_, READY) << " unexpected state " << AsString(state_); + + /** + * 在任何给定时间，只能有一个解码操作正在进行中 + */ + DCHECK(input_queue_.empty()); + + input_queue_.push_back( + std::make_pair(std::move(buffer), std::move(cb))); + + decoder_loop_->ExpectWork(); +} + + +void OHOSAudioDecoder::Reset(base::OnceClosure closure) { + ClearInputQueue(DecoderStatus::Codes::kAborted); + AudioDecoderAdapterCode respCode = audio_decoder_->FlushDecoder(); + bool success = respCode == AudioDecoderAdapterCode::DECODER_OK; + SetState(success ? READY : ERROR); + task_runner_->PostTask(FROM_HERE, std::move(closure)); +} + + +bool OHOSAudioDecoder::NeedsBitstreamConversion() const { + // An AAC stream needs to be converted as ADTS stream. + DCHECK_NE(config_.codec(), AudioCodec::kUnknown); + return config_.codec() == AudioCodec::kAAC; +} + +void OHOSAudioDecoder::OnInputDataQueued(bool success) { + if (input_queue_.front().first->end_of_stream() && success) + return; + // 排队成功，设置回调为ok + std::move(input_queue_.front().second) + .Run(success ? DecoderStatus::Codes::kOk : DecoderStatus::Codes::kFailed); + input_queue_.pop_front(); +} + +bool OHOSAudioDecoder::IsAnyInputPending() const { + // 非 Ready 状态肯定没有InputBuffer阻塞 + if (state_ != READY) { + return false; + } + return !input_queue_.empty(); +} + +OHOSAudioDecoderLoop::InputData OHOSAudioDecoder::ProvideInputData() { + //从输入队列input_queue_的前端获取一个DecoderBuffer对象的指针。input_queue_可能是一个存储待处理数据的队列。 + const DecoderBuffer* decoder_buffer = input_queue_.front().first.get(); + OHOSAudioDecoderLoop::InputData data; + //检查decoder_buffer是否表示流的结束。如果是，设置input_data.is_eos为true，表示输入数据是流的结束信号。 + if (decoder_buffer->end_of_stream()) { + data.is_eos = true; + } else { + + data.memory = static_cast(decoder_buffer->data()); + data.length = decoder_buffer->data_size(); + const DecryptConfig* decrypt_config = decoder_buffer->decrypt_config(); + if (decrypt_config) { + data.key_id = decrypt_config->key_id(); + data.iv = decrypt_config->iv(); + data.subsamples = decrypt_config->subsamples(); + data.encryption_scheme = decrypt_config->encryption_scheme(); + data.encryption_pattern = decrypt_config->encryption_pattern(); + } + data.presentation_time = decoder_buffer->timestamp(); + } + return data; +} + +bool OHOSAudioDecoder::OnDecodedEos(const OutputBufferData& out) { + if (!input_queue_.size() || !input_queue_.front().first->end_of_stream()) { + LOG(WARNING) << "OHOSAudioDecoder::OnDecodedEos: received unexpected eos"; + return false; + } + DCHECK_NE(state_, ERROR); + + // EOS 帧直接回调ok + std::move(input_queue_.front()).second.Run(DecoderStatus::Codes::kOk); + input_queue_.pop_front(); + return true; +} + +bool OHOSAudioDecoder::OnDecodedFrame(const OutputBufferData& out) { +// 校验buffer size 不为0，index有效 + DCHECK_NE(out.size, 0U); + DCHECK_NE(out.index, OHOSAudioDecoderLoop::kInvalidBufferIndex); + DCHECK(decoder_loop_); + // For proper |frame_count| calculation we need to use the actual number + // of channels which can be different from |config_| value. + DCHECK_GT(channel_count_, 0); + //声明一个变量 frame_count 用于存储帧数，初始值为1。 + size_t frame_count = 1; + scoped_refptr audio_buffer; + + //如果是直通模式（passthrough mode），则执行特定的处理。 + if (is_passthrough_) { + //创建一个比特流缓冲区。 + audio_buffer = AudioBuffer::CreateBitstreamBuffer( + sample_format_, channel_layout_, channel_count_, sample_rate_, + frame_count, out.size, pool_); + //从输出缓冲区复制数据。 + try { + std::copy(out.data_, audio_buffer->channel_data()[0], out.size); + } catch (const std::exception& e) { + LOG(ERROR) << "OHOSAudioDecoder::OnDecodedFrame copy frame data error. Exception caught: " << e.what() << std::endl; + audio_decoder_.ReleaseOutputBufferDec(out.index); + return false; + } + + // 根据不同的 codec 类型，计算帧数 + if (config_.codec() == AudioCodec::kAC3) { + frame_count = Ac3Util::ParseTotalAc3SampleCount( + audio_buffer->channel_data()[0], out.size); + } else if (config_.codec() == AudioCodec::kEAC3) { + frame_count = Ac3Util::ParseTotalEac3SampleCount( + audio_buffer->channel_data()[0], out.size); +#if BUILDFLAG(ENABLE_PLATFORM_DTS_AUDIO) + } else if (config_.codec() == AudioCodec::kDTS) { + frame_count = media::dts::ParseTotalSampleCount( + audio_buffer->channel_data()[0], out.size, AudioCodec::kDTS); + DVLOG(2) << ": DTS Frame Count = " << frame_count; +#endif // BUILDFLAG(ENABLE_PLATFORM_DTS_AUDIO) + } else { + NOTREACHED() << "Unsupported passthrough format."; + } + + // 通过新的 frame_count 创建buffer + // Create AudioOutput buffer based on current parameters. + audio_buffer = AudioBuffer::CreateBitstreamBuffer( + sample_format_, channel_layout_, channel_count_, sample_rate_, + frame_count, out.size, pool_); + } else { + // 安卓直接 计算帧数 + // Android MediaCodec can only output 16bit PCM audio. + const int bytes_per_frame = sizeof(uint16_t) * channel_count_; + frame_count = out.size / bytes_per_frame; + // 创建buffer + // Create AudioOutput buffer based on current parameters. + audio_buffer = AudioBuffer::CreateBuffer(sample_format_, channel_layout_, + channel_count_, sample_rate_, + frame_count, pool_); + } + + // Copy data into AudioBuffer. + CHECK_LE(out.size, audio_buffer->data_size()); + + // 取数据 + try { + std::copy(out.data_, audio_buffer->channel_data()[0], out.size); + } catch (const std::exception& e) { + LOG(ERROR) << "OHOSAudioDecoder::OnDecodedFrame copy frame data error. Exception caught: " << e.what() << std::endl; + audio_decoder_.ReleaseOutputBufferDec(out.index); + return false; + } + audio_decoder_.ReleaseOutputBufferDec(out.index); + + // Calculate and set buffer timestamp. + //这行代码检查 timestamp_helper_ 的基础时间戳是否为 kNoTimestamp（通常表示未设置或无效的时间戳）。如果是，则表示这是第一个缓冲区。 + + const bool first_buffer = timestamp_helper_->base_timestamp() == kNoTimestamp; + if (first_buffer) { + // Clamp the base timestamp to zero. + //设置 timestamp_helper_ 的基础时间戳。使用 std::max 函数确保基础时间戳不小于零（即 base::TimeDelta() 表示零时间差）。out.pts 是当前输出缓冲区的时间戳。 + timestamp_helper_->SetBaseTimestamp(std::max(base::TimeDelta(), out.pts)); + } + //设置音频缓冲区的时间戳。timestamp_helper_->GetTimestamp() 根据基础时间戳和已添加的帧数计算当前时间戳。 + audio_buffer->set_timestamp(timestamp_helper_->GetTimestamp()); + //在 timestamp_helper_ 中增加帧数。这用于更新时间戳计算，以便下一个缓冲区的时间戳是正确的。 + timestamp_helper_->AddFrames(frame_count); + //调用输出回调函数 output_cb_，并将音频缓冲区 audio_buffer 作为参数传递。这通常用于将解码后的音频数据传递给下一个处理阶段或最终的音频播放系统。 + // Call the |output_cb_|. + output_cb_.Run(audio_buffer); + + return true; +} + +void OHOSAudioDecoder::OnCodecLoopError() { + SetState(ERROR); + ClearInputQueue(DecoderStatus::Codes::kFailed); +} + +int32_t OHOSAudioDecoder::DequeueInputBuffer(InputBuffer& input) { + if (!input_buffer_queue_.empty()) { + uint32_t index = input_buffer_queue_.pop_front(); + input = InputBuffer(index, false); + return 0; + } + return -1; +} + +int32_t OHOSAudioDecoder::DequeueOutputBuffer(OutputBufferData& out) { + if (!output_buffer_queue_.empty()) { + out = output_buffer_queue_.pop_front(); + return 0; + } + return -1; +} + +} // namespace media \ No newline at end of file diff --git a/media/filters/ohos/ohos_audio_decoder.h b/media/filters/ohos/ohos_audio_decoder.h new file mode 100644 index 0000000000000000000000000000000000000000..a5c1d91dc2f617a46d885f640b601562add1cd52 --- /dev/null +++ b/media/filters/ohos/ohos_audio_decoder.h @@ -0,0 +1,158 @@ +#ifndef OHOS_AUDIO_DECODER_H +#define OHOS_AUDIO_DECODER_H +#include +#include +#include + +#include "base/containers/circular_deque.h" +#include "base/memory/raw_ptr.h" +#include "base/memory/weak_ptr.h" +#include "base/time/time.h" +#include "base/timer/timer.h" +#include "audio_codec_decoder_adapter.h" +#include "base/logging.h" +#include "media/base/audio_buffer.h" +#include "media/base/audio_decoder.h" +#include "media/base/audio_decoder_config.h" +#include "media/base/callback_registry.h" +#include "media/base/cdm_context.h" +#include "media/base/media_export.h" +#include "ohos_audio_decoder_loop.h" + +namespace media { +using namespace OHOS::NWeb; +class AudioTimestampHelper; + +class OHOSAudioDecoderFormat : public AudioDecoderFormatAdapter { +public: + OHOSAudioDecoderFormat() = default; + + int32_t GetSampleRate() override; + + int32_t GetChannelCount() override; + + int32_t GetBitRate() override; + + int32_t GetMaxInputSize() override; + + void SetSampleRate(int32_t sampleRate) override; + + void SetChannelCount(int32_t channelCount) override; + + void SetBitRate(int32_t bitRate) override; + + void SetMaxInputSize(int32_t maxInputSize) override; +private: + int32_t sample_rate_; + int32_t channel_count_; + int32_t bit_rate_; + int32_t max_input_size_; +}; + +class AudioDecoderCallback : public AudioDecoderCallbackAdapter { +public: + AudioDecoderCallback() = default; + + ~AudioDecoderCallback(); + + void OnError(int32_t errorCode) override; + + void OnOutputFormatChanged(const std::shared_ptr format) override; + + void OnInputBufferAvailable(uint32_t index) override; + + void OnOutputBufferAvailable(uint32_t index, uint8_t *bufferData, uint32_t size) override; +}; + + +class OHOSAudioDecoder : public AudioDecoder, + public OHOSAudioDecoderLoop::Client { +public: + enum State { + UNINITIALIZED, + WAITING_FOR_MEDIA_CRYPTO, + READY, + ERROR, + }; + // AudioDecoder implementation. + AudioDecoderType GetDecoderType() const override; + void Initialize(const AudioDecoderConfig& config, + CdmContext* cdm_context, + InitCB init_cb, + const OutputCB& output_cb, + const WaitingCB& waiting_cb) override; + void Decode(scoped_refptr buffer, DecodeCB decode_cb) override; + void Reset(base::OnceClosure closure) override; + bool NeedsBitstreamConversion() const override; + + // OHOSAudioDecoderLoop::Client implementation + bool IsAnyInputPending() const override; + OHOSAudioDecoderLoop::InputData ProvideInputData() override; + void OnInputDataQueued(bool success) override; + bool OnDecodedEos(const OutputBufferData& out) override; + bool OnDecodedFrame(const OutputBufferData& out) override; + void OnCodecLoopError() override; + int32_t DequeueInputBuffer() override; + int32_t DequeueOutputBuffer(OutputBufferData& out) override; + + void ClearInputQueue(DecoderStatus decode_status); + void SetState(State new_state); + void SetCdm(CdmContext* cdm_context, InitCB init_cb); + bool InitAudioDecoder(std::string mime_type); + bool CreateOhosDecoderLoop(); +private: + friend class AudioDecoderCallback; + + std::string mime_type_; + + State state_ = State::UNINITIALIZED; + + std::unique_ptr audio_decoder_ = nullptr; + + std::shared_ptr decoder_callback_ = nullptr; + + std::deque input_buffer_queue_; + + std::deque output_buffer_queue_; + + // Cached decoder config. + AudioDecoderConfig config_; + + // Indication to use passthrough decoder or not. + bool is_passthrough_; + + // The audio sample format of the audio decoder output. + SampleFormat sample_format_; + + // Actual channel count that comes from decoder may be different than config. + int channel_count_; + ChannelLayout channel_layout_; + + // Actual sample rate that comes from the decoder, may be different than + // config. + int sample_rate_; + + // Callback that delivers output frames. + OutputCB output_cb_; + + WaitingCB waiting_cb_; + + std::unique_ptr decoder_loop_; + + scoped_refptr task_runner_; + + using BufferCBPair = std::pair, DecodeCB>; + using InputQueue = base::circular_deque; + InputQueue input_queue_; + + std::unique_ptr timestamp_helper_; + + // To keep the CdmContext event callback registered. + std::unique_ptr event_cb_registration_; + + // Pool which helps avoid thrashing memory when returning audio buffers. + scoped_refptr pool_; + +}; + +} \ No newline at end of file diff --git a/media/filters/ohos/ohos_audio_decoder_loop.cc b/media/filters/ohos/ohos_audio_decoder_loop.cc new file mode 100644 index 0000000000000000000000000000000000000000..25d7c4e140696b22980eb8c528e1aea4b8071705 --- /dev/null +++ b/media/filters/ohos/ohos_audio_decoder_loop.cc @@ -0,0 +1,285 @@ +#include "ohos_audio_decoder_loop.h" +#include "base/functional/bind.h" +#include "base/functional/callback_helpers.h" +#include "base/logging.h" +#include "base/task/single_thread_task_runner.h" +#include "media/base/timestamp_constants.h" +namespace media { + namespace { + + constexpr base::TimeDelta kDecodePollDelay = base::Milliseconds(10); + // constexpr base::TimeDelta kNoWaitTimeout = base::Microseconds(0); + constexpr base::TimeDelta kIdleTimerTimeout = base::Seconds(1); + + } // namespace + +OHOSAudioDecoderLoop::InputData::InputData() {} + +OHOSAudioDecoderLoop::InputData::InputData(const InputData& other) + : memory(other.memory), + length(other.length), + key_id(other.key_id), + iv(other.iv), + subsamples(other.subsamples), + presentation_time(other.presentation_time), + is_eos(other.is_eos), + encryption_scheme(other.encryption_scheme) {} + +OHOSAudioDecoderLoop::InputData::~InputData() {} + +OHOSAudioDecoderLoop::OHOSAudioDecoderLoop( + Client* client, + std::unique_ptr audio_decoder, + scoped_refptr timer_task_runner, + bool disable_timer) + : state_(READY), + client_(client), + audio_decoder_(std::move(audio_decoder)), + pending_input_buf_index_(kInvalidBufferIndex), + disable_timer_(disable_timer) { + if (timer_task_runner) + io_timer_.SetTaskRunner(timer_task_runner); + // TODO(liberato): should this DCHECK? + if (audio_decoder_ == nullptr) + SetState(ERROR); +} + +OHOSAudioDecoderLoop::~OHOSAudioDecoderLoop() { + io_timer_.Stop(); +} + +// 有可用密钥，启动loop +void OHOSAudioDecoderLoop::OnKeyAdded() { + + ExpectWork(); +} + +bool OHOSAudioDecoderLoop::TryFlush() { + // We do not clear the input queue here. It depends on the caller. + // For decoder reset, then it is appropriate. Otherwise, the requests might + // simply be sent to us later, such as on a format change. + + // STATE_DRAINED seems like it allows flush, but it causes test failures. + // crbug.com/624878 + if (state_ == ERROR || state_ == DRAINED) + return false; + + // Actually try to flush! + io_timer_.Stop(); + + if (audio_decoder_->FlushDecoder() != AudioDecoderAdapterCode::DECODER_OK) { + // TODO(liberato): we might not want to notify the client about this. + SetState(ERROR); + return false; + } + + SetState(READY); + return true; +} + +void OHOSAudioDecoderLoop::ExpectWork() { + // Start / reset the timer, since we believe that progress can be made soon, + // even if not immediately. + ManageTimer(true); + DoPendingWork(); +} + +void OHOSAudioDecoderLoop::DoPendingWork() { + if (state_ == ERROR) + return; + + bool did_work = false, did_input = false, did_output = false; + do { + // 从内核取一个buffer数据送给 mediacodec + did_input = ProcessOneInputBuffer(); + // 从 mediacodec 拿一个解码后的 buffer 数据 + did_output = ProcessOneOutputBuffer(); + if (did_input || did_output) + did_work = true; + } while (did_input || did_output); + + ManageTimer(did_work); +} + +void OHOSAudioDecoderLoop::ManageTimer(bool did_work) { + if (disable_timer_) + return; + + bool should_be_running = true; + + // One might also use DefaultTickClock, but then ownership becomes harder. + // test_tick_clock_ 用于测试，可能是个null。这里获取当前时间 + base::TimeTicks now = base::TimeTicks::Now(); + //idle_time_begin_ 代表上一次我们触发定时器的时间 + if (did_work || idle_time_begin_ == base::TimeTicks()) { + idle_time_begin_ = now; + } else { + // Make sure that we have done work recently enough, else stop the timer. + // 检查自上次调用 DoPending 以来是否已经过去了足够的时间（超过 1s）。如果是，则设置 should_be_running 为 false。 + if (now - idle_time_begin_ > kIdleTimerTimeout) + should_be_running = false; + } + + if (should_be_running && !io_timer_.IsRunning()) { + // 这行代码启动定时器，延迟 10ms 后调用 MediaCodecLoop::DoPendingWork 方法。 + io_timer_.Start(FROM_HERE, kDecodePollDelay, this, + &OHOSAudioDecoderLoop::DoPendingWork); + } else if (!should_be_running && io_timer_.IsRunning()) { // 如果 should_be_running 为false，且timer 还在运行，关掉timer + io_timer_.Stop(); + } +} + +void OHOSAudioDecoderLoop::SetState(State new_state) { + const State old_state = state_; + state_ = new_state; + if (old_state != new_state && new_state == ERROR) + client_->OnCodecLoopError(); +} + +bool OHOSAudioDecoderLoop::ProcessOneInputBuffer() { + if (state_ != READY) + return false; + +/** +* 检查是否有有效的输入缓冲区待处理。如果没有挂起的输入缓冲区（pending_input_buf_index_ 等于 kInvalidBufferIndex）且客户端没有待处理的输入（MediaCodecAudioDecoder::IsAnyInputPending() 返回 false），则函数返回 false。 +*/ + // We can only queue a buffer if there is input from the client, or if we + // tried previously but had to wait for a key. In the latter case, MediaCodec + // already has the data. + if (pending_input_buf_index_ == kInvalidBufferIndex && + !client_->IsAnyInputPending()) { + return false; + } + +/** +* 调用 DequeueInputBuffer 方法从编解码器中出队一个输入缓冲区。这个方法可能会将状态设置为 STATE_ERROR。 +*/ + // DequeueInputBuffer() may set STATE_ERROR. + InputBuffer input_buffer = DequeueInputBuffer(); + + if (input_buffer.index == kInvalidBufferIndex) + return false; +/** +* 调用 EnqueueInputBuffer 方法将输入缓冲区入队到编解码器。这个方法可能会改变当前状态。 +*/ + // EnqueueInputBuffer() may change the state. + EnqueueInputBuffer(input_buffer); + return state_ != ERROR; +} + +OHOSAudioDecoderLoop::InputBuffer OHOSAudioDecoderLoop::DequeueInputBuffer() { + + // Do not dequeue a new input buffer if we failed with MEDIA_CODEC_NO_KEY. + // That status does not return the input buffer back to the pool of + // available input buffers. We have to reuse it later when calling + // MediaCodec's QueueSecureInputBuffer(). +/** + 检查是否有挂起的输入缓冲区索引。如果有，就创建一个InputBuffer对象并返回它，同时将pending_input_buf_index_重置为kInvalidBufferIndex。 +*/ + if (pending_input_buf_index_ != kInvalidBufferIndex) { + InputBuffer result(pending_input_buf_index_, true); + pending_input_buf_index_ = kInvalidBufferIndex; + return result; + } + + int input_buf_index = kInvalidBufferIndex; + + if (input_buf_index >= 0) { + input_buf_index = client_->DequeueInputBuffer(); + } + // false 是 is_pending 状态 + return InputBuffer(input_buf_index, false); +} + +void OHOSAudioDecoderLoop::EnqueueInputBuffer(const InputBuffer& input_buffer) { + DCHECK_NE(input_buffer.index, kInvalidBufferIndex); + + InputData input_data; +/** +* 个条件判断用于区分是否需要从客户端获取新的输入数据。如果是挂起的缓冲区（is_pending为true），则不需要再次复制数据；否则，调用客户端的ProvideInputData方法获取新的输入数据。 +*/ + if (input_buffer.is_pending) { + // A pending buffer is already filled with data, no need to copy it again. + input_data = pending_input_buf_data_; + } else { + input_data = client_->ProvideInputData(); + } +/** +* 如果输入数据表示流的结束（is_eos为true），则调用media_codec_的QueueEOS方法将结束信号入队，并将状态设置为STATE_DRAINING，然后通知客户端输入数据已入队。 +*/ + BufferFlag flag = BufferFlag::CODEC_BUFFER_FLAG_NONE; + if (input_data.is_eos) { + flag = BufferFlag::CODEC_BUFFER_FLAG_EOS; + SetState(DRAINING); + client_->OnInputDataQueued(true); + } + AudioDecoderAdapterCode code = audio_decoder_->QueueInputBufferDec( + input_buffer.index, input_data.memory, input_data.presentation_time.InMilliseconds(), input_data.length, flag); + + switch (code) { + case AudioDecoderAdapterCode::DECODER_RETRY: + // 表示缺少解密密钥，不调用完成回调，将缓冲区标记为挂起，保存输入数据，并通知客户端等待密钥，将状态设置为STATE_WAITING_FOR_KEY。 + // Do not call the completion cb here. It will be called when we retry + // after getting the key. + pending_input_buf_index_ = input_buffer.index; + pending_input_buf_data_ = input_data; + SetState(WAITING); + // Do not call OnInputDataQueued yet. + break; + + case AudioDecoderAdapterCode::DECODER_OK: + //表示成功入队，通知客户端入队成功。 + client_->OnInputDataQueued(true); + break; + + case AudioDecoderAdapterCode::DECODER_ERROR: + default: + // 表示发生了错误，记录错误日志，通知客户端入队失败，并将状态设置为STATE_ERROR。 + LOG(ERROR) << "DECODER_ERROR from QueueInputBuffer"; + client_->OnInputDataQueued(false); + SetState(ERROR); + break; + } +} + +bool OHOSAudioDecoderLoop::ProcessOneOutputBuffer() { + if (state_ == ERROR) + return false; + /** + * 调用media_codec_对象的DequeueOutputBuffer方法尝试获取一个输出缓冲区。该方法的参数用于接收缓冲区的索引、偏移量、大小、呈现时间戳（PTS）、是否是流结束（EOS）标志以及是否是关键帧。 +*/ + OutputBufferData out; + int32_t code = client_->DequeueOutputBuffer(out); + if (code == -1) { + return false; + } + + if (out.GetBufferFlag() == BufferFlag::CODEC_BUFFER_FLAG_EOS) { + // Set state STATE_DRAINED after we have received EOS frame at the + // output. media_decoder_job.cc says: once output EOS has occurred, we + // should not be asked to decode again. + /** + * 这意味着一旦在输出中接收到EOS帧，表明媒体流已经完全解码并且没有更多的数据需要处理，此时应该将解码器的状态设置为STATE_DRAINED。这个状态表示解码器已经排空（drained），即所有输入数据都已经被处理，并且所有输出数据都已经被提取。 + 这句话引用了media_decoder_job.cc文件中的说明，指出一旦输出EOS发生，就不应该再要求解码器进行解码。这通常意味着解码过程已经完成，解码器不应该再接收新的输入数据进行解码。 +*/ + DCHECK_EQ(state_, DRAINING); + SetState(DRAINED); + + DCHECK_NE(out.index_, kInvalidBufferIndex); + DCHECK(audio_decoder_); + + audio_decoder_->ReleaseOutputBufferDec(out.GetIndex()); + + if (!client_->OnDecodedEos(out)) + SetState(ERROR); + } else { + // 通知 audio decoder 处理解密后的一个buffer 数据 + if (!client_->OnDecodedFrame(out)) + SetState(ERROR); + } + return true; +} + + + +} // namespace media \ No newline at end of file diff --git a/media/filters/ohos/ohos_audio_decoder_loop.h b/media/filters/ohos/ohos_audio_decoder_loop.h new file mode 100644 index 0000000000000000000000000000000000000000..8648152af62371e709ed98e73472bf30247f062c --- /dev/null +++ b/media/filters/ohos/ohos_audio_decoder_loop.h @@ -0,0 +1,178 @@ +#ifndef OHOS_AUDIO_DECODER_LOOP_H +#define OHOS_AUDIO_DECODER_LOOP_H + +#include +#include +#include + +#include "base/memory/raw_ptr.h" +#include "base/memory/scoped_refptr.h" +#include "base/memory/weak_ptr.h" +#include "base/task/single_thread_task_runner.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" +#include "base/timer/timer.h" +#include "media/base/decoder_status.h" +#include "media/base/encryption_scheme.h" +#include "media/base/media_export.h" +#include "media/base/subsample_entry.h" +#include "media/base/waiting.h" +#include "third_party/abseil-cpp/absl/types/optional.h" +#include "audio_codec_decoder_adapter.h" + +namespace media { +using namespace OHOS::NWeb; + +class OutputBufferData { +public: + uint32_t index_; + uint8_t* data_; + uint32_t size_; + int64_t pts_; + BufferFlag flag_; + OutputBufferData() { + + } + + OutputBufferData(uint32_t index, uint8_t* data, uint32_t size, int64_t pts, BufferFlag flag) : + index_(index), data_(data), size_(size), pts_(pts), flag_(flag){} + + ~OutputBufferData() { + } + + uint32_t GetIndex() const { + return index_; + } + + uint8_t* GetData() const { + return data_; + } + + uint32_t GetSize() const { + return size_; + } + + int64_t GetPts() const { + return pts_; + } + + BufferFlag GetBufferFlag() const { + return flag_; + } +}; + +class OHOSAudioDecoderLoop { + public: + // Data that the client wants to put into an input buffer. + struct InputData { + InputData(); + InputData(const InputData&); + ~InputData(); + + uint8_t* memory = nullptr; + size_t length = 0; + + std::string key_id; + std::string iv; + std::vector subsamples; + + base::TimeDelta presentation_time; + + bool is_eos = false; + EncryptionScheme encryption_scheme = EncryptionScheme::kUnencrypted; + absl::optional encryption_pattern; + }; + + // Handy enum for "no buffer". + enum { kInvalidBufferIndex = -1 }; + + class Client { + public: + + virtual bool IsAnyInputPending() const = 0; + + virtual InputData ProvideInputData() = 0; + + virtual void OnInputDataQueued(bool success) = 0; + + virtual bool OnDecodedEos(const OutputBufferData& out) = 0; + + virtual bool OnDecodedFrame(const OutputBufferData& out) = 0; + + virtual void OnCodecLoopError() = 0; + + virtual int32_t DequeueInputBuffer() = 0; + + virtual int32_t DequeueOutputBuffer(OutputBufferData& out) = 0; + + protected: + virtual ~Client() {} + }; + + OHOSAudioDecoderLoop(Client* client, + std::unique_ptr audio_decoder, + scoped_refptr timer_task_runner, + bool disable_timer = false); + + OHOSAudioDecoderLoop(const OHOSAudioDecoderLoop&) = delete; + OHOSAudioDecoderLoop& operator=(const OHOSAudioDecoderLoop&) = delete; + + ~OHOSAudioDecoderLoop(); + + void ExpectWork(); + + bool TryFlush(); + + void OnKeyAdded(); + + protected: + enum State { + READY, + WAITING, + DRAINING, + DRAINED, + ERROR, + }; + + // Information about dequeued input buffer. + struct InputBuffer { + InputBuffer() {} + InputBuffer(uint32_t i, bool p) : index(i), is_pending(p) {} + + uint32_t index = kInvalidBufferIndex; + + bool is_pending = false; + }; + + void DoPendingWork(); + + bool ProcessOneInputBuffer(); + + void EnqueueInputBuffer(const InputBuffer& input_buffer); + + InputBuffer DequeueInputBuffer(); + + bool ProcessOneOutputBuffer(); + + void ManageTimer(bool start); + + void SetState(State new_state); + + State state_; + + raw_ptr client_; + + std::unique_ptr audio_decoder_; + + base::RepeatingTimer io_timer_; + + base::TimeTicks idle_time_begin_; + + uint32_t pending_input_buf_index_; + + InputData pending_input_buf_data_; + + const bool disable_timer_; +}; + +} // namespace media \ No newline at end of file