MP3 数据解码

Reference

• Rhythm Game sample

---

加载 assets 中的 MP3 文件

AAssetDataSource.cpp

#include <utils/logging.h>
#include <oboe/Oboe.h>

#include "AAssetDataSource.h"

#if !defined(USE_FFMPEG)
#error USE_FFMPEG should be defined in app.gradle
#endif

#if USE_FFMPEG == 1
#include "FFMpegExtractor.h"
#else
#include "NDKExtractor.h"
#endif

constexpr int kMaxCompressionRatio{12};

AAssetDataSource *AAssetDataSource::newFromCompressedAsset(AAssetManager &assetManager, const char *filename, const AudioProperties targetProperties) {
    AAsset *asset = AAssetManager_open(&assetManager, filename, AASSET_MODE_UNKNOWN);
    if (!asset) {
        LOGE("Failed to open asset %s", filename);
        return nullptr;
    }

    off_t assetSize = AAsset_getLength(asset);
    LOGD("Opened %s, size %ld", filename, assetSize);

    // Allocate memory to store the decompressed audio.
    // We don't know the exact size of the decoded data until after decoding so we make an assumption about the maximum compression ratio and the decoded sample format (float for FFmpeg, int16 for NDK).
    // 分配内存来存储解压后的音频。
    // 我们直到解码后才知道解码数据的确切大小，因此我们对最大压缩比和解码样本格式（FFmpeg 为 float，NDK 为 int16）进行假设。
#if USE_FFMPEG == true
    const long maximumDataSizeInBytes = kMaxCompressionRatio * assetSize * sizeof(float);
    auto decodedData = new uint8_t[maximumDataSizeInBytes];

    int64_t bytesDecoded = FFMpegExtractor::decode(asset, decodedData, targetProperties);
    auto numSamples = bytesDecoded / sizeof(float);
#else
    const long maximumDataSizeInBytes = kMaxCompressionRatio * assetSize * sizeof(int16_t);
    auto decodedData = new uint8_t[maximumDataSizeInBytes];

    int64_t bytesDecoded = NDKExtractor::decode(asset, decodedData, targetProperties);
    auto numSamples = bytesDecoded / sizeof(int16_t);
#endif

    // Now we know the exact number of samples we can create a float array to hold the audio data
    // 现在我们知道样本的确切数量，我们可以创建一个浮点数组来保存音频数据
    auto outputBuffer = std::make_unique<float[]>(numSamples);

#if USE_FFMPEG == 1
    memcpy(outputBuffer.get(), decodedData, (size_t)bytesDecoded);
#else
    // The NDK decoder can only decode to int16, we need to convert to floats
    oboe::convertPcm16ToFloat(reinterpret_cast<int16_t *>(decodedData), outputBuffer.get(), bytesDecoded / sizeof(int16_t));
#endif

    delete[] decodedData;
    AAsset_close(asset);

    return new AAssetDataSource(std::move(outputBuffer), numSamples, targetProperties);
}

---

解码 MP3 数据

使用 NDK 解码

参考自定义媒体组件，数据解析包括提取器和解码器。

提取器进程会自动从 Google 提供的 APEX 软件包和 /system/lib[64]/extractors/ 加载提取器插件，如果默认的媒体提取器无法满足您的需求，您可以在 /system/lib[64]/extractors/ 中放置自定义提取器插件。

解码器会通过media.player服务查找对应mime的解码器。

运行时，获取解码器的输入缓冲队列和输出缓冲队列，将提取器处理后的数据放入输入缓冲队列，由解码器处理后的数据会被放入输出缓冲队列，从输出缓冲队列中取出的数据就是可以用于播放的数据。

NDKExtractor.cpp

#include <sys/types.h>

#include <cstring>

#include <media/NdkMediaExtractor.h>
#include <utils/logging.h>
#include <cinttypes>

#include "NDKExtractor.h"

int32_t NDKExtractor::decode(AAsset *asset, uint8_t *targetData, AudioProperties targetProperties) {
    LOGD("Using NDK decoder");

    // open asset as file descriptor
    // 打开资产作为文件描述符
    off_t start, length;
    int fd = AAsset_openFileDescriptor(asset, &start, &length);

    // Extract the audio frames
    // 提取音频帧
    AMediaExtractor *extractor = AMediaExtractor_new();
    media_status_t amresult = AMediaExtractor_setDataSourceFd(extractor, fd, static_cast<off64_t>(start), static_cast<off64_t>(length));
    if (amresult != AMEDIA_OK) {
        LOGE("Error setting extractor data source, err %d", amresult);
        return 0;
    }

    // Specify our desired output format by creating it from our source
    // 通过从我们的源创建它来指定我们想要的输出格式
    AMediaFormat *format = AMediaExtractor_getTrackFormat(extractor, 0);

    int32_t sampleRate;
    if (AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_SAMPLE_RATE, &sampleRate)) {
        LOGD("Source sample rate %d", sampleRate);
        if (sampleRate != targetProperties.sampleRate) {
            LOGE("Input (%d) and output (%d) sample rates do not match. NDK decoder does not support resampling.", sampleRate, targetProperties.sampleRate);
            return 0;
        }
    } else {
        LOGE("Failed to get sample rate");
        return 0;
    };

    int32_t channelCount;
    if (AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_CHANNEL_COUNT, &channelCount)) {
        LOGD("Got channel count %d", channelCount);
        if (channelCount != targetProperties.channelCount) {
            LOGE("NDK decoder does not support different input (%d) and output (%d) channel counts", channelCount, targetProperties.channelCount);
        }
    } else {
        LOGE("Failed to get channel count");
        return 0;
    }

    const char *formatStr = AMediaFormat_toString(format);
    LOGD("Output format %s", formatStr);

    const char *mimeType;
    if (AMediaFormat_getString(format, AMEDIAFORMAT_KEY_MIME, &mimeType)) {
        LOGD("Got mime type %s", mimeType);
    } else {
        LOGE("Failed to get mime type");
        return 0;
    }

    // Obtain the correct decoder
    // 获取正确的解码器
    AMediaCodec *codec = nullptr;
    AMediaExtractor_selectTrack(extractor, 0);
    codec = AMediaCodec_createDecoderByType(mimeType);
    AMediaCodec_configure(codec, format, nullptr, nullptr, 0);
    AMediaCodec_start(codec);

    /**
     * 开始解码
     */
    bool isExtracting = true;
    bool isDecoding = true;
    int32_t bytesWritten = 0;
    while (isExtracting || isDecoding) {
        if (isExtracting) {
            // Obtain the index of the next available input buffer
            // 获取下一个可用输入缓冲区的索引
            ssize_t inputIndex = AMediaCodec_dequeueInputBuffer(codec, 2000);
            //LOGV("Got input buffer %d", inputIndex);

            // The input index acts as a status if its negative
            // 输入索引作为一个状态，如果它是负的
            if (inputIndex < 0) {
                if (inputIndex == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
                    // LOGV("Codec.dequeueInputBuffer try again later");
                } else {
                    LOGE("Codec.dequeueInputBuffer unknown error status");
                }
            } else {
                // Obtain the actual buffer and read the encoded data into it
                // 获取实际缓冲区并将编码数据读入其中
                size_t inputSize;
                uint8_t *inputBuffer = AMediaCodec_getInputBuffer(codec, inputIndex, &inputSize);
                //LOGV("Sample size is: %d", inputSize);

                ssize_t sampleSize = AMediaExtractor_readSampleData(extractor, inputBuffer, inputSize);
                auto presentationTimeUs = AMediaExtractor_getSampleTime(extractor);
                if (sampleSize > 0) {
                    // Enqueue the encoded data
                    // 将编码数据入队
                    AMediaCodec_queueInputBuffer(codec, inputIndex, 0, sampleSize, presentationTimeUs, 0);
                    AMediaExtractor_advance(extractor);
                } else {
                    LOGD("End of extractor data stream");
                    isExtracting = false;

                    // We need to tell the codec that we've reached the end of the stream
                    // 我们需要告诉编解码器我们已经到达流的末尾
                    AMediaCodec_queueInputBuffer(codec, inputIndex, 0, 0, presentationTimeUs, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM);
                }
            }
        }

        if (isDecoding) {
            // Dequeue the decoded data
            // 将解码数据出列
            AMediaCodecBufferInfo info;
            ssize_t outputIndex = AMediaCodec_dequeueOutputBuffer(codec, &info, 0);
            if (outputIndex >= 0) {
                // Check whether this is set earlier
                if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
                    LOGD("Reached end of decoding stream");
                    isDecoding = false;
                }

                // Valid index, acquire buffer
                size_t outputSize;
                uint8_t *outputBuffer = AMediaCodec_getOutputBuffer(codec, outputIndex, &outputSize);

                /*LOGV("Got output buffer index %d, buffer size: %d, info size: %d writing to pcm index %d",
                     outputIndex,
                     outputSize,
                     info.size,
                     m_writeIndex);*/

                // copy the data out of the buffer
                // 从缓冲区复制数据
                memcpy(targetData + bytesWritten, outputBuffer, info.size);
                bytesWritten += info.size;
                AMediaCodec_releaseOutputBuffer(codec, outputIndex, false);
            } else {
                // The outputIndex doubles as a status return if its value is < 0
                // 如果其值 < 0，则 outputIndex 作为状态返回值加倍
                switch (outputIndex) {
                    case AMEDIACODEC_INFO_TRY_AGAIN_LATER:
                        LOGD("dequeueOutputBuffer: try again later");
                        break;

                    case AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED:
                        LOGD("dequeueOutputBuffer: output buffers changed");
                        break;

                    case AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED:
                        LOGD("dequeueOutputBuffer: output outputFormat changed");
                        format = AMediaCodec_getOutputFormat(codec);
                        LOGD("outputFormat changed to: %s", AMediaFormat_toString(format));
                        break;
                }
            }
        }
    }

    // Clean up
    AMediaFormat_delete(format);
    AMediaCodec_delete(codec);
    AMediaExtractor_delete(extractor);

    return bytesWritten;
}

时序图：

涉及的 NDK 中的源代码：

• NdkMediaExtractor.cpp
• NuMediaExtractor.cpp
• FileSource.cpp
• MediaExtractorFactory.cpp
• MediaExtractorPluginApi.h
• MediaExtractor.cpp
• InterfaceUtils.cpp
• RemoteMediaExtractor.cpp
• Utils.cpp
• NdkMediaFormat.cpp
• NdkMediaCodec.cpp
• MediaCodec.h
• MediaCodec.cpp
• MediaCodecList.cpp
• AHandler.cpp
• ALooper.h
• ALooper.cpp
• AMessage.h
• AMessage.cpp

使用 FFmpeg 解码

// TODO