media/codec2/components/opus/C2SoftOpusEnc.cpp

/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//#define LOG_NDEBUG 0
#define LOG_TAG "C2SoftOpusEnc"
#include <utils/Log.h>

#include <C2PlatformSupport.h>
#include <SimpleC2Interface.h>
#include <media/stagefright/foundation/MediaDefs.h>
#include <media/stagefright/foundation/OpusHeader.h>
#include "C2SoftOpusEnc.h"

extern "C" {
    #include <opus.h>
    #include <opus_multistream.h>
}

#define DEFAULT_FRAME_DURATION_MS 20
namespace android {

namespace {

constexpr char COMPONENT_NAME[] = "c2.android.opus.encoder";

}  // namespace

static const int kMaxNumChannelsSupported = 2;

class C2SoftOpusEnc::IntfImpl : public SimpleInterface<void>::BaseParams {
public:
    explicit IntfImpl(const std::shared_ptr<C2ReflectorHelper> &helper)
        : SimpleInterface<void>::BaseParams(
                helper,
                COMPONENT_NAME,
                C2Component::KIND_ENCODER,
                C2Component::DOMAIN_AUDIO,
                MEDIA_MIMETYPE_AUDIO_OPUS) {
        noPrivateBuffers();
        noInputReferences();
        noOutputReferences();
        noInputLatency();
        noTimeStretch();
        setDerivedInstance(this);

        addParameter(
                DefineParam(mAttrib, C2_PARAMKEY_COMPONENT_ATTRIBUTES)
                .withConstValue(new C2ComponentAttributesSetting(
                    C2Component::ATTRIB_IS_TEMPORAL))
                .build());

        addParameter(
                DefineParam(mSampleRate, C2_PARAMKEY_SAMPLE_RATE)
                .withDefault(new C2StreamSampleRateInfo::input(0u, 48000))
                .withFields({C2F(mSampleRate, value).oneOf({
                    8000, 12000, 16000, 24000, 48000})})
                .withSetter((Setter<decltype(*mSampleRate)>::StrictValueWithNoDeps))
                .build());

        addParameter(
                DefineParam(mChannelCount, C2_PARAMKEY_CHANNEL_COUNT)
                .withDefault(new C2StreamChannelCountInfo::input(0u, 1))
                .withFields({C2F(mChannelCount, value).inRange(1, kMaxNumChannelsSupported)})
                .withSetter((Setter<decltype(*mChannelCount)>::StrictValueWithNoDeps))
                .build());

        addParameter(
                DefineParam(mBitrate, C2_PARAMKEY_BITRATE)
                .withDefault(new C2StreamBitrateInfo::output(0u, 128000))
                .withFields({C2F(mBitrate, value).inRange(500, 512000)})
                .withSetter(Setter<decltype(*mBitrate)>::NonStrictValueWithNoDeps)
                .build());

        addParameter(
                DefineParam(mComplexity, C2_PARAMKEY_COMPLEXITY)
                .withDefault(new C2StreamComplexityTuning::output(0u, 10))
                .withFields({C2F(mComplexity, value).inRange(1, 10)})
                .withSetter(Setter<decltype(*mComplexity)>::NonStrictValueWithNoDeps)
                .build());

        addParameter(
                DefineParam(mInputMaxBufSize, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE)
                .withConstValue(new C2StreamMaxBufferSizeInfo::input(0u, 3840))
                .build());
    }

    uint32_t getSampleRate() const { return mSampleRate->value; }
    uint32_t getChannelCount() const { return mChannelCount->value; }
    uint32_t getBitrate() const { return mBitrate->value; }
    uint32_t getComplexity() const { return mComplexity->value; }

private:
    std::shared_ptr<C2StreamSampleRateInfo::input> mSampleRate;
    std::shared_ptr<C2StreamChannelCountInfo::input> mChannelCount;
    std::shared_ptr<C2StreamBitrateInfo::output> mBitrate;
    std::shared_ptr<C2StreamComplexityTuning::output> mComplexity;
    std::shared_ptr<C2StreamMaxBufferSizeInfo::input> mInputMaxBufSize;
};

C2SoftOpusEnc::C2SoftOpusEnc(const char* name, c2_node_id_t id,
                               const std::shared_ptr<IntfImpl>& intfImpl)
    : SimpleC2Component(
          std::make_shared<SimpleInterface<IntfImpl>>(name, id, intfImpl)),
      mIntf(intfImpl),
      mOutputBlock(nullptr),
      mEncoder(nullptr),
      mInputBufferPcm16(nullptr),
      mOutIndex(0u) {
}

C2SoftOpusEnc::~C2SoftOpusEnc() {
    onRelease();
}

c2_status_t C2SoftOpusEnc::onInit() {
    return initEncoder();
}

c2_status_t C2SoftOpusEnc::configureEncoder() {
    static const unsigned char mono_mapping[256] = {0};
    static const unsigned char stereo_mapping[256] = {0, 1};
    mSampleRate = mIntf->getSampleRate();
    mChannelCount = mIntf->getChannelCount();
    uint32_t bitrate = mIntf->getBitrate();
    int complexity = mIntf->getComplexity();
    mNumSamplesPerFrame = mSampleRate / (1000 / mFrameDurationMs);
    mNumPcmBytesPerInputFrame =
        mChannelCount * mNumSamplesPerFrame * sizeof(int16_t);
    int err = C2_OK;

    const unsigned char* mapping;
    if (mChannelCount == 1) {
        mapping = mono_mapping;
    } else if (mChannelCount == 2) {
        mapping = stereo_mapping;
    } else {
        ALOGE("Number of channels (%d) is not supported", mChannelCount);
        return C2_BAD_VALUE;
    }

    if (mEncoder != nullptr) {
        opus_multistream_encoder_destroy(mEncoder);
    }

    mEncoder = opus_multistream_encoder_create(mSampleRate, mChannelCount,
        1, mChannelCount - 1, mapping, OPUS_APPLICATION_AUDIO, &err);
    if (err) {
        ALOGE("Could not create libopus encoder. Error code: %i", err);
        return C2_CORRUPTED;
    }

    // Complexity
    if (opus_multistream_encoder_ctl(
            mEncoder, OPUS_SET_COMPLEXITY(complexity)) != OPUS_OK) {
        ALOGE("failed to set complexity");
        return C2_BAD_VALUE;
    }

    // DTX
    if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_DTX(0) != OPUS_OK)) {
        ALOGE("failed to set dtx");
        return C2_BAD_VALUE;
    }

    // Application
    if (opus_multistream_encoder_ctl(mEncoder,
            OPUS_SET_APPLICATION(OPUS_APPLICATION_AUDIO)) != OPUS_OK) {
        ALOGE("failed to set application");
        return C2_BAD_VALUE;
    }

    // Signal type
    if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_SIGNAL(OPUS_AUTO)) !=
        OPUS_OK) {
        ALOGE("failed to set signal");
        return C2_BAD_VALUE;
    }

    // Constrained VBR
    if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_VBR(1) != OPUS_OK)) {
        ALOGE("failed to set vbr type");
        return C2_BAD_VALUE;
    }
    if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_VBR_CONSTRAINT(1) !=
            OPUS_OK)) {
        ALOGE("failed to set vbr constraint");
        return C2_BAD_VALUE;
    }

    // Bitrate
    if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_BITRATE(bitrate)) !=
            OPUS_OK) {
        ALOGE("failed to set bitrate");
        return C2_BAD_VALUE;
    }

    // Set seek preroll to 80 ms
    mSeekPreRoll = 80000000;
    return C2_OK;
}

c2_status_t C2SoftOpusEnc::initEncoder() {
    mSignalledEos = false;
    mSignalledError = false;
    mHeaderGenerated = false;
    mIsFirstFrame = true;
    mEncoderFlushed = false;
    mBufferAvailable = false;
    mAnchorTimeStamp = 0ull;
    mProcessedSamples = 0;
    mFilledLen = 0;
    mFrameDurationMs = DEFAULT_FRAME_DURATION_MS;
    if (!mInputBufferPcm16) {
        mInputBufferPcm16 =
            (int16_t*)malloc(kFrameSize * kMaxNumChannels * sizeof(int16_t));
    }
    if (!mInputBufferPcm16) return C2_NO_MEMORY;

    /* Default Configurations */
    c2_status_t status = configureEncoder();
    return status;
}

c2_status_t C2SoftOpusEnc::onStop() {
    mSignalledEos = false;
    mSignalledError = false;
    mIsFirstFrame = true;
    mEncoderFlushed = false;
    mBufferAvailable = false;
    mAnchorTimeStamp = 0ull;
    mProcessedSamples = 0u;
    mFilledLen = 0;
    if (mEncoder) {
        int status = opus_multistream_encoder_ctl(mEncoder, OPUS_RESET_STATE);
        if (status != OPUS_OK) {
            ALOGE("OPUS_RESET_STATE failed status = %s", opus_strerror(status));
            mSignalledError = true;
            return C2_CORRUPTED;
        }
    }
    if (mOutputBlock) mOutputBlock.reset();
    mOutputBlock = nullptr;

    return C2_OK;
}

void C2SoftOpusEnc::onReset() {
    (void)onStop();
}

void C2SoftOpusEnc::onRelease() {
    (void)onStop();
    if (mInputBufferPcm16) {
        free(mInputBufferPcm16);
        mInputBufferPcm16 = nullptr;
    }
    if (mEncoder) {
        opus_multistream_encoder_destroy(mEncoder);
        mEncoder = nullptr;
    }
}

c2_status_t C2SoftOpusEnc::onFlush_sm() {
    return onStop();
}

// Drain the encoder to get last frames (if any)
int C2SoftOpusEnc::drainEncoder(uint8_t* outPtr) {
    memset((uint8_t *)mInputBufferPcm16 + mFilledLen, 0,
        (mNumPcmBytesPerInputFrame - mFilledLen));
    int encodedBytes = opus_multistream_encode(
        mEncoder, mInputBufferPcm16, mNumSamplesPerFrame, outPtr, kMaxPayload);
    if (encodedBytes > mOutputBlock->capacity()) {
        ALOGE("not enough space left to write encoded data, dropping %d bytes",
              mBytesEncoded);
        // a fatal error would stop the encoding
        return -1;
    }
    ALOGV("encoded %i Opus bytes from %zu PCM bytes", encodedBytes,
          mNumPcmBytesPerInputFrame);
    mEncoderFlushed = true;
    mFilledLen = 0;
    return encodedBytes;
}

void C2SoftOpusEnc::process(const std::unique_ptr<C2Work>& work,
                            const std::shared_ptr<C2BlockPool>& pool) {
    // Initialize output work
    work->result = C2_OK;
    work->workletsProcessed = 1u;
    work->worklets.front()->output.flags = work->input.flags;

    if (mSignalledError || mSignalledEos) {
        work->result = C2_BAD_VALUE;
        return;
    }

    bool eos = (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) != 0;
    C2ReadView rView = mDummyReadView;
    size_t inOffset = 0u;
    size_t inSize = 0u;
    c2_status_t err = C2_OK;
    if (!work->input.buffers.empty()) {
        rView =
            work->input.buffers[0]->data().linearBlocks().front().map().get();
        inSize = rView.capacity();
        if (inSize && rView.error()) {
            ALOGE("read view map failed %d", rView.error());
            work->result = C2_CORRUPTED;
            return;
        }
    }

    ALOGV("in buffer attr. size %zu timestamp %d frameindex %d, flags %x",
          inSize, (int)work->input.ordinal.timestamp.peeku(),
          (int)work->input.ordinal.frameIndex.peeku(), work->input.flags);

    if (!mEncoder) {
        if (initEncoder() != C2_OK) {
            ALOGE("initEncoder failed with status %d", err);
            work->result = err;
            mSignalledError = true;
            return;
        }
    }
    if (mIsFirstFrame && inSize > 0) {
        mAnchorTimeStamp = work->input.ordinal.timestamp.peekull();
        mIsFirstFrame = false;
    }

    C2MemoryUsage usage = {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
    err = pool->fetchLinearBlock(kMaxPayload, usage, &mOutputBlock);
    if (err != C2_OK) {
        ALOGE("fetchLinearBlock for Output failed with status %d", err);
        work->result = C2_NO_MEMORY;
        return;
    }

    C2WriteView wView = mOutputBlock->map().get();
    if (wView.error()) {
        ALOGE("write view map failed %d", wView.error());
        work->result = C2_CORRUPTED;
        mOutputBlock.reset();
        return;
    }

    size_t inPos = 0;
    size_t processSize = 0;
    mBytesEncoded = 0;
    uint64_t outTimeStamp = 0u;
    std::shared_ptr<C2Buffer> buffer;
    uint64_t inputIndex = work->input.ordinal.frameIndex.peeku();
    const uint8_t* inPtr = rView.data() + inOffset;

    class FillWork {
    public:
        FillWork(uint32_t flags, C2WorkOrdinalStruct ordinal,
                 const std::shared_ptr<C2Buffer> &buffer)
            : mFlags(flags), mOrdinal(ordinal), mBuffer(buffer) {
        }
        ~FillWork() = default;

        void operator()(const std::unique_ptr<C2Work>& work) {
            work->worklets.front()->output.flags = (C2FrameData::flags_t)mFlags;
            work->worklets.front()->output.buffers.clear();
            work->worklets.front()->output.ordinal = mOrdinal;
            work->workletsProcessed = 1u;
            work->result = C2_OK;
            if (mBuffer) {
                work->worklets.front()->output.buffers.push_back(mBuffer);
            }
            ALOGV("timestamp = %lld, index = %lld, w/%s buffer",
                  mOrdinal.timestamp.peekll(),
                  mOrdinal.frameIndex.peekll(),
                  mBuffer ? "" : "o");
        }

    private:
        const uint32_t mFlags;
        const C2WorkOrdinalStruct mOrdinal;
        const std::shared_ptr<C2Buffer> mBuffer;
    };

    C2WorkOrdinalStruct outOrdinal = work->input.ordinal;

    if (!mHeaderGenerated) {
        uint8_t header[AOPUS_UNIFIED_CSD_MAXSIZE];
        memset(header, 0, sizeof(header));

        // Get codecDelay
        int32_t lookahead;
        if (opus_multistream_encoder_ctl(mEncoder, OPUS_GET_LOOKAHEAD(&lookahead)) !=
                OPUS_OK) {
            ALOGE("failed to get lookahead");
            mSignalledError = true;
            work->result = C2_CORRUPTED;
            return;
        }
        mCodecDelay = lookahead * 1000000000ll / mSampleRate;

        OpusHeader opusHeader;
        memset(&opusHeader, 0, sizeof(opusHeader));
        opusHeader.channels = mChannelCount;
        opusHeader.num_streams = mChannelCount;
        opusHeader.num_coupled = 0;
        opusHeader.channel_mapping = ((mChannelCount > 8) ? 255 : (mChannelCount > 2));
        opusHeader.gain_db = 0;
        opusHeader.skip_samples = lookahead;
        int headerLen = WriteOpusHeaders(opusHeader, mSampleRate, header,
            sizeof(header), mCodecDelay, mSeekPreRoll);

        std::unique_ptr<C2StreamInitDataInfo::output> csd =
            C2StreamInitDataInfo::output::AllocUnique(headerLen, 0u);
        if (!csd) {
            ALOGE("CSD allocation failed");
            mSignalledError = true;
            work->result = C2_NO_MEMORY;
            return;
        }
        ALOGV("put csd, %d bytes", headerLen);
        memcpy(csd->m.value, header, headerLen);
        work->worklets.front()->output.configUpdate.push_back(std::move(csd));
        mHeaderGenerated = true;
    }

    /*
     * For buffer size which is not a multiple of mNumPcmBytesPerInputFrame, we will
     * accumulate the input and keep it. Once the input is filled with expected number
     * of bytes, we will send it to encoder. mFilledLen manages the bytes of input yet
     * to be processed. The next call will fill mNumPcmBytesPerInputFrame - mFilledLen
     * bytes to input and send it to the encoder.
     */
    while (inPos < inSize) {
        const uint8_t* pcmBytes = inPtr + inPos;
        int filledSamples = mFilledLen / sizeof(int16_t);
        if ((inPos + (mNumPcmBytesPerInputFrame - mFilledLen)) <= inSize) {
            processSize = mNumPcmBytesPerInputFrame - mFilledLen;
            mBufferAvailable = true;
        } else {
            processSize = inSize - inPos;
            mBufferAvailable = false;
            if (eos) {
                memset(mInputBufferPcm16 + filledSamples, 0,
                       (mNumPcmBytesPerInputFrame - mFilledLen));
                mBufferAvailable = true;
            }
        }
        const unsigned nInputSamples = processSize / sizeof(int16_t);

        for (unsigned i = 0; i < nInputSamples; i++) {
            int32_t data = pcmBytes[2 * i + 1] << 8 | pcmBytes[2 * i];
            data = ((data & 0xFFFF) ^ 0x8000) - 0x8000;
            mInputBufferPcm16[i + filledSamples] = data;
        }
        inPos += processSize;
        mFilledLen += processSize;
        if (!mBufferAvailable) break;
        uint8_t* outPtr = wView.data() + mBytesEncoded;
        int encodedBytes =
            opus_multistream_encode(mEncoder, mInputBufferPcm16,
                                    mNumSamplesPerFrame, outPtr, kMaxPayload - mBytesEncoded);
        ALOGV("encoded %i Opus bytes from %zu PCM bytes", encodedBytes,
              processSize);

        if (encodedBytes < 0 || encodedBytes > (kMaxPayload - mBytesEncoded)) {
            ALOGE("opus_encode failed, encodedBytes : %d", encodedBytes);
            mSignalledError = true;
            work->result = C2_CORRUPTED;
            return;
        }
        if (buffer) {
            outOrdinal.frameIndex = mOutIndex++;
            outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
            cloneAndSend(
                inputIndex, work,
                FillWork(C2FrameData::FLAG_INCOMPLETE, outOrdinal, buffer));
            buffer.reset();
        }
        if (encodedBytes > 0) {
            buffer =
                createLinearBuffer(mOutputBlock, mBytesEncoded, encodedBytes);
        }
        mBytesEncoded += encodedBytes;
        mProcessedSamples += (filledSamples + nInputSamples);
        outTimeStamp =
            mProcessedSamples * 1000000ll / mChannelCount / mSampleRate;
        if ((processSize + mFilledLen) < mNumPcmBytesPerInputFrame)
            mEncoderFlushed = true;
        mFilledLen = 0;
    }

    uint32_t flags = 0;
    if (eos) {
        ALOGV("signalled eos");
        mSignalledEos = true;
        if (!mEncoderFlushed) {
            if (buffer) {
                outOrdinal.frameIndex = mOutIndex++;
                outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
                cloneAndSend(
                    inputIndex, work,
                    FillWork(C2FrameData::FLAG_INCOMPLETE, outOrdinal, buffer));
                buffer.reset();
            }
            // drain the encoder for last buffer
            drainInternal(pool, work);
        }
        flags = C2FrameData::FLAG_END_OF_STREAM;
    }
    if (buffer) {
        outOrdinal.frameIndex = mOutIndex++;
        outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
        FillWork((C2FrameData::flags_t)(flags), outOrdinal, buffer)(work);
        buffer.reset();
    }
    mOutputBlock = nullptr;
}

c2_status_t C2SoftOpusEnc::drainInternal(
        const std::shared_ptr<C2BlockPool>& pool,
        const std::unique_ptr<C2Work>& work) {
    mBytesEncoded = 0;
    std::shared_ptr<C2Buffer> buffer = nullptr;
    C2WorkOrdinalStruct outOrdinal = work->input.ordinal;
    bool eos = (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) != 0;

    C2MemoryUsage usage = {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
    c2_status_t err = pool->fetchLinearBlock(kMaxPayload, usage, &mOutputBlock);
    if (err != C2_OK) {
        ALOGE("fetchLinearBlock for Output failed with status %d", err);
        return C2_NO_MEMORY;
    }

    C2WriteView wView = mOutputBlock->map().get();
    if (wView.error()) {
        ALOGE("write view map failed %d", wView.error());
        mOutputBlock.reset();
        return C2_CORRUPTED;
    }

    int encBytes = drainEncoder(wView.data());
    if (encBytes > 0) mBytesEncoded += encBytes;
    if (mBytesEncoded > 0) {
        buffer = createLinearBuffer(mOutputBlock, 0, mBytesEncoded);
        mOutputBlock.reset();
    }
    mProcessedSamples += (mNumPcmBytesPerInputFrame / sizeof(int16_t));
    uint64_t outTimeStamp =
        mProcessedSamples * 1000000ll / mChannelCount / mSampleRate;
    outOrdinal.frameIndex = mOutIndex++;
    outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
    work->worklets.front()->output.flags =
        (C2FrameData::flags_t)(eos ? C2FrameData::FLAG_END_OF_STREAM : 0);
    work->worklets.front()->output.buffers.clear();
    work->worklets.front()->output.ordinal = outOrdinal;
    work->workletsProcessed = 1u;
    work->result = C2_OK;
    if (buffer) {
        work->worklets.front()->output.buffers.push_back(buffer);
    }
    mOutputBlock = nullptr;
    return C2_OK;
}

c2_status_t C2SoftOpusEnc::drain(uint32_t drainMode,
                                 const std::shared_ptr<C2BlockPool>& pool) {
    if (drainMode == NO_DRAIN) {
        ALOGW("drain with NO_DRAIN: no-op");
        return C2_OK;
    }
    if (drainMode == DRAIN_CHAIN) {
        ALOGW("DRAIN_CHAIN not supported");
        return C2_OMITTED;
    }
    mIsFirstFrame = true;
    mAnchorTimeStamp = 0ull;
    mProcessedSamples = 0u;
    return drainInternal(pool, nullptr);
}

class C2SoftOpusEncFactory : public C2ComponentFactory {
public:
    C2SoftOpusEncFactory()
        : mHelper(std::static_pointer_cast<C2ReflectorHelper>(
              GetCodec2PlatformComponentStore()->getParamReflector())) {}

    virtual c2_status_t createComponent(
        c2_node_id_t id, std::shared_ptr<C2Component>* const component,
        std::function<void(C2Component*)> deleter) override {
        *component = std::shared_ptr<C2Component>(
            new C2SoftOpusEnc(
                COMPONENT_NAME, id,
                std::make_shared<C2SoftOpusEnc::IntfImpl>(mHelper)),
            deleter);
        return C2_OK;
    }

    virtual c2_status_t createInterface(
        c2_node_id_t id, std::shared_ptr<C2ComponentInterface>* const interface,
        std::function<void(C2ComponentInterface*)> deleter) override {
        *interface = std::shared_ptr<C2ComponentInterface>(
            new SimpleInterface<C2SoftOpusEnc::IntfImpl>(
                COMPONENT_NAME, id,
                std::make_shared<C2SoftOpusEnc::IntfImpl>(mHelper)),
            deleter);
        return C2_OK;
    }

    virtual ~C2SoftOpusEncFactory() override = default;
private:
    std::shared_ptr<C2ReflectorHelper> mHelper;
};

}  // namespace android

extern "C" ::C2ComponentFactory* CreateCodec2Factory() {
    ALOGV("in %s", __func__);
    return new ::android::C2SoftOpusEncFactory();
}

extern "C" void DestroyCodec2Factory(::C2ComponentFactory* factory) {
    ALOGV("in %s", __func__);
    delete factory;
}