libvideoeditor/lvpp/NativeWindowRenderer.cpp

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define LOG_TAG "NativeWindowRenderer"
#include "NativeWindowRenderer.h"

#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <cutils/log.h>
#include <gui/SurfaceTexture.h>
#include <gui/SurfaceTextureClient.h>
#include <stagefright/MediaBuffer.h>
#include <stagefright/MediaDebug.h>
#include <stagefright/MetaData.h>
#include <surfaceflinger/Surface.h>
#include "VideoEditorTools.h"

#define CHECK_EGL_ERROR CHECK(EGL_SUCCESS == eglGetError())
#define CHECK_GL_ERROR CHECK(GLenum(GL_NO_ERROR) == glGetError())

//
// Vertex and fragment programs
//

// The matrix is derived from
// frameworks/base/media/libstagefright/colorconversion/ColorConverter.cpp
//
// R * 255 = 1.164 * (Y - 16) + 1.596 * (V - 128)
// G * 255 = 1.164 * (Y - 16) - 0.813 * (V - 128) - 0.391 * (U - 128)
// B * 255 = 1.164 * (Y - 16) + 2.018 * (U - 128)
//
// Here we assume YUV are in the range of [0,255], RGB are in the range of
// [0, 1]
#define RGB2YUV_MATRIX \
"const mat4 rgb2yuv = mat4("\
"    65.52255,   -37.79398,   111.98732,     0.00000,"\
"   128.62729,   -74.19334,   -93.81088,     0.00000,"\
"    24.92233,   111.98732,   -18.17644,     0.00000,"\
"    16.00000,   128.00000,   128.00000,     1.00000);\n"

#define YUV2RGB_MATRIX \
"const mat4 yuv2rgb = mat4("\
"   0.00456,   0.00456,   0.00456,   0.00000,"\
"   0.00000,  -0.00153,   0.00791,   0.00000,"\
"   0.00626,  -0.00319,   0.00000,   0.00000,"\
"  -0.87416,   0.53133,  -1.08599,   1.00000);\n"

static const char vSrcNormal[] =
    "attribute vec4 vPosition;\n"
    "attribute vec2 vTexPos;\n"
    "uniform mat4 texMatrix;\n"
    "varying vec2 texCoords;\n"
    "varying float topDown;\n"
    "void main() {\n"
    "  gl_Position = vPosition;\n"
    "  texCoords = (texMatrix * vec4(vTexPos, 0.0, 1.0)).xy;\n"
    "  topDown = vTexPos.y;\n"
    "}\n";

static const char fSrcNormal[] =
    "#extension GL_OES_EGL_image_external : require\n"
    "precision mediump float;\n"
    "uniform samplerExternalOES texSampler;\n"
    "varying vec2 texCoords;\n"
    "void main() {\n"
    "  gl_FragColor = texture2D(texSampler, texCoords);\n"
    "}\n";

static const char fSrcSepia[] =
    "#extension GL_OES_EGL_image_external : require\n"
    "precision mediump float;\n"
    "uniform samplerExternalOES texSampler;\n"
    "varying vec2 texCoords;\n"
    RGB2YUV_MATRIX
    YUV2RGB_MATRIX
    "void main() {\n"
    "  vec4 rgb = texture2D(texSampler, texCoords);\n"
    "  vec4 yuv = rgb2yuv * rgb;\n"
    "  yuv = vec4(yuv.x, 117.0, 139.0, 1.0);\n"
    "  gl_FragColor = yuv2rgb * yuv;\n"
    "}\n";

static const char fSrcNegative[] =
    "#extension GL_OES_EGL_image_external : require\n"
    "precision mediump float;\n"
    "uniform samplerExternalOES texSampler;\n"
    "varying vec2 texCoords;\n"
    RGB2YUV_MATRIX
    YUV2RGB_MATRIX
    "void main() {\n"
    "  vec4 rgb = texture2D(texSampler, texCoords);\n"
    "  vec4 yuv = rgb2yuv * rgb;\n"
    "  yuv = vec4(255.0 - yuv.x, yuv.y, yuv.z, 1.0);\n"
    "  gl_FragColor = yuv2rgb * yuv;\n"
    "}\n";

static const char fSrcGradient[] =
    "#extension GL_OES_EGL_image_external : require\n"
    "precision mediump float;\n"
    "uniform samplerExternalOES texSampler;\n"
    "varying vec2 texCoords;\n"
    "varying float topDown;\n"
    RGB2YUV_MATRIX
    YUV2RGB_MATRIX
    "void main() {\n"
    "  vec4 rgb = texture2D(texSampler, texCoords);\n"
    "  vec4 yuv = rgb2yuv * rgb;\n"
    "  vec4 mixin = vec4(15.0/31.0, 59.0/63.0, 31.0/31.0, 1.0);\n"
    "  vec4 yuv2 = rgb2yuv * vec4((mixin.xyz * topDown), 1);\n"
    "  yuv = vec4(yuv.x, yuv2.y, yuv2.z, 1);\n"
    "  gl_FragColor = yuv2rgb * yuv;\n"
    "}\n";

namespace android {

NativeWindowRenderer::NativeWindowRenderer(sp<ANativeWindow> nativeWindow,
        int width, int height)
    : mNativeWindow(nativeWindow)
    , mDstWidth(width)
    , mDstHeight(height)
    , mLastVideoEffect(-1)
    , mNextTextureId(100)
    , mActiveInputs(0)
    , mThreadCmd(CMD_IDLE) {
    createThread(threadStart, this);
}

// The functions below run in the GL thread.
//
// All GL-related work is done in this thread, and other threads send
// requests to this thread using a command code. We expect most of the
// time there will only be one thread sending in requests, so we let
// other threads wait until the request is finished by GL thread.

int NativeWindowRenderer::threadStart(void* self) {
    LOGD("create thread");
    ((NativeWindowRenderer*)self)->glThread();
    return 0;
}

void NativeWindowRenderer::glThread() {
    initializeEGL();
    createPrograms();

    Mutex::Autolock autoLock(mLock);
    bool quit = false;
    while (!quit) {
        switch (mThreadCmd) {
            case CMD_IDLE:
                mCond.wait(mLock);
                continue;
            case CMD_RENDER_INPUT:
                render(mThreadRenderInput);
                break;
            case CMD_RESERVE_TEXTURE:
                glBindTexture(GL_TEXTURE_EXTERNAL_OES, mThreadTextureId);
                CHECK_GL_ERROR;
                break;
            case CMD_DELETE_TEXTURE:
                glDeleteTextures(1, &mThreadTextureId);
                break;
            case CMD_QUIT:
                terminateEGL();
                quit = true;
                break;
        }
        // Tell the requester that the command is finished.
        mThreadCmd = CMD_IDLE;
        mCond.broadcast();
    }
    LOGD("quit");
}

void NativeWindowRenderer::initializeEGL() {
    mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
    CHECK_EGL_ERROR;

    EGLint majorVersion;
    EGLint minorVersion;
    eglInitialize(mEglDisplay, &majorVersion, &minorVersion);
    CHECK_EGL_ERROR;

    EGLConfig config;
    EGLint numConfigs = -1;
    EGLint configAttribs[] = {
        EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
        EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
        EGL_RED_SIZE, 8,
        EGL_GREEN_SIZE, 8,
        EGL_BLUE_SIZE, 8,
        EGL_NONE
    };
    eglChooseConfig(mEglDisplay, configAttribs, &config, 1, &numConfigs);
    CHECK_EGL_ERROR;

    mEglSurface = eglCreateWindowSurface(mEglDisplay, config,
        mNativeWindow.get(), NULL);
    CHECK_EGL_ERROR;

    EGLint contextAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
    mEglContext = eglCreateContext(mEglDisplay, config, EGL_NO_CONTEXT,
        contextAttribs);
    CHECK_EGL_ERROR;

    eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, mEglContext);
    CHECK_EGL_ERROR;
}

void NativeWindowRenderer::terminateEGL() {
    eglDestroyContext(mEglDisplay, mEglContext);
    eglDestroySurface(mEglDisplay, mEglSurface);
    eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
    eglTerminate(mEglDisplay);
}

void NativeWindowRenderer::createPrograms() {
    GLuint vShader;
    loadShader(GL_VERTEX_SHADER, vSrcNormal, &vShader);

    const char* fSrc[NUMBER_OF_EFFECTS] = {
        fSrcNormal, fSrcSepia, fSrcNegative, fSrcGradient
    };

    for (int i = 0; i < NUMBER_OF_EFFECTS; i++) {
        GLuint fShader;
        loadShader(GL_FRAGMENT_SHADER, fSrc[i], &fShader);
        createProgram(vShader, fShader, &mProgram[i]);
        glDeleteShader(fShader);
        CHECK_GL_ERROR;
    }

    glDeleteShader(vShader);
    CHECK_GL_ERROR;
}

void NativeWindowRenderer::createProgram(
    GLuint vertexShader, GLuint fragmentShader, GLuint* outPgm) {

    GLuint program = glCreateProgram();
    CHECK_GL_ERROR;

    glAttachShader(program, vertexShader);
    CHECK_GL_ERROR;

    glAttachShader(program, fragmentShader);
    CHECK_GL_ERROR;

    glLinkProgram(program);
    CHECK_GL_ERROR;

    GLint linkStatus = GL_FALSE;
    glGetProgramiv(program, GL_LINK_STATUS, &linkStatus);
    if (linkStatus != GL_TRUE) {
        GLint infoLen = 0;
        glGetProgramiv(program, GL_INFO_LOG_LENGTH, &infoLen);
        if (infoLen) {
            char* buf = (char*) malloc(infoLen);
            if (buf) {
                glGetProgramInfoLog(program, infoLen, NULL, buf);
                LOGE("Program link log:\n%s\n", buf);
                free(buf);
            }
        }
        glDeleteProgram(program);
        program = 0;
    }

    *outPgm = program;
}

void NativeWindowRenderer::loadShader(GLenum shaderType, const char* pSource,
        GLuint* outShader) {
    GLuint shader = glCreateShader(shaderType);
    CHECK_GL_ERROR;

    glShaderSource(shader, 1, &pSource, NULL);
    CHECK_GL_ERROR;

    glCompileShader(shader);
    CHECK_GL_ERROR;

    GLint compiled = 0;
    glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
    if (!compiled) {
        GLint infoLen = 0;
        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen);
        char* buf = (char*) malloc(infoLen);
        if (buf) {
            glGetShaderInfoLog(shader, infoLen, NULL, buf);
            LOGE("Shader compile log:\n%s\n", buf);
            free(buf);
        }
        glDeleteShader(shader);
        shader = 0;
    }
    *outShader = shader;
}

NativeWindowRenderer::~NativeWindowRenderer() {
    CHECK(mActiveInputs == 0);
    startRequest(CMD_QUIT);
    sendRequest();
}

void NativeWindowRenderer::render(RenderInput* input) {
    sp<SurfaceTexture> ST = input->mST;
    sp<SurfaceTextureClient> STC = input->mSTC;

    if (input->mIsExternalBuffer) {
        queueExternalBuffer(STC.get(), input->mBuffer,
            input->mWidth, input->mHeight);
    } else {
        queueInternalBuffer(STC.get(), input->mBuffer);
    }

    ST->updateTexImage();
    glClearColor(0, 0, 0, 0);
    glClear(GL_COLOR_BUFFER_BIT);

    calculatePositionCoordinates(input->mRenderingMode,
        input->mWidth, input->mHeight);

    const GLfloat textureCoordinates[] = {
         0.0f,  1.0f,
         0.0f,  0.0f,
         1.0f,  0.0f,
         1.0f,  1.0f,
    };

    updateProgramAndHandle(input->mVideoEffect);

    glVertexAttribPointer(mPositionHandle, 2, GL_FLOAT, GL_FALSE, 0,
        mPositionCoordinates);
    CHECK_GL_ERROR;

    glEnableVertexAttribArray(mPositionHandle);
    CHECK_GL_ERROR;

    glVertexAttribPointer(mTexPosHandle, 2, GL_FLOAT, GL_FALSE, 0,
        textureCoordinates);
    CHECK_GL_ERROR;

    glEnableVertexAttribArray(mTexPosHandle);
    CHECK_GL_ERROR;

    GLfloat texMatrix[16];
    ST->getTransformMatrix(texMatrix);
    glUniformMatrix4fv(mTexMatrixHandle, 1, GL_FALSE, texMatrix);
    CHECK_GL_ERROR;

    glBindTexture(GL_TEXTURE_EXTERNAL_OES, input->mTextureId);
    CHECK_GL_ERROR;

    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexParameteri(
        GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(
        GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    CHECK_GL_ERROR;

    glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
    CHECK_GL_ERROR;

    eglSwapBuffers(mEglDisplay, mEglSurface);
}

void NativeWindowRenderer::queueInternalBuffer(ANativeWindow *anw,
    MediaBuffer* buffer) {
    int64_t timeUs;
    CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
    native_window_set_buffers_timestamp(anw, timeUs * 1000);
    status_t err = anw->queueBuffer(anw, buffer->graphicBuffer().get());
    if (err != 0) {
        LOGE("queueBuffer failed with error %s (%d)", strerror(-err), -err);
        return;
    }

    sp<MetaData> metaData = buffer->meta_data();
    metaData->setInt32(kKeyRendered, 1);
}

void NativeWindowRenderer::queueExternalBuffer(ANativeWindow* anw,
    MediaBuffer* buffer, int width, int height) {
    native_window_set_buffers_geometry(anw, width, height,
            HAL_PIXEL_FORMAT_YV12);
    native_window_set_usage(anw, GRALLOC_USAGE_SW_WRITE_OFTEN);

    ANativeWindowBuffer* anb;
    anw->dequeueBuffer(anw, &anb);
    CHECK(anb != NULL);

    sp<GraphicBuffer> buf(new GraphicBuffer(anb, false));
    CHECK(NO_ERROR == anw->lockBuffer(anw, buf->getNativeBuffer()));

    // Copy the buffer
    uint8_t* img = NULL;
    buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img));
    copyYV12Buffer(buffer, img, width, height, buf->getStride());
    buf->unlock();
    CHECK(NO_ERROR == anw->queueBuffer(anw, buf->getNativeBuffer()));
}

void NativeWindowRenderer::copyYV12Buffer(MediaBuffer* src, uint8_t* dst,
        int srcWidth, int srcHeight, int stride) {
    int strideUV = (stride / 2 + 0xf) & ~0xf;
    uint8_t* p = (uint8_t*)src->data() + src->range_offset();
    // Y
    for (int i = srcHeight; i > 0; i--) {
        memcpy(dst, p, srcWidth);
        dst += stride;
        p += srcWidth;
    }
    // The src is I420, the dst is YV12.
    // U
    p += srcWidth * srcHeight / 4;
    for (int i = srcHeight / 2; i > 0; i--) {
        memcpy(dst, p, srcWidth / 2);
        dst += strideUV;
        p += srcWidth / 2;
    }
    // V
    p -= srcWidth * srcHeight / 2;
    for (int i = srcHeight / 2; i > 0; i--) {
        memcpy(dst, p, srcWidth / 2);
        dst += strideUV;
        p += srcWidth / 2;
    }
}

void NativeWindowRenderer::updateProgramAndHandle(uint32_t videoEffect) {
    if (mLastVideoEffect == videoEffect) {
        return;
    }

    mLastVideoEffect = videoEffect;
    int i;
    switch (mLastVideoEffect) {
        case VIDEO_EFFECT_NONE:
            i = 0;
            break;
        case VIDEO_EFFECT_SEPIA:
            i = 1;
            break;
        case VIDEO_EFFECT_NEGATIVE:
            i = 2;
            break;
        case VIDEO_EFFECT_GRADIENT:
            i = 3;
            break;
        default:
            i = 0;
            break;
    }
    glUseProgram(mProgram[i]);
    CHECK_GL_ERROR;

    mPositionHandle = glGetAttribLocation(mProgram[i], "vPosition");
    mTexPosHandle = glGetAttribLocation(mProgram[i], "vTexPos");
    mTexMatrixHandle = glGetUniformLocation(mProgram[i], "texMatrix");
    CHECK_GL_ERROR;
}

void NativeWindowRenderer::calculatePositionCoordinates(
        M4xVSS_MediaRendering renderingMode, int srcWidth, int srcHeight) {
    float x, y;
    switch (renderingMode) {
        case M4xVSS_kResizing:
        default:
            x = 1;
            y = 1;
            break;
        case M4xVSS_kCropping:
            x = float(srcWidth) / mDstWidth;
            y = float(srcHeight) / mDstHeight;
            // Make the smaller side 1
            if (x > y) {
                x /= y;
                y = 1;
            } else {
                y /= x;
                x = 1;
            }
            break;
        case M4xVSS_kBlackBorders:
            x = float(srcWidth) / mDstWidth;
            y = float(srcHeight) / mDstHeight;
            // Make the larger side 1
            if (x > y) {
                y /= x;
                x = 1;
            } else {
                x /= y;
                y = 1;
            }
            break;
    }

    mPositionCoordinates[0] = -x;
    mPositionCoordinates[1] = y;
    mPositionCoordinates[2] = -x;
    mPositionCoordinates[3] = -y;
    mPositionCoordinates[4] = x;
    mPositionCoordinates[5] = -y;
    mPositionCoordinates[6] = x;
    mPositionCoordinates[7] = y;
}

//
//  The functions below run in other threads.
//

void NativeWindowRenderer::startRequest(int cmd) {
    mLock.lock();
    while (mThreadCmd != CMD_IDLE) {
        mCond.wait(mLock);
    }
    mThreadCmd = cmd;
}

void NativeWindowRenderer::sendRequest() {
    mCond.broadcast();
    while (mThreadCmd != CMD_IDLE) {
        mCond.wait(mLock);
    }
    mLock.unlock();
}

RenderInput* NativeWindowRenderer::createRenderInput() {
    LOGD("new render input %d", mNextTextureId);
    RenderInput* input = new RenderInput(this, mNextTextureId);

    startRequest(CMD_RESERVE_TEXTURE);
    mThreadTextureId = mNextTextureId;
    sendRequest();

    mNextTextureId++;
    mActiveInputs++;
    return input;
}

void NativeWindowRenderer::destroyRenderInput(RenderInput* input) {
    LOGD("destroy render input %d", input->mTextureId);
    GLuint textureId = input->mTextureId;
    delete input;

    startRequest(CMD_DELETE_TEXTURE);
    mThreadTextureId = textureId;
    sendRequest();

    mActiveInputs--;
}

//
//  RenderInput
//

RenderInput::RenderInput(NativeWindowRenderer* renderer, GLuint textureId)
    : mRenderer(renderer)
    , mTextureId(textureId) {
    mST = new SurfaceTexture(mTextureId);
    mSTC = new SurfaceTextureClient(mST);
}

RenderInput::~RenderInput() {
}

ANativeWindow* RenderInput::getTargetWindow() {
    return mSTC.get();
}

void RenderInput::updateVideoSize(sp<MetaData> meta) {
    CHECK(meta->findInt32(kKeyWidth, &mWidth));
    CHECK(meta->findInt32(kKeyHeight, &mHeight));

    int left, top, right, bottom;
    if (meta->findRect(kKeyCropRect, &left, &top, &right, &bottom)) {
        mWidth = right - left + 1;
        mHeight = bottom - top + 1;
    }

    // If rotation degrees is 90 or 270, swap width and height
    // (mWidth and mHeight are the _rotated_ source rectangle).
    int32_t rotationDegrees;
    if (!meta->findInt32(kKeyRotation, &rotationDegrees)) {
        rotationDegrees = 0;
    }

    if (rotationDegrees == 90 || rotationDegrees == 270) {
        int tmp = mWidth;
        mWidth = mHeight;
        mHeight = tmp;
    }
}

void RenderInput::render(MediaBuffer* buffer, uint32_t videoEffect,
        M4xVSS_MediaRendering renderingMode, bool isExternalBuffer) {
    mVideoEffect = videoEffect;
    mRenderingMode = renderingMode;
    mIsExternalBuffer = isExternalBuffer;
    mBuffer = buffer;

    mRenderer->startRequest(NativeWindowRenderer::CMD_RENDER_INPUT);
    mRenderer->mThreadRenderInput = this;
    mRenderer->sendRequest();
}

}  // namespace android