安卓下多線程OpenGL共享Context (四)

     之前的方案假定Java層更新紋理時使用的是RGB或RBGA格式的數據，但是在播放視頻這種應用場景下，解碼器解碼出來的數據如果是YUV格式，渲染起來就比較麻煩了。一種方式是使用CPU進行YUV轉RGB，然后再進行渲染，但是這種方式性能極差；另一種方式是使用GPU進行轉換，利用GPU的並行計算能力加速轉換。我們需要編寫Shader來實現。如前文所述，Unity只需要Java層的紋理ID，當使用Shader進行YUV轉RGB時，怎么實現更新該紋理的數據呢？答案是Render to Texture (參見[1])。具體做法是，創建一個FrameBuffer，調用glFramebufferTexture2D將紋理與FrameBuffer關聯起來，這樣在FrameBuffer上進行的繪制，就會被寫入到該紋理中。Java代碼如下：

public void setupGL(int width, int height) {
    // 創建紋理
    int tempBuffer[] = new int[1];
    GLES20.glGenTextures(1, tempBuffer, 0);
    mTextureId = tempBuffer[0];
    if (mTextureId == 0) {
        glLogE("setupGL, glGenTextures for render texture failed");
        return;
    }
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureId);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, width, height, 0, GLES20.GL_RGBA,
            GLES20.GL_UNSIGNED_BYTE, null);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);

    // 創建YUV紋理
    GLES20.glGenTextures(3, mYuvTextures, 0);
    if (mYuvTextures[0] == 0 || mYuvTextures[1] == 0 || mYuvTextures[2] == 0) {
        MyLog.e(TAG, "setupGL, glGenTextures for yuv texture failed");
        return;
    }
    for (int yuvTexture : mYuvTextures) {
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTexture);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
    }
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);

    // 創建幀緩沖區
    GLES20.glGenFramebuffers(1, tempBuffer, 0);
    mFramebuffer = tempBuffer[0];
    if (mFramebuffer == 0) {
        glLogE("setupGL, glGenFramebuffers failed");
        return;
    }
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFramebuffer);
    GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
            GLES20.GL_TEXTURE_2D, mTextureId, 0);
    int errCode = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER);
    if (errCode != GLES20.GL_FRAMEBUFFER_COMPLETE) {
        glLogE("setupGL, glCheckFramebufferStatus failed, errCode=0x" + Integer.toHexString(errCode));
        return;
    }
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
}

     繪制時，先綁定FrameBuffer，再進行繪制操作，Java代碼如下：

public void updateTexture() {
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFramebuffer);
    GLES20.glViewport(0, 0, mWidth, mHeight);
    GLES20.glClearColor(0, 0, 0, 1);
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    // 此處添加繪制操作
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
}

      為了實現YUV數據的渲染，需要編寫Shader。此處，我們渲染的YUV數據格式為YUV420 (YUV420格式的具體介紹，請讀者自行百度)，Y、U、V通道數據分別存放在三個緩沖區中。將YUV數據分別賦給三個紋理，然后指定Shader的頂點坐標和紋理坐標，繪制一個矩形即可 (參見[2])。

      首先，在setupGL函數中為YUV生成三個紋理，Java代碼如下：

// 創建YUV紋理
GLES20.glGenTextures(3, mYuvTextures, 0);
if (mYuvTextures[0] == 0 || mYuvTextures[1] == 0 || mYuvTextures[2] == 0) {
    MyLog.e(TAG, "setupGL, glGenTextures for yuv texture failed");
    return;
}
for (int yuvTexture : mYuvTextures) {
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTexture);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
}
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);

繪制時，將YUV數據分別賦給三個紋理，並將三個紋理分別與GLES20.GL_TEXTURE0，GLES20.GL_TEXTURE1，GLES20.GL_TEXTURE2綁定，Java代碼如下：

for (int i = 0; i < 3; ++i) {
    GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mYuvTextures[i]);
    int w = i == 0 ? yuvFrame.yuvStrides[0] : (yuvFrame.yuvStrides[0] / 2);
    int h = i == 0 ? yuvFrame.height : (yuvFrame.height / 2);
    GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0,
            GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, yuvFrame.yuvPlanes[i]);
}

　　然后，我們需要創建Program，為Program創建Vertex Shader和Fragment Shader (參見[2])。Java代碼如下：

public static final String VERTEX_SHADER_STRING = "attribute vec4 in_pos;\n"
        + "attribute vec2 in_tc;\n"
        + "varying vec2 out_tc;\n"
        + "void main() {\n"
        + "    gl_Position = in_pos;\n"
        + "    out_tc = in_tc;\n"
        + "}\n";

public static final String FRAGMENT_SHADER_STRING = "precision mediump float;\n"
        + "uniform sampler2D tex_y;\n"
        + "uniform sampler2D tex_u;\n"
        + "uniform sampler2D tex_v;\n"
        + "varying vec2 out_tc;\n"
        + "void main() {\n"
        + "    vec4 c = vec4((texture2D(tex_y, out_tc).r - 16./255.) * 1.164);\n"
        + "    vec4 U = vec4(texture2D(tex_u, out_tc).r - 128./255.);\n"
        + "    vec4 V = vec4(texture2D(tex_v, out_tc).r - 128./255.);\n"
        + "    c += V * vec4(1.596, -0.813, 0, 0);\n"
        + "    c += U * vec4(0, -0.392, 2.017, 0);\n"
        + "    c.a = 1.0;\n"
        + "    gl_FragColor = c;\n"
        + "}\n";

protected final void addShaderTo(int type, String source, int program) throws RuntimeException {
    int shader = GLES20.glCreateShader(type);
    if (shader == 0) {
        throw new RuntimeException("Create shader failed, err=" + GLES10.glGetError());
    }
    GLES20.glShaderSource(shader, source);
    GLES20.glCompileShader(shader);
    int[] result = new int[]{GLES20.GL_FALSE};
    GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0);
    if (result[0] != GLES20.GL_TRUE) {
        GLES20.glDeleteShader(shader);
        throw new RuntimeException("Compile shader failed, err=" + GLES10.glGetError());
    }
    GLES20.glAttachShader(program, shader);
    GLES20.glDeleteShader(shader);
}

public void setupGL(int width, int height) {
    ...
    // 創建Program
    mProgram = GLES20.glCreateProgram();
    addShaderTo(GLES20.GL_VERTEX_SHADER, EglRender.VERTEX_SHADER_STRING, mProgram);
    addShaderTo(GLES20.GL_FRAGMENT_SHADER, EglRender.FRAGMENT_SHADER_STRING, mProgram);
    GLES20.glLinkProgram(mProgram);
    GLES20.glGetProgramiv(mProgram, GLES20.GL_LINK_STATUS, tempBuffer, 0);
    if (tempBuffer[0] != GLES20.GL_TRUE) {
        glLogE("setupGL, create program failed");
        return;
    }
    GLES20.glUseProgram(mProgram);
    int y_tex = GLES20.glGetUniformLocation(mProgram, "tex_y");
    GLES20.glUniform1i(y_tex, 0);
    int u_tex = GLES20.glGetUniformLocation(mProgram, "tex_u");
    GLES20.glUniform1i(u_tex, 1);
    int v_tex = GLES20.glGetUniformLocation(mProgram, "tex_v");
    GLES20.glUniform1i(v_tex, 2);
    mVertexLocation = GLES20.glGetAttribLocation(mProgram, "in_pos");
    mTextureLocation = GLES20.glGetAttribLocation(mProgram, "in_tc");
    GLES20.glUseProgram(0);
}

完整繪制代碼如下：

public void updateTexture() {
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFramebuffer);

    GLES20.glViewport(0, 0, mWidth, mHeight);
    GLES20.glClearColor(0, 0, 0, 1);
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    GLES20.glDisable(GLES20.GL_CULL_FACE);
    GLES20.glDisable(GLES20.GL_DEPTH_TEST);
    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0);
    GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, 0);

    GLES20.glUseProgram(mProgram);
    // 更新YUV數據
    for (int i = 0; i < 3; ++i) {
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mYuvTextures[i]);
        int w = i == 0 ? yuvFrame.yuvStrides[0] : (yuvFrame.yuvStrides[0] / 2);
        int h = i == 0 ? yuvFrame.height : (yuvFrame.height / 2);
        GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0,
                GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, yuvFrame.yuvPlanes[i]);
    }
    // 設置頂點坐標和紋理坐標
    GLES20.glEnableVertexAttribArray(mVertexLocation);
    mVertexCoord.position(0);
    GLES20.glVertexAttribPointer(mVertexLocation, 2, GLES20.GL_FLOAT, false, 0, mVertexCoord);
    GLES20.glEnableVertexAttribArray(mTextureLocation);
    mTextureCoord.position(0);
    GLES20.glVertexAttribPointer(mTextureLocation, 2, GLES20.GL_FLOAT, false, 0, mTextureCoord);
    // 繪制矩形
    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);

    GLES20.glDisableVertexAttribArray(mVertexLocation);
    GLES20.glDisableVertexAttribArray(mTextureLocation);
    for (int i = 0; i < 3; ++i) {
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
    }

    GLES20.glUseProgram(0);
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
    GLES20.glClear(0);
}

其中，9～10行是必需的，否則，glDrawArrays調用會失敗，報GL_INVALID_OPERATION錯誤 (參見[3])。當繪制完成之后，需要通知Unity3D，在C#中調用GL.InvalidateState，否則會影響Unity的繪制 (參見[4])。C#代碼如下：

void Update () {
    mPluginTexture.Call ("updateTexture");
    GL.InvalidateState ();
}

其中，Update函數是Unity在每次刷新幀時回調，我們在該回調中調用Java層的updateTexture函數更新紋理數據，然后調用GL.InvalidateState，通知Unity重置OpenGL狀態。筆者一開始采用當Java解碼出一幀時，便更新紋理數據進行繪制，然后通知C#調用GL.InvalidateState。但是這種方式存在兩個問題，一是當App退到后台，繪制操作仍會進行，只是繪制會失敗；二是畫面更新一段時間之后，便會卡住，過很久才會恢復。具體原因並未查出。后來，改用從C#的Update回調更新紋理，這兩個問題得以解決。

總結：

     Unity官方給出的Plugin繪制方式是通過在C#層調用GL.IssuePluginEvent，C++層接收從Unity Render線程過來的回調，在該回調中更新紋理 (參見[5])。該方案需要編寫JNI和C++，實現起來比較麻煩。本文給出的方案全部在Java層即可實現。供感興趣的讀者參考。

[參考文獻]

[1] Tutorial 14 : Render To Texture

[2] 最簡單的視音頻播放示例6：OpenGL播放YUV420P（通過Texture，使用Shader）

[3] robertcastle/UnityFBO

[4] GL.InvalidateState

[5] Low-level Native Plugin Interface