通過GLSL將YV12轉換爲RGB的問題

Question

我正在嘗試使用GLSL着色器來完成YV12轉RGB轉換提及in this post。

我的應用程序從磁盤加載一個原始YV12幀並嘗試使用GLSL着色器執行轉換。但是，由此產生的圖像垂直翻轉，並有一些顏色問題。我認爲問題可能是圖像正在以char（1字節）的數組讀取，然後轉換爲GLushort（2字節）的數組。你怎麼看？

這是原始YUV框架的樣子：

and the raw frame loaded by the application can be downloaded from here。

，這是我得到的輸出：

我分享下面的應用程序的源代碼：

#include <assert.h> 
#include <math.h> 
#include <stdio.h> 
#include <stdlib.h> 
#include <string.h> 

#include <GL/glew.h> 
#include <GL/glut.h> 
#include <GL/glu.h> 

#include <iostream> 

#include <fstream> 

#ifndef SEEK_SET 
# define SEEK_SET 0 
#endif 

static GLfloat Xrot = 0, Yrot = 0, Zrot = 0; 
static GLint ImgWidth, ImgHeight; 
static GLushort *ImageYUV = NULL; 


static void DrawObject(void) 
{ 
    glBegin(GL_QUADS); 

    glTexCoord2f(0, 0); 
    glVertex2f(-1.0, -1.0); 

    glTexCoord2f(ImgWidth, 0); 
    glVertex2f(1.0, -1.0); 

    glTexCoord2f(ImgWidth, ImgHeight); 
    glVertex2f(1.0, 1.0); 

    glTexCoord2f(0, ImgHeight); 
    glVertex2f(-1.0, 1.0); 

    glEnd(); 
} 

static void Display(void) 
{ 
    glClear(GL_COLOR_BUFFER_BIT); 

    glPushMatrix(); 
     glRotatef(Xrot, 1.0, 0.0, 0.0); 
     glRotatef(Yrot, 0.0, 1.0, 0.0); 
     glRotatef(Zrot, 0.0, 0.0, 1.0); 
     DrawObject(); 
    glPopMatrix(); 

    glutSwapBuffers(); 
} 

static void Reshape(int width, int height) 
{ 
    glViewport(0, 0, width, height); 
    glMatrixMode(GL_PROJECTION); 
    glLoadIdentity(); 

    // Vertical flip so texture appears right 
    glFrustum(-1.0, 1.0, 1.0, -1.0, 10.0, 100.0); 
    //glFrustum(-1.0, 1.0, -1.0, 1.0, 10.0, 100.0); 

    glMatrixMode(GL_MODELVIEW); 
    glLoadIdentity(); 
    glTranslatef(0.0, 0.0, -15.0); 
} 

static void Key(unsigned char key, int x, int y) 
{ 
    (void) x; 
    (void) y; 
    switch (key) { 
     case 27: 
     exit(0); 
     break; 
    } 
    glutPostRedisplay(); 
} 

static void SpecialKey(int key, int x, int y) 
{ 
    float step = 3.0; 
    (void) x; 
    (void) y; 

    switch (key) { 
     case GLUT_KEY_UP: 
     Xrot += step; 
     break; 
     case GLUT_KEY_DOWN: 
     Xrot -= step; 
     break; 
     case GLUT_KEY_LEFT: 
     Yrot += step; 
     break; 
     case GLUT_KEY_RIGHT: 
     Yrot -= step; 
     break; 
    } 
    glutPostRedisplay(); 
}   

bool CheckShader(int n_shader_object) 
{ 
    int n_tmp; 
    glGetShaderiv(n_shader_object, GL_COMPILE_STATUS, &n_tmp); 
    bool b_compiled = n_tmp == GL_TRUE; 
    int n_log_length; 
    glGetShaderiv(n_shader_object, GL_INFO_LOG_LENGTH, &n_log_length); 
    // query status ... 

    if(n_log_length > 1) { 
     char *p_s_temp_info_log; 
     if(!(p_s_temp_info_log = (char*)malloc(n_log_length))) 
      return false; 
     int n_tmp; 
     glGetShaderInfoLog(n_shader_object, n_log_length, &n_tmp, 
      p_s_temp_info_log); 
     assert(n_tmp <= n_log_length); 

     fprintf(stderr, "%s\n", p_s_temp_info_log); 
     free(p_s_temp_info_log); 
    } 
    // get/concat info-log 

    return b_compiled; 
} 

static void Init(int argc, char *argv[]) 
{ 
    GLuint texObj = 100; 
    const char *file; 

    printf("Checking GL_ARB_texture_rectangle\n"); 
    if (!glutExtensionSupported("GL_ARB_texture_rectangle")) { 
     printf("Sorry, GL_ARB_texture_rectangle is required\n"); 
     exit(0); 
    } 

    glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 

    glBindTexture(GL_TEXTURE_RECTANGLE_ARB, texObj); 
#ifdef LINEAR_FILTER 
    /* linear filtering looks much nicer but is much slower for Mesa */ 
    glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 
    glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 
#else 
    glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 
    glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 
#endif 

    std::ifstream yuv_file("data.yv12", std::ios::in | std::ios::binary | std::ios::ate); 
    if (!yuv_file.is_open()) 
    { 
     std::cout << "> GLWidget::GLWidget !!! Failed to load yuv file"; 
     return; 
    } 
    int yuv_file_sz = yuv_file.tellg(); 

    ImgWidth = 1280; 
    ImgHeight = 720; 
    ImageYUV = new GLushort[yuv_file_sz]; 

    char* memblock = new char[yuv_file_sz]; 
    if (!memblock) 
    { 
     std::cout << "> GLWidget::GLWidget !!! Failed to allocate memblock"; 
     return; 
    } 

    yuv_file.seekg(0, std::ios::beg); 
    yuv_file.read(memblock, yuv_file_sz); 
    yuv_file.close(); 

    // A simple "memcpy(ImageYUV, memblock, yuv_file_sz);" 
    // won't work because the data read is stored as char (1 byte) and GLushort is 2 bytes. 
    // So, doing a manual copy: 
    for (int i = 0; i < yuv_file_sz; i++) 
    { 
     ImageYUV[i] = (GLushort)memblock[i]; 
    } 
    delete[] memblock; 

    printf("Image: %dx%d\n", ImgWidth, ImgHeight); 

    glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, 
       GL_LUMINANCE_ALPHA, ImgWidth, ImgHeight, 0, 
       GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, ImageYUV); 

    assert(glGetError() == GL_NO_ERROR); 

    glTexSubImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, 
        0, 0, ImgWidth, ImgHeight, 
        GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, ImageYUV); 

    assert(glGetError() == GL_NO_ERROR); 

    delete[] ImageYUV; 

    glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE); 

    glEnable(GL_TEXTURE_RECTANGLE_ARB); 

    glShadeModel(GL_FLAT); 
    glClearColor(0.3, 0.3, 0.4, 1.0); 

    static const char *p_s_vertex_shader = 
     "varying vec2 t;" 
     "void main()" 
     "{" 
     " t = gl_MultiTexCoord0.xy;" 
     " gl_Position = ftransform();" 
     "}"; 
    static const char *p_s_fragment_shader = 
     "#extension GL_ARB_texture_rectangle : enable\n" 
     "varying vec2 t;" 
     "uniform sampler2DRect tex;" 
     "void main()" 
     "{" 
     " vec2 tcEven = vec2(floor(t.x * .5) * 2.0, t.y);" 
     " vec2 tcOdd = vec2(tcEven.x + 1.0, t.y);" 
     " float Cb = texture2DRect(tex, tcEven).x - .5;" 
     " float Cr = texture2DRect(tex, tcOdd).x - .5;" 
     " float y = texture2DRect(tex, t).w;" // redundant texture read optimized away by texture cache 
     " float r = y + 1.28033 * Cr;" 
     " float g = y - .21482 * Cb - .38059 * Cr;" 
     " float b = y + 2.12798 * Cb;" 
     " gl_FragColor = vec4(r, g, b, 1.0);" 
     "}"; 

    int v = glCreateShader(GL_VERTEX_SHADER); 
    int f = glCreateShader(GL_FRAGMENT_SHADER); 
    int p = glCreateProgram(); 
    glShaderSource(v, 1, &p_s_vertex_shader, 0); 
    glShaderSource(f, 1, &p_s_fragment_shader, 0); 
    glCompileShader(v); 
    CheckShader(v); 
    glCompileShader(f); 
    CheckShader(f); 
    glAttachShader(p, v); 
    glAttachShader(p, f); 
    glLinkProgram(p); 
    glUseProgram(p); 
    glUniform1i(glGetUniformLocation(p, "tex"), 0); 

    if (argc > 1 && strcmp(argv[1], "-info")==0) { 
     printf("GL_RENDERER = %s\n", (char *) glGetString(GL_RENDERER)); 
     printf("GL_VERSION = %s\n", (char *) glGetString(GL_VERSION)); 
     printf("GL_VENDOR  = %s\n", (char *) glGetString(GL_VENDOR)); 
     printf("GL_EXTENSIONS = %s\n", (char *) glGetString(GL_EXTENSIONS)); 
    } 
} 


int main(int argc, char *argv[]) 
{ 
    glutInit(&argc, argv); 
    glutInitWindowSize(1280, 720); 
    glutInitWindowPosition(0, 0); 
    glutInitDisplayMode(GLUT_RGB | GLUT_DOUBLE); 
    glutCreateWindow(argv[0]); 
    glewInit(); 

    Init(argc, argv); 

    glutReshapeFunc(Reshape); 
    glutKeyboardFunc(Key); 
    glutSpecialFunc(SpecialKey); 
    glutDisplayFunc(Display); 

    glutMainLoop(); 
    return 0; 
} 

Answer 1

這裏的問題是，圖像實際上不是YV12，色度和亮度平面不是交錯的，而是以塊爲單位。這可以通過兩種方式來解決，或者在將平面加載到紋理之前交織平面並按原樣使用剩餘的代碼，或者可以在着色器中完成。我刪除了iostream，並用stdio替換了它（我使用的是相當老的編譯器）。這裏是我的加載圖像和交錯它的代碼：

GLubyte *memblock; 
{ 
    FILE *p_fr = fopen("data.yv12", "rb"); 
    if(!p_fr) { 
     fprintf(stderr, "!!! Failed to load yuv file\n"); 
     return; 
    } 
    fseek(p_fr, 0, SEEK_END); 
    int yuv_file_sz = ftell(p_fr); 
    fseek(p_fr, 0, SEEK_SET); 
    memblock = new GLubyte[yuv_file_sz]; 
    if(!memblock) { 
     fprintf(stderr, "!!! Failed to allocate memblock\n"); 
     return; 
    } 
    fread(memblock, yuv_file_sz, 1, p_fr); 
    fclose(p_fr); 
} 
// load .raw file 

ImgWidth = 1280; 
ImgHeight = 720; 
ImageYUV = new GLushort[ImgWidth * ImgHeight]; 
// allocate an image 

int chromaWidth = ImgWidth/2; 
int chromaHeight = ImgHeight/2; // 2x2 luminance subsampling 
const GLubyte *pCb = memblock + ImgWidth * ImgHeight; // Cb block after Y 
const GLubyte *pCr = pCb + chromaWidth * chromaHeight; // Cr block after Cb 
// get pointers to smaller Cb and Cr blocks (the image is *not* interleaved) 

for(int i = 0; i < ImgWidth * ImgHeight; ++ i) { 
    int x = i % ImgWidth; 
    int y = i/ImgWidth; 
    GLubyte cb = pCb[(x/2) + (y/2) * chromaWidth]; 
    GLubyte cr = pCr[(x/2) + (y/2) * chromaWidth]; 
    ImageYUV[i] = (memblock[i] << 8) | ((x & 1)? cr : cb); 
} 
// convert (interleave) the data to YV12 

這是非常簡單的，可用於上面的着色器。

現在如果我們想要跳過交錯？首先，我要弄清楚如何在這裏的尋址作品（我們要像圖像有點高單色圖像，色度飛機服用亮度平面上方的空間）：

for(int y = 0; y < ImgHeight; ++ y) { 
    for(int x = 0; x < ImgWidth; ++ x) { 
     int CbY = ImgHeight + (y/4); 
     int CrY = ImgHeight + chromaHeight/2 + (y/4); 
     int CbCrX = (x/2) + chromaWidth * ((y/2) & 1); 
     // calculate x, y of cr and cb pixels in the grayscale image 
     // where the Y, Cb anc Cr blocks are next to each other 

     assert(&memblock[CbCrX + CbY * ImgWidth] == &pCb[(x/2) + (y/2) * chromaWidth]); 
     assert(&memblock[CbCrX + CrY * ImgWidth] == &pCr[(x/2) + (y/2) * chromaWidth]); 
     // make sure the addresses are correct (and they are) 

     GLubyte cb = memblock[CbCrX + CbY * ImgWidth]; 
     GLubyte cr = memblock[CbCrX + CrY * ImgWidth]; 
     GLubyte Y = memblock[x + y * ImgWidth]; 

     ImageYUV[x + y * ImgWidth] = (Y << 8) | ((x & 1)? cr : cb); 
    } 
} 
// convert (interleave) the data to YV12 (a little bit different way, use physical layout in memory) 

這有幾乎相同的效果。現在我們可以將計算位置的代碼放入着色器中。

static const char *p_s_fragment_shader = 
    "#extension GL_ARB_texture_rectangle : enable\n" 
    "uniform sampler2DRect tex;" 
    "uniform float ImgHeight, chromaHeight_Half, chromaWidth;" 
    "void main()" 
    "{" 
    " vec2 t = gl_TexCoord[0].xy;" // get texcoord from fixed-function pipeline 
    " float CbY = ImgHeight + floor(t.y/4.0);" 
    " float CrY = ImgHeight + chromaHeight_Half + floor(t.y/4.0);" 
    " float CbCrX = floor(t.x/2.0) + chromaWidth * floor(mod(t.y, 2.0));" 
    " float Cb = texture2DRect(tex, vec2(CbCrX, CbY)).x - .5;" 
    " float Cr = texture2DRect(tex, vec2(CbCrX, CrY)).x - .5;" 
    " float y = texture2DRect(tex, t).x;" // redundant texture read optimized away by texture cache 
    " float r = y + 1.28033 * Cr;" 
    " float g = y - .21482 * Cb - .38059 * Cr;" 
    " float b = y + 2.12798 * Cb;" 
    " gl_FragColor = vec4(r, g, b, 1.0);" 
    "}"; 

通過使用這種着色器，我們可以直接將原始數據上傳到質地，但它是有點高，只有GL_LUMINANCE：

glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, 
       GL_LUMINANCE, ImgWidth, ImgHeight + ImgHeight/2, 0, // !! 
       GL_LUMINANCE, GL_UNSIGNED_BYTE, memblock); // !! 

我會離開它。下面是完整的源代碼：

interleaving in shader (faster, preferrable)
manual interleaving in "C"

對不起，我快結束時，我都會有問題，如果我不離開我的工作儘快:)。