iOS - 在OpenGL ES中使用BiPlanar像素格式渲染YUV420p圖像2

我正嘗試使用iOS 10.3.3在iPhone 6S上使用Swift 3渲染yuv420p編碼視頻到OpenGL ES2紋理。iOS - 在OpenGL ES中使用BiPlanar像素格式渲染YUV420p圖像2

紋理設置：

var formatType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange 
    var lumaTexture: CVOpenGLESTexture? 
    var chromaTexture: CVOpenGLESTexture? 
    var mediapixelBuffer: CVPixelBuffer? 
    var ioSurfaceBuffer: CVPixelBuffer? 

    media.videoSamplesBuffer = media.assetReaderOutput?.copyNextSampleBuffer() 
    mediapixelBuffer = CMSampleBufferGetImageBuffer(media.videoSamplesBuffer!)! 
    CVPixelBufferLockBaseAddress(mediapixelBuffer!, .readOnly) 

    let bufferWidth0: Int = CVPixelBufferGetWidthOfPlane(mediapixelBuffer!, 0) 
    let bufferWidth1: Int = CVPixelBufferGetWidthOfPlane(mediapixelBuffer!, 1) 
    let bufferHeight0: Int = CVPixelBufferGetWidthOfPlane(mediapixelBuffer!, 0) 
    let bufferHeight1: Int = CVPixelBufferGetWidthOfPlane(mediapixelBuffer!, 1) 
    let bytesPerRow0: Int = CVPixelBufferGetBytesPerRowOfPlane(mediapixelBuffer!, 0) 
    let bytesPerRow1: Int = CVPixelBufferGetBytesPerRowOfPlane(mediapixelBuffer!, 1) 

    let pixelBufferBaseAddress = CVPixelBufferGetBaseAddress(mediapixelBuffer!) 
    let pixelBufferPlaneAddress0 = CVPixelBufferGetBaseAddressOfPlane(mediapixelBuffer!, 0) 
    let pixelBufferPlaneAddress1 = CVPixelBufferGetBaseAddressOfPlane(mediapixelBuffer!, 1) 

    let ioBufferRet = CVPixelBufferCreate(kCFAllocatorDefault, 
              bufferWidth_, 
              bufferHeight_, 
              self.formatType, 
              attr, 
              &ioSurfaceBuffer) 
    if ioBufferRet != 0 { print("error at `CVPixelBufferCreate`", ioBufferRet) } 

    CVPixelBufferLockBaseAddress(ioSurfaceBuffer!, .readOnly) 

    var copyBufferPlaneAddress0 = CVPixelBufferGetBaseAddressOfPlane(ioSurfaceBuffer!, 0) 
    var copyBufferPlaneAddress1 = CVPixelBufferGetBaseAddressOfPlane(ioSurfaceBuffer!, 1) 

    memcpy(copyBufferPlaneAddress0, pixelBufferPlaneAddress0, bufferHeight0 * bytesPerRow0/2) // Y 
    memcpy(copyBufferPlaneAddress1, pixelBufferPlaneAddress1, bufferHeight1 * bytesPerRow1/2) // UV 

    glActiveTexture(GLenum(GL_TEXTURE0)) 
    if nil != ioSurfaceBuffer && nil != media.vidTexCachePtr { 
     var cvRet = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, 
                   media.vidTexCachePtr!, 
                   ioSurfaceBuffer!, 
                   nil, 
                   GLenum(GL_TEXTURE_2D), 
                   GLint(GL_RED_EXT), 
                   GLsizei(bufferWidth0), 
                   GLsizei(bufferHeight0), 
                   GLenum(GL_RED_EXT), 
                   GLenum(GL_UNSIGNED_BYTE), 
                   0, 
                   &lumaTexture) 
     if cvRet != 0 { print("0 error at `CVOpenGLESTextureCacheCreateTextureFromImage`", cvRet) } 
    } 
    if nil != lumaTexture { 
     glBindTexture(CVOpenGLESTextureGetTarget(lumaTexture!), CVOpenGLESTextureGetName(lumaTexture!)) 
    } 
    glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_MIN_FILTER), GL_LINEAR) 
    glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_S), GL_CLAMP_TO_EDGE) 
    glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_T), GL_CLAMP_TO_EDGE) 

    glActiveTexture(GLenum(GL_TEXTURE1)) 
    if nil != ioSurfaceBuffer && nil != media.vidTexCachePtr { 
     var cvRet = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, 
                  media.vidTexCachePtr!, 
                  ioSurfaceBuffer!, 
                  nil, 
                  GLenum(GL_TEXTURE_2D), 
                  GLint(GL_RG_EXT), 
                  GLsizei(bufferWidth1), 
                  GLsizei(bufferHeight1), 
                  GLenum(GL_RG_EXT), 
                  GLenum(GL_UNSIGNED_BYTE), 
                  1, 
                  &chromaTexture) 
     if cvRet != 0 { print("1 error at `CVOpenGLESTextureCacheCreateTextureFromImage`", cvRet) } 
    } 
    if nil != chromaTexture { 
     glBindTexture(CVOpenGLESTextureGetTarget(chromaTexture!), CVOpenGLESTextureGetName(chromaTexture!)) 
    } 
    glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_MIN_FILTER), GL_LINEAR) 
    glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_S), GL_CLAMP_TO_EDGE) 
    glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_T), GL_CLAMP_TO_EDGE) 

    CVPixelBufferUnlockBaseAddress(mediapixelBuffer!, .readOnly) 
    CVPixelBufferUnlockBaseAddress(ioSurfaceBuffer!, .readOnly)

片段着色器：

#version 100 
    precision mediump float; 

    varying vec2 vUV; 
    uniform sampler2D SamplerY; 
    uniform sampler2D SamplerUV; 

    void main() { 

     mediump vec3 yuv; 
     lowp vec3 rgb; 

     yuv.x = texture2D(SamplerY, vUV).r; 
     yuv.yz = texture2D(SamplerUV, vUV).rg - vec2(0.5, 0.5); 

     // Using BT.709 which is the standard for HDTV 
     rgb = mat3(  1,  1,  1, 
         0, -.18732, 1.8556, 
        1.57481, -.46813,  0) * yuv; 

     gl_FragColor = vec4(rgb, 1); 
    }

了獨立的亮度質地看起來右，但獨立的色度質地似乎只有CR通道。我知道因爲視頻是4：2：0，第二個色度通道是空的，所以也許我不應該「看見」Cb通道，但最終結果（應該是彩條色）看起來像this。它缺少紅色。（我認爲這是因爲輸出是BGRA，如果是RGBA，藍色會丟失）。我如何獲得紅色回來？

This post描述了一個類似的問題，我正在經歷。但該解決方案使用3個平面（分別爲Y，U和V），而我試圖用2個平面（Y和UV）實現此平面。我嘗試使用kCVPixelFormatType_420YpCbCr8Planar格式類型來訪問3架飛機，但是然後CVOpenGLESTextureCacheCreateTextureFromImage未能創建IOSurface。我也嘗試了一些不同的YUV-> RGB着色器方程，並研究了使用ffmpeg來提供CVPixelBuffer，但我無法爲我的iPhone架構（arm64）構建它。預先感謝您，任何幫助將不勝感激！

來源

2017-09-23 vipyne

所以事實證明SamplerUV紋理實際上並沒有被髮送到着色器。（但它在GPU捕獲的幀中可見，這是誤導）。我認爲（錯誤地）是因爲SamplerY被自動發送到着色器，第二紋理SamplerUV也是如此。所以我之前看到的結果是用於Y和UV紋理的亮度紋理。

是固定的問題，缺少的行：

var SamplerY: GLint = 0 
var SamplerUV: GLint = 1 

SamplerY = glGetUniformLocation(shaderProgram, "SamplerY") 
SamplerUV = glGetUniformLocation(shaderProgram, "SamplerUV") 

glUniform1i(SamplerY, 0) 
glUniform1i(SamplerUV, 1)

來源

2017-09-25 15:49:02 vipyne

iOS - 在OpenGL ES中使用BiPlanar像素格式渲染YUV420p圖像2

回答

相關問題