MediaCodec從視頻獲取所有幀

Question

我試圖使用MediaCodec從視頻中檢索所有幀以進行圖像處理，我試圖渲染視頻並從outBuffers中捕獲幀，但我可以從接收到的字節中啓動一個位圖實例。我試圖把它渲染到一個表面或沒有（空），因爲我已經注意到，當你渲染爲null時，outBuffers獲取渲染幀的字節。

這是代碼：

private static final String SAMPLE = Environment.getExternalStorageDirectory() + "/test_videos/sample2.mp4"; 
private PlayerThread mPlayer = null; 

@Override 
protected void onCreate(Bundle savedInstanceState) { 
    super.onCreate(savedInstanceState); 

    SurfaceView sv = new SurfaceView(this); 
    sv.getHolder().addCallback(this); 
    setContentView(sv); 
} 

protected void onDestroy() { 
    super.onDestroy(); 
} 

@Override 
public void surfaceCreated(SurfaceHolder holder) { 
} 

@Override 
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { 
    if (mPlayer == null) { 
     mPlayer = new PlayerThread(holder.getSurface()); 
     mPlayer.start(); 
    } 
} 

@Override 
public void surfaceDestroyed(SurfaceHolder holder) { 
    if (mPlayer != null) { 
     mPlayer.interrupt(); 
    } 
} 

private void writeFrameToSDCard(byte[] bytes, int i, int sampleSize) { 
    try { 
     Bitmap bmp = BitmapFactory.decodeByteArray(bytes, 0, sampleSize); 

     File file = new File(Environment.getExternalStorageDirectory() + "/test_videos/sample" + i + ".png"); 
     if (file.exists()) 
      file.delete(); 

     file.createNewFile(); 

     FileOutputStream out = new FileOutputStream(file.getAbsoluteFile()); 

     bmp.compress(Bitmap.CompressFormat.PNG, 90, out); 
     out.close(); 

    } catch (Exception e) { 
     e.printStackTrace(); 
    } 
} 

private class PlayerThread extends Thread { 
    private MediaExtractor extractor; 
    private MediaCodec decoder; 
    private Surface surface; 

    public PlayerThread(Surface surface) { 
     this.surface = surface; 
    } 

    @Override 
    public void run() { 
     extractor = new MediaExtractor(); 
     extractor.setDataSource(SAMPLE); 

     int index = extractor.getTrackCount(); 
     Log.d("MediaCodecTag", "Track count: " + index); 

     for (int i = 0; i < extractor.getTrackCount(); i++) { 
      MediaFormat format = extractor.getTrackFormat(i); 
      String mime = format.getString(MediaFormat.KEY_MIME); 
      if (mime.startsWith("video/")) { 
       extractor.selectTrack(i); 
       decoder = MediaCodec.createDecoderByType(mime); 
       decoder.configure(format, surface, null, 0); 
       break; 
      } 
     } 

     if (decoder == null) { 
      Log.e("DecodeActivity", "Can't find video info!"); 
      return; 
     } 

     decoder.start(); 

     ByteBuffer[] inputBuffers = decoder.getInputBuffers(); 
     ByteBuffer[] outputBuffers = decoder.getOutputBuffers(); 
     BufferInfo info = new BufferInfo(); 
     boolean isEOS = false; 
     long startMs = System.currentTimeMillis(); 

     int i = 0; 
     while (!Thread.interrupted()) { 
      if (!isEOS) { 
       int inIndex = decoder.dequeueInputBuffer(10000); 
       if (inIndex >= 0) { 
        ByteBuffer buffer = inputBuffers[inIndex]; 

        int sampleSize = extractor.readSampleData(buffer, 0); 

        if (sampleSize < 0) { 
         decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 
         isEOS = true; 
        } else { 
         decoder.queueInputBuffer(inIndex, 0, sampleSize, extractor.getSampleTime(), 0); 
         extractor.advance(); 
        } 
       } 
      } 

      /* saves frame to sdcard */ 
      int outIndex = decoder.dequeueOutputBuffer(info, 10000); // outIndex most of the times null 

      switch (outIndex) { 
      case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: 
       Log.d("DecodeActivity", "INFO_OUTPUT_BUFFERS_CHANGED"); 
       outputBuffers = decoder.getOutputBuffers(); 
       break; 
      case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: 
       Log.d("DecodeActivity", "New format " + decoder.getOutputFormat()); 
       break; 
      case MediaCodec.INFO_TRY_AGAIN_LATER: 
       Log.d("DecodeActivity", "dequeueOutputBuffer timed out!"); 
       break; 
      default: 
       ByteBuffer buffer = outputBuffers[outIndex]; 
       Log.v("DecodeActivity", "We can't use this buffer but render it due to the API limit, " + buffer); 

       // We use a very simple clock to keep the video FPS, or the video 
       // playback will be too fast 
       while (info.presentationTimeUs/1000 > System.currentTimeMillis() - startMs) { 
        try { 
         sleep(10); 
        } catch (InterruptedException e) { 
         e.printStackTrace(); 
         break; 
        } 
       } 
       decoder.releaseOutputBuffer(outIndex, true); 
       try { 
        byte[] dst = new byte[outputBuffers[outIndex].capacity()]; 
        outputBuffers[outIndex].get(dst); 
        writeFrameToSDCard(dst, i, dst.length); 
        i++; 
       } catch (Exception e) { 
        Log.d("iDecodeActivity", "Error while creating bitmap with: " + e.getMessage()); 
       } 

       break; 
      } 

      // All decoded frames have been rendered, we can stop playing now 
      if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 
       Log.d("DecodeActivity", "OutputBuffer BUFFER_FLAG_END_OF_STREAM"); 
       break; 
      } 
     } 

     decoder.stop(); 
     decoder.release(); 
     extractor.release(); 
    } 
} 

任何幫助將大大appriciated

Answer 1

可以解碼到Surface或到ByteBuffer，但不能同時使用。由於您正在配置Surface，因此輸出緩衝區中始終有零個字節的數據。

如果您配置爲ByteBuffer解碼，數據格式將有所不同，但據我所知，將永遠不會是Bitmap理解的ARGB格式。您可以在方法checkFrame()的CTS EncodeDecodeTest中的緩衝區到緩衝區測試中查看兩個YUV格式的示例。但是，請注意，它所做的第一件事是檢查格式並在沒有識別時立即返回。

目前（Android 4.4），唯一可行的方法是解碼爲SurfaceTexture，使用GLES進行渲染，然後使用glReadPixels()提取RGB數據。示例代碼在bigflake上可用 - 請參閱ExtractMpegFramesTest（需要API 16+）。