0
因此，我成功地使用MediaCodec連接了多個視頻文件的視頻流 - 儘可能多的MediaExtractor和解碼器MediaCodec s作爲視頻文件。現在我的問題是關於連接所述視頻的音頻流。MediaCodec - 如何將兩個mp4文件的音頻流連接成單一統一格式並將其複用回
使用改性ExtractDecodeEditEncodeMux測試，我試圖我用來連接用於音頻流中的視頻流中的相同的方法，確保最終的音頻編碼器有一個單一的預設格式：
private void audioExtractorLoop(MediaExtractor localAudioExtractor, MediaCodec destinationAudioDecoder, ByteBuffer[] dstAudioDecoderInputBuffers)
{
//Audio Extractor code begin
boolean localAudioExtractorIsOriginal = (localAudioExtractor == audioExtractor);
boolean localDone = localAudioExtractorIsOriginal ? audioExtractorDone : audioExtractorAppendDone;
Log.i("local_audio_extractor", localAudioExtractorIsOriginal+" "+localDone);
while (mCopyAudio && !localDone && (encoderOutputAudioFormat == null || muxing)) {
int decoderInputBufferIndex = destinationAudioDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE)
Log.d(TAG, "no audio decoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned input buffer: "
+ decoderInputBufferIndex);
}
ByteBuffer decoderInputBuffer = dstAudioDecoderInputBuffers[decoderInputBufferIndex];
int size = localAudioExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = localAudioExtractor.getSampleTime();
if(localAudioExtractorIsOriginal)currentFrameTimestamp = presentationTime;
if (VERBOSE) {
Log.d(TAG, "audio extractor: returned buffer of size "
+ size);
Log.d(TAG, "audio extractor: returned buffer for time "
+ presentationTime);
}
if (size >= 0) {
destinationAudioDecoder.queueInputBuffer(decoderInputBufferIndex, 0,
size, presentationTime,
localAudioExtractor.getSampleFlags());
}
localDone = !localAudioExtractor.advance();
if (localDone) {
if (VERBOSE)
Log.d(TAG, "audio extractor: EOS");
if(localAudioExtractorIsOriginal) {
initAudioExtractorFinalTimestamp = currentFrameTimestamp;
audioExtractorDone = true;
}
destinationAudioDecoder.queueInputBuffer(decoderInputBufferIndex, 0,
0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
audioExtractedFrameCount++;
break;
}
//Audio Extractor code end
}
private void localizedAudioDecoderLoop(MediaCodec localAudioDecoder)
{
boolean localAudioDecoderIsOriginal = (localAudioDecoder == audioDecoder);
boolean localDone = localAudioDecoderIsOriginal ? audioDecoderDone : audioDecoderAppendDone;
Log.i("local_audio_decoder", localAudioDecoderIsOriginal+"");
ByteBuffer[] localDecoderOutByteBufArray = localAudioDecoderIsOriginal ? audioDecoderOutputBuffers : audioDecoderAppendOutputBuffers;
MediaCodec.BufferInfo localDecoderBufInfo = localAudioDecoderIsOriginal ? audioDecoderOutputBufferInfo : audioDecoderAppendOutputBufferInfo;
while (mCopyAudio && !localDone && pendingAudioDecoderOutputBufferIndex == -1 && (encoderOutputAudioFormat == null || muxing)) {
int decoderOutputBufferIndex = localAudioDecoder.dequeueOutputBuffer(localDecoderBufInfo, TIMEOUT_USEC);
if(!localAudioDecoderIsOriginal)localDecoderBufInfo.presentationTimeUs += initAudioExtractorFinalTimestamp+33333;
//Log.i("decoder_out_buf_info", audioDecoderOutputBufferInfo.size + " " + audioDecoderOutputBufferInfo.offset);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE)
Log.d(TAG, "no audio decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE)
Log.d(TAG, "audio decoder: output buffers changed");
//audioDecoderOutputBuffers = audioDecoder.getOutputBuffers();
localDecoderOutByteBufArray = audioDecoder.getOutputBuffers();
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
decoderOutputAudioFormat = localAudioDecoder.getOutputFormat();
decoderOutputChannelNum = decoderOutputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
decoderOutputAudioSampleRate = decoderOutputAudioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
if (VERBOSE) {
Log.d(TAG, "audio decoder: output format changed: "
+ decoderOutputAudioFormat);
}
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned output buffer: "
+ decoderOutputBufferIndex);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned buffer of size "
+ localDecoderBufInfo.size);
}
ByteBuffer decoderOutputBuffer = localDecoderOutByteBufArray[decoderOutputBufferIndex];
if ((localDecoderBufInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE)
Log.d(TAG, "audio decoder: codec config buffer");
localAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex,
false);
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned buffer for time "
+ localDecoderBufInfo.presentationTimeUs);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: output buffer is now pending: "
+ pendingAudioDecoderOutputBufferIndex);
}
pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex;
audioDecodedFrameCount++;
break;
}
while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) {
if (VERBOSE) {
Log.d(TAG,
"audio decoder: attempting to process pending buffer: "
+ pendingAudioDecoderOutputBufferIndex);
}
int encoderInputBufferIndex = audioEncoder
.dequeueInputBuffer(TIMEOUT_USEC);
if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE)
Log.d(TAG, "no audio encoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned input buffer: "
+ encoderInputBufferIndex);
}
ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex];
int size = localDecoderBufInfo.size;
long presentationTime = localDecoderBufInfo.presentationTimeUs;
if (VERBOSE) {
Log.d(TAG, "audio decoder: processing pending buffer: "
+ pendingAudioDecoderOutputBufferIndex);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: pending buffer of size " + size);
Log.d(TAG, "audio decoder: pending buffer for time "
+ presentationTime);
}
if (size >= 0) {
ByteBuffer decoderOutputBuffer = localDecoderOutByteBufArray[pendingAudioDecoderOutputBufferIndex]
.duplicate();
byte[] testBufferContents = new byte[size];
//int bufferSize = (extractorInputChannelNum == 1 && decoderOutputChannelNum == 2) ? size/2 : size;
float samplingFactor = (decoderOutputChannelNum/extractorInputChannelNum) * (decoderOutputAudioSampleRate/extractorAudioSampleRate);
int bufferSize = size/(int)samplingFactor;
Log.i("sampling_factor", samplingFactor+" "+bufferSize);
if (decoderOutputBuffer.remaining() < size) {
for (int i = decoderOutputBuffer.remaining(); i < size; i++) {
testBufferContents[i] = 0; // pad with extra 0s to make a full frame.
}
decoderOutputBuffer.get(testBufferContents, 0, decoderOutputBuffer.remaining());
} else {
decoderOutputBuffer.get(testBufferContents, 0, size);
}
//WARNING: This works for 11025-22050-44100 or 8000-16000-24000-48000
//What about in-between?
//BTW, the size of the bytebuffer may be less than 4096 depending on the sampling factor
//(Now that I think about it I should've realized this back when I decoded the video result from the encoding - 2048 bytes decoded)
if (((int)samplingFactor) > 1) {
Log.i("s2m_conversion", "Stereo to Mono and/or downsampling");
byte[] finalByteBufferContent = new byte[size/2];
for (int i = 0; i < bufferSize; i+=2) {
if((i+1)*((int)samplingFactor) > testBufferContents.length)
{
finalByteBufferContent[i] = 0;
finalByteBufferContent[i+1] = 0;
}
else
{
finalByteBufferContent[i] = testBufferContents[i*((int)samplingFactor)];
finalByteBufferContent[i+1] = testBufferContents[i*((int)samplingFactor) + 1];
}
}
decoderOutputBuffer = ByteBuffer.wrap(finalByteBufferContent);
}
decoderOutputBuffer.position(localDecoderBufInfo.offset);
decoderOutputBuffer.limit(localDecoderBufInfo.offset + bufferSize);
//decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size);
encoderInputBuffer.position(0);
Log.d(TAG, "hans, audioDecoderOutputBufferInfo:" + localDecoderBufInfo.offset);
Log.d(TAG, "hans, decoderOutputBuffer:" + decoderOutputBuffer.remaining());
Log.d(TAG, "hans, encoderinputbuffer:" + encoderInputBuffer.remaining());
encoderInputBuffer.put(decoderOutputBuffer);
audioEncoder.queueInputBuffer(encoderInputBufferIndex, 0, bufferSize, presentationTime, localDecoderBufInfo.flags);
//audioEncoder.queueInputBuffer(encoderInputBufferIndex, 0, size, presentationTime, audioDecoderOutputBufferInfo.flags);
}
audioDecoder.releaseOutputBuffer(
pendingAudioDecoderOutputBufferIndex, false);
pendingAudioDecoderOutputBufferIndex = -1;
if ((localDecoderBufInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE)
Log.d(TAG, "audio decoder: EOS");
if(localDecoderBufInfo == audioDecoderOutputBufferInfo){audioDecoderDone = true;}
else{audioDecoderAppendDone = true;}
}
break;
}
}
進入這些功能我將傳遞第一個音頻流的MediaExtractor和解碼器MediaCodec對象並遍歷它們直到它們到達EOS，然後我將交換MediaExtractor和解碼器MediaCodec與第二個音頻流的對象。
此代碼工作正常的第一音頻流，但交換後，我得到以下堆棧跟蹤：
10-11 15:14:59.941 3067-22024/? E/SEC_AAC_DEC: saacd_decode() failed ret_val: -3, Indata 0x 11 90 00 00, length : 683
10-11 15:14:59.941 3067-22024/? E/SEC_AAC_DEC: ASI 0x 11, 90 00 00
10-11 15:14:59.951 29907-22020/com.picmix.mobile E/ACodec: OMXCodec::onEvent, OMX_ErrorStreamCorrupt
10-11 15:14:59.951 29907-22020/com.picmix.mobile W/AHierarchicalStateMachine: Warning message AMessage(what = 'omxI') = {
int32_t type = 0
int32_t event = 1
int32_t data1 = -2147479541
int32_t data2 = 0
} unhandled in root state.
我以爲解碼器會剛剛結束了44100赫茲樣本進行解碼的所有音頻串流傳送到audio/raw類型速率和2個通道，因此編碼器可以將數據採集並編碼爲最終格式。
我需要額外考慮哪些音頻，以及如何在交換Extractor-Decoder對時防止音頻流損壞？
編輯：
我添加這些線，以檢查所提取的樣品中的內容在MediaExtractor：
ByteBuffer decoderInputBuffer = dstAudioDecoderInputBuffers[decoderInputBufferIndex];
int size = localAudioExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = localAudioExtractor.getSampleTime();
//new lines begin
byte[] debugBytes = new byte[decoderInputBuffer.remaining()];
decoderInputBuffer.duplicate().get(debugBytes);
Log.i(TAG, "DEBUG - extracted frame: "+ audioExtractedFrameCount +" | bytebuffer contents: "+new String(debugBytes));
//new lines end
在decoderInputBuffer.duplicate().get(debugBytes);線，我得到IllegalStateException: buffer is inaccessible錯誤。
這是否意味着我設置提取器是錯誤的？
編輯2：
當我看着它進一步，這只是與附加音頻提取，不是第一音頻提取問題。