An overview of the
The function is very simple, and the general process is:
- MediaCodec decodes video files to get YUV and PCM data
- OpenGL converts YUV to RGB and renders it on the Surface
- OpenSL/AudoTrack gets PCM data and plays it
The pre-requisite knowledge is:
- YUV, PCM and other basic audio and video knowledge, such as YUV to RGB
- The use of MediaCodec
- OpenGL, including EGL, texture, etc
- Use of OpenSL or AudioTrack
MediaCodec decoding
General process and common decoding similar, in the preparation of video player this function, need to pay attention to two places:
- Listening decoding flow
public interface OnDecodeListener {
void onImageDecoded(byte[] data);
void onSampleDecoded(byte[] data);
void onDecodeEnded();
}
Copy the codeYou can also add an onDecodeError() interface if needed.
- Play and decode synchronization
Because of the large amount of video data, it is impossible to save the decoded YUV data in a queue and then slowly take it out and use OpenGL rendering (it is easy to get OOM). Therefore, it is necessary to control the decoding rate. The simplest control method is to synchronize with playback, as shown below:
ByteBuffer outputBuffer = outputBuffers[outIndex];
outputBuffer.position(bufferInfo.offset);
outputBuffer.limit(bufferInfo.offset + bufferInfo.size);
byte[] data = new byte[bufferInfo.size];
outputBuffer.get(data);
if (mIsDecodeWithPts) {
if (startTime == 0) {
startTime = System.nanoTime();
} else {
passTime = (System.nanoTime() - startTime) / 1000;
if(passTime < bufferInfo.presentationTimeUs) { TimeUnit.MICROSECONDS.sleep(bufferInfo.presentationTimeUs - passTime); }}}if(mediaType == HWCodec.MEDIA_TYPE_VIDEO && listener ! = null) { listener.onImageDecoded(data); }else if(listener ! = null) { listener.onSampleDecoded(data); }Copy the codeOpenGL renders YUV data
Similar to the process of rendering texture, the difference is that YUV data needs to be converted to RGB, and YUV data has YUV420P, YUV420SP and other formats, so before converting RGB, the format of YUV data needs to be unified. YUV420P is used here.
YUV data format conversion can be written, such as YUV420SP to YUV420P, only need to put the last U and V data into an array one by one, but considering video cropping, rotation, and various YUV data processing functions that may be used later, So here is a library for libyuv, which is very simple to use:
Yuv* convertToI420(AVModel *model) {
...
Yuv *yuv = new Yuv(model->width, model->height);
ConvertToI420(model->image, (size_t) model->imageLen, yuv->bufY, yuv->strideY,
yuv->bufU, yuv->strideU, yuv->bufV, yuv->strideV,
0, 0, model->width, model->height, model->width, model->height,
kRotate0, getFourCC(model->pixelFormat));
return yuv;
}
Copy the codeAVModel, Yuv is my custom two classes, respectively used to save audio and video data and related information, Yuv data and related information, source visible GitHub.
The correlation coefficient of YUV to RGB can be searched online, and the fragment shader is as follows:
#version 300 es
precision highp float;
uniform sampler2D yTexture;
uniform sampler2D uTexture;
uniform sampler2D vTexture;
in vec2 vTexCoord;
layout(location=0) out vec4 fragColor;
void main() {
highp float y = texture(yTexture, vTexCoord).r;
highp floatU = texture(uTexture, vTexCoord).r-0.5; highpfloatV = texture(vTexture, vTexCoord).r-0.5; highpfloatR = y + 1.402 * v; highpfloatG = y-0.344 * u-0.714 * v; highpfloatB = y + 1.772 * u; FragColor = vec4(r, g, b, 1.0); }Copy the codeThe key code of OpenGL is as follows:
bool YuvRenderer::doInit() {
std::string *vShader = readShaderFromAsset(mAssetManager, "yuv_renderer.vert");
std::string *fShader = readShaderFromAsset(mAssetManager, "yuv_renderer.frag");
mProgram = loadProgram(vShader->c_str(), fShader->c_str());
mMatrixLoc = glGetUniformLocation(mProgram, "mMatrix");
mSamplerY = glGetUniformLocation(mProgram, "yTexture");
mSamplerU = glGetUniformLocation(mProgram, "uTexture");
mSamplerV = glGetUniformLocation(mProgram, "vTexture"); glPixelStorei(GL_UNPACK_ALIGNMENT, 1); GlGenTextures (3, mTextures); // Generate three textures, respectively, for loading Y, U, V data. glBindTexture(GL_TEXTURE_2D, mTextures[0]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth, mTexHeight, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glBindTexture(GL_TEXTURE_2D, mTextures[1]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glBindTexture(GL_TEXTURE_2D, mTextures[2]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); GlGenBuffers (3, mVboIds); // Buffers(3, mVboIds); glBindBuffer(GL_ARRAY_BUFFER, mVboIds[0]); glBufferData(GL_ARRAY_BUFFER, sizeof(VERTICES), VERTICES, GL_STATIC_DRAW); glBindBuffer(GL_ARRAY_BUFFER, mVboIds[1]); glBufferData(GL_ARRAY_BUFFER, sizeof(TEX_COORDS), TEX_COORDS, GL_STATIC_DRAW); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mVboIds[2]); glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(INDICES), INDICES, GL_STATIC_DRAW); GlGenVertexArrays (1, &mvaoid); glBindVertexArray(mVaoId); glBindBuffer(GL_ARRAY_BUFFER, mVboIds[0]); glEnableVertexAttribArray(ATTRIB_POSITION); glVertexAttribPointer(ATTRIB_POSITION, VERTEX_POS_SIZE, GL_FLOAT, GL_FALSE, sizeof(GLfloat) * VERTEX_POS_SIZE, 0);
glBindBuffer(GL_ARRAY_BUFFER, mVboIds[1]);
glEnableVertexAttribArray(ATTRIB_TEX_COORD);
glVertexAttribPointer(ATTRIB_TEX_COORD, TEX_COORD_SIZE, GL_FLOAT, GL_FALSE,
sizeof(GLfloat) * TEX_COORD_SIZE, 0); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mVboIds[2]); glBindVertexArray(0); glBindBuffer(GL_ARRAY_BUFFER, 0); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); GlClearColor (1.0 f, f, 1.0 1.0 f, 1.0 f); delete vShader; delete fShader;return true;
}
void YuvRenderer::doDraw() {
glViewport(0, 0, mWidth, mHeight);
glClear(GL_COLOR_BUFFER_BIT);
glUseProgram(mProgram);
glUniformMatrix4fv(mMatrixLoc, 1, GL_FALSE, mMatrix);
if(! mYuv) { LOGW("YuvRenderer doDraw failed: yuv data have not assigned");
return; } // Load Y, U, V data into the corresponding texture glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, mTextures[0]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth, mTexHeight, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, mYuv->bufY); glUniform1i(mSamplerY, 0); glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, mTextures[1]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, mYuv->bufU); glUniform1i(mSamplerU, 1); glActiveTexture(GL_TEXTURE2); glBindTexture(GL_TEXTURE_2D, mTextures[2]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, mYuv->bufV); glUniform1i(mSamplerV, 2); GlBindVertexArray (mVaoId); glBindVertexArray(mVaoId); glDrawElements(GL_TRIANGLES, INDEX_NUMBER, GL_UNSIGNED_SHORT, 0); glBindVertexArray(0); glBindTexture(GL_TEXTURE_2D, 0); }Copy the codeOpenSL plays PCM data
Initialize the player:
bool BQAudioPlayer::init() { SLresult result; SLDataLocator_AndroidSimpleBufferQueue locBufq = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2}; // channelMask: The bits are equal to the channel, 0 (SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT SLDataFormat_PCM formatPcm = {SL_DATAFORMAT_PCM, (SLuint32) mChannels, mSampleRate, (SLuint32) mSampleFormat, (SLuint32) mSampleFormat, mChannels == 2 ? 0 : SL_SPEAKER_FRONT_CENTER, SL_BYTEORDER_LITTLEENDIAN};if (mSampleRate) {
formatPcm.samplesPerSec = mSampleRate;
}
SLDataSource audioSrc = {&locBufq, &formatPcm};
SLDataLocator_OutputMix locOutpuMix = {SL_DATALOCATOR_OUTPUTMIX, mAudioEngine->outputMixObj};
SLDataSink audioSink = {&locOutpuMix, nullptr};
const SLInterfaceID ids[3] = {SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_EFFECTSEND};
const SLboolean req[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE};
result = (*mAudioEngine->engine)->CreateAudioPlayer(mAudioEngine->engine, &mPlayerObj,
&audioSrc, &audioSink,
mSampleRate ? 2 : 3, ids, req);
if(result ! = SL_RESULT_SUCCESS) { LOGE("CreateAudioPlayer failed: %d", result);
return false;
}
result = (*mPlayerObj)->Realize(mPlayerObj, SL_BOOLEAN_FALSE);
if(result ! = SL_RESULT_SUCCESS) { LOGE("mPlayerObj Realize failed: %d", result);
return false;
}
result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_PLAY, &mPlayer);
if(result ! = SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result);
return false;
}
result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_BUFFERQUEUE, &mBufferQueue);
if(result ! = SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result);
return false;
}
result = (*mBufferQueue)->RegisterCallback(mBufferQueue, playerCallback, this);
if(result ! = SL_RESULT_SUCCESS) { LOGE("mPlayerObj RegisterCallback failed: %d", result);
return false;
}
mEffectSend = nullptr;
if (mSampleRate == 0) {
result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_EFFECTSEND, &mEffectSend);
if(result ! = SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result);
return false;
}
}
result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_VOLUME, &mVolume);
if(result ! = SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result);
return false;
}
result = (*mPlayer)->SetPlayState(mPlayer, SL_PLAYSTATE_PLAYING);
if(result ! = SL_RESULT_SUCCESS) { LOGE("mPlayerObj SetPlayState failed: %d", result);
return false;
}
return true;
}
Copy the codeThen all you need to do is add PCM to the team:
/ / a frame after the audio playback will callback the function void playerCallback (SLAndroidSimpleBufferQueueItf bq, void *context) { BQAudioPlayer *player = (BQAudioPlayer *) context; assert(bq == player->mBufferQueue); pthread_mutex_unlock(&player->mMutex); } void BQAudioPlayer::enqueueSample(void *data, Pthread_mutex_lock (&mmutex);if (mBufSize < length) {
mBufSize = length;
if (mBuffers[0]) {
delete[] mBuffers[0];
}
if (mBuffers[1]) {
delete[] mBuffers[1];
}
mBuffers[0] = new uint8_t[mBufSize];
mBuffers[1] = new uint8_t[mBufSize];
}
memcpy(mBuffers[mIndex], data, length);
(*mBufferQueue)->Enqueue(mBufferQueue, mBuffers[mIndex], length);
mIndex = 1 - mIndex;
}
Copy the codeEnd of play:
void BQAudioPlayer::release() {
pthread_mutex_lock(&mMutex);
if (mPlayerObj) {
(*mPlayerObj)->Destroy(mPlayerObj);
mPlayerObj = nullptr;
mPlayer = nullptr;
mBufferQueue = nullptr;
mEffectSend = nullptr;
mVolume = nullptr;
}
if (mAudioEngine) {
delete mAudioEngine;
mAudioEngine = nullptr;
}
if (mBuffers[0]) {
delete[] mBuffers[0];
mBuffers[0] = nullptr;
}
if (mBuffers[1]) {
delete[] mBuffers[1];
mBuffers[1] = nullptr;
}
pthread_mutex_unlock(&mMutex);
pthread_mutex_destroy(&mMutex);
}
Copy the codeAudioTrack plays PCM data
Compared with OpenSL, AudioTrack has much less code. Set AudioTrack:
private void setupAudioTrack() { int channelConfig = mChannels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO; ENCODING_PCM_16BIT int bufferSize = Audiotrack.getm is used by defaultinBufferSize(mSampleRate, channelConfig, AudioFormat.ENCODING_PCM_16BIT);
mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, mSampleRate, channelConfig,
AudioFormat.ENCODING_PCM_16BIT, bufferSize, AudioTrack.MODE_STREAM);
}
Copy the codePlay PCM data:
@Override
public void onSampleDecoded(byte[] data) {
if(mIsPlaying) { mAudioTrack.write(data, 0, data.length); mAudioTrack.play(); }}Copy the codeEnd of play:
private void releaseAudioTrack() {
if (mAudioTrack != null) {
mAudioTrack.stop();
mAudioTrack.release();
mAudioTrack = null;
}
}
Copy the codeAbove, a simple video player is completed, of course, there are many details are not dealt with, interested can refer to iJkPlayer to improve.
The source code has been uploaded to GitHub.