Research Android Audio and Video -1- Basics using MediaExtractor and MediaMuxer

Major problems solved

1. Separate MP4 into RAW data, AAC and H264

2. Separate MP4 into audio and video

3. Repackage audio and video as MP4

MediaExtractor: Audio and video data separator for demultiplexing (Demuxer)

Get the demultiplexed information

// Check the video information
// Demultiplexer demuxer
val mediaExtractor = MediaExtractor();
mediaExtractor.setDataSource(videoPath);
val numTracks = mediaExtractor.trackCount;
println("extrarot streams $numTracks;")
for (i in 0 until numTracks) {
    val mediaFormat = mediaExtractor.getTrackFormat(i)
    val mine = mediaFormat.getString(MediaFormat.KEY_MIME)
    println("stream $i mime: $mine")}// Prints information
2021-04-16 18:30:21.099 /System.out: extrarot streams 2;
2021-04-16 18:30:21.099 /System.out: stream 0 mime: video/avc
2021-04-16 18:30:21.100 /System.out: stream 1 mime: audio/mp4a-latm
Copy the code

Get Audio and Video

var audioFrameIndex = -1
var videoFrameIndex = -1
for (i in 0 until numTracks) {
    val mediaFormat = mediaExtractor.getTrackFormat(i)
    val mine = mediaFormat.getString(MediaFormat.KEY_MIME)// Get the MIME format content
    println("stream $i [mime: $mine]. "")
    if(mine? .startsWith("video/")!! { videoFrameIndex = i }if (mine.startsWith("audio/")) {
        audioFrameIndex = i
    }
}
Copy the code

Example 1: Parse and save audio information, < raw data > saved data cannot be processed.

if(audioFrameIndex ! = -1) {
 mediaExtractor.selectTrack(audioFrameIndex)
 val outAudioFile = File("$filesDir/a1.aac")
 outAudioFile.deleteOnExit()
 val fos = FileOutputStream(outAudioFile)
 val byteBuffer = ByteBuffer.allocate(1024 * 1024)
 val byteArray = ByteArray(byteBuffer.capacity());
 while (true) {
     val readCount = mediaExtractor.readSampleData(byteBuffer, 0)
     if (readCount < 0) {
         println(Reading no audio data, stop!)
         break
     }
     println("Write audio data [$readCount] bytes")
     byteBuffer.get(byteArray, 0, readCount)
     fos.write(byteArray, 0, readCount)
     byteBuffer.clear()
     mediaExtractor.advance()
     if(! mediaExtractor.advance()) { println("No data after switch to next read point, stop!")
         break
     }
 }
 fos.flush()
 fos.close()
 mediaExtractor.unselectTrack(audioFrameIndex)
}
Copy the code

Example 2: Parse and save video information, < raw data > saved data cannot be processed.

if(videoFrameIndex ! = -1) {
 mediaExtractor.selectTrack(videoFrameIndex)
 val outVideoFile = File("$filesDir/a1.h264")
 outVideoFile.deleteOnExit()
 val fos = FileOutputStream(outVideoFile)
 val byteBuffer = ByteBuffer.allocate(1024 * 1024)
 val byteArray = ByteArray(byteBuffer.capacity());
 while (true) {
     val readCount = mediaExtractor.readSampleData(byteBuffer, 0)
     if (readCount < 0) {
         println("Reading no video data, stop!")
         break
     }
     println("Write video data [$readCount] bytes")
     byteBuffer.get(byteArray, 0, readCount)
     fos.write(byteArray, 0, readCount)
     byteBuffer.clear()
     mediaExtractor.advance()
     if(! mediaExtractor.advance()) { println("After switch to next read point, no video data, stop!")
         break
     }
 }
 fos.flush()
 fos.close()
 mediaExtractor.unselectTrack(videoFrameIndex)
}
Copy the code

MediaMuxer solves video multiplexing, respectively solving audio MP3 and video MP4

//
val path2Save = "$filesDir/test.mp3"
mediaExtractor.extractorMedia(audioFrameIndex, path2Save)

val path2Save = "$filesDir/test.mp4"
mediaExtractor.extractorMedia(videoFrameIndex, path2Save)

/** * separates the media in the corresponding track and saves it to the specified path */
fun MediaExtractor.extractorMedia(frameIndex: Int, path2Save: String) {
    val mediaMuxer = MediaMuxer(path2Save, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
    mediaMuxer.demuxerMedia(this, frameIndex)
    mediaMuxer.release()
}

/** * writes the specified index data to the specified mixer */
fun MediaMuxer.demuxerMedia(extractor: MediaExtractor, frameIndex: Int) {
    extractor.selectTrack(frameIndex)
    val format = extractor.getTrackFormat(frameIndex)

    val trackIndex = addTrack(format)
    start()
    writeMediaBuffer(extractor, format, trackIndex)

    extractor.unselectTrack(frameIndex)
}

/** * the actual writing method */
private fun MediaMuxer.writeMediaBuffer(
    extractor: MediaExtractor,
    format: MediaFormat,
    trackIndex: Int
) {
    val maxMediaBufferCount: Int = format.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE)
    val byteBuffer = ByteBuffer.allocate(maxMediaBufferCount)
    val bufferInfo = MediaCodec.BufferInfo()
    while (true) {
        val readSampleDataSize = extractor.readSampleData(byteBuffer, 0)
        if (readSampleDataSize < 0) {
// println("index [$trackIndex] ") )
            break
        }
        bufferInfo.size = readSampleDataSize
        bufferInfo.offset = 0
        bufferInfo.presentationTimeUs = extractor.sampleTime
        bufferInfo.flags = extractor.sampleFlags
// println("index [$trackIndex] write data:[$readSampleDataSize]")
        writeSampleData(trackIndex, byteBuffer, bufferInfo)
        if(! extractor.advance()) {// println("index [$trackIndex] ") )
            break}}}Copy the code

MediaMuxer unmultiplexes video, combining audio MP3 and video MP4 separately from the previous step into a new MP4

//
fun muxerAudioAndVideo(audioPath: String, videoPath: String, outPath: String) {
    // Merge parsed audio and video into MP4

    val audioExtractor = MediaExtractor();
    audioExtractor.setDataSource(audioPath)
    val audioFrameIndex = audioExtractor.findTargetStreamIndex("audio/")

    val videoExtractor = MediaExtractor()
    videoExtractor.setDataSource(videoPath)
    val videoFrameIndex = videoExtractor.findTargetStreamIndex("video/")

    if(audioFrameIndex ! = -1&& videoFrameIndex ! = -1) {
        val mediaMuxer =
        MediaMuxer(outPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
        mediaMuxer.muxerAudioAndVideo(
            audioExtractor,
            videoExtractor,
            audioFrameIndex,
            videoFrameIndex
        )
        mediaMuxer.release()
    } else {
        println("Input audio [$audioPath: $audioFrameIndex] or video [$videoPath: $videoFrameIndex] If there is an exception, please check")
    }

    audioExtractor.release()
    videoExtractor.release()
}

/** * Whether to reuse the specified header, video/ or audio/ */
fun MediaExtractor.findTargetStreamIndex(header: String): Int {
    val trackCount = trackCount
    for (i in 0 until trackCount) {
        val format = getTrackFormat(i)
        val mine = format.getString(MediaFormat.KEY_MIME)
        if(mine? .startsWith(header)!!) { println("found target stream [$header] index[$i]. "")
            return i
        }
    }
    println("not found target stream [$header]. "")
    return -1
}

/** * merge two streams */
fun MediaMuxer.muxerAudioAndVideo(
    audioExtractor: MediaExtractor,
    videoExtractor: MediaExtractor,
    audioFrameIndex: Int,
    videoFrameIndex: Int
) {

    var audioTrackIndex = -1
    audioExtractor.selectTrack(audioFrameIndex)
    var audioFormat = audioExtractor.getTrackFormat(audioFrameIndex)

    var videoTrackIndex = -1
    videoExtractor.selectTrack(videoFrameIndex)
    val videoFormat = videoExtractor.getTrackFormat(videoFrameIndex)

    audioTrackIndex = addTrack(audioFormat)// Trace the audio channel
    videoTrackIndex = addTrack(videoFormat)// Trace the video channel
    start()// Start to prepare the mixture
    // Write the audio stream
    writeMediaBuffer(audioExtractor, audioFormat, audioTrackIndex)
    // Write the video stream
    writeMediaBuffer(videoExtractor, videoFormat, videoTrackIndex)

    audioExtractor.unselectTrack(audioFrameIndex)
    videoExtractor.unselectTrack(videoFrameIndex)
    release()
}
Copy the code

​