FFmpeg video cover extraction exploration

This article mainly introduces some technical details of the VideosNAP implementation in Snapshot, which is used to replace the VideoDecoder implementation in Glide to improve decoding efficiency, reduce memory footprint and reduce the probability of large-resolution video crashes.

Compile ffmpeg

Compile environment

• Ffmepg version:4.2.2 and above
• Desktop OS: Ubuntu 20.04
• NDK version: Android-NdK-R21-linux-x86_64 or later

Compilation step

• Configuration yasm

• Configure the NDK

• Download the FFmpeg source code, and compile the script to the source code decompressed in the root directory

• Execute the script and wait for completion

#! /bin/sh
make distclean
NDK="/usr/ndk/android-ndk-r21e"
HOST="linux-x86_64"
SYSROOT="$NDK/toolchains/llvm/prebuilt/$HOST/sysroot"
LLVM_TOOLCHAIN="$NDK/toolchains/llvm/prebuilt/$HOST/bin"
PKG_CONFIG="/usr/bin/pkg-config"
ASM="$SYSROOT/usr/include/$TARGET"
CFLAGS="-Os -fpic"

build () {
    ARCH=The $1
    API=$2
    CONFIGURATION="--disable-asm --enable-cross-compile --disable-static --disable-programs --disable-doc --enable-shared --disable-avdevice --disable-swscale --disable-avfilter --enable-protocol=file --enable-pic --enable-version3 --enable-nonfree --enable-small --disable-encoders"
    LDFLAGS="-nostdlib"
    
    case $ARCH in
        "armeabi-v7a")
            TARGET="arm-linux-androideabi"
            SPECIAL_TARGET="armv7a-linux-androideabi"
            LDFLAGS="--fix-cortex-a8 $LDFLAGS"
            FF_ARCH="arm"
            CPU="armv7-a"
        ;;
        "arm64-v8a")
            TARGET="aarch64-linux-android"
            SPECIAL_TARGET=$TARGET
            CONFIGURATION="$CONFIGURATION --disable-pthreads"
            FF_ARCH="aarch64"
            CPU="armv8-a"
        ;;
        "x86")
            TARGET="i686-linux-android"
            SPECIAL_TARGET=$TARGET
            FF_ARCH="x86"
            CPU="x86"
        ;;
        "x86_64")
            TARGET="x86_64-linux-android"
            SPECIAL_TARGET=$TARGET
            FF_ARCH="x86_64"
            CPU="x86_64"
        ;;
    esac

    CC=$LLVM_TOOLCHAIN/$SPECIAL_TARGET$API-clang
    CXX=$LLVM_TOOLCHAIN/$SPECIAL_TARGET$API-clang++
    # CROSS_PREFIX=$LLVM_TOOLCHAIN/$SPECIAL_TARGET-
    AS=$LLVM_TOOLCHAIN/$TARGET-as
    AR=$LLVM_TOOLCHAIN/$TARGET-ar
    LD=$LLVM_TOOLCHAIN/$TARGET-ld
    STRIP=$LLVM_TOOLCHAIN/$TARGET-strip
    PREFIX="$(pwd)/android/$ARCH"
    sed -i "s/-Wl,-soname/-soname/" configure

    ./configure --prefix=$PREFIX  \
                $CONFIGURATION \
                --arch=$FF_ARCH \
                --cpu=$CPU \
                --ar=$AR --strip=$STRIP --ld=$LD --cc=$CC --cxx=$CXX --as=$AS \
                --target-os=android \
                --pkg-config=$PKG_CONFIG \
                --sysroot=$SYSROOT \
                --extra-cflags="-I$ASM $CFLAGS" \
                --extra-ldflags="-L$SYSROOT/usr/lib $LDFLAGS" \
                $ADDITIONAL_CONFIGURE_FLAG

    sed  -i "s/#define getenv(x) NULL/\/\/ #define getenv(x) NULL/" config.h
    sed  -i "s/#define HAVE_CBRT 0/#define HAVE_CBRT 1/" config.h
    sed  -i "s/#define HAVE_CBRTF 0/#define HAVE_CBRTF 1/" config.h
    sed  -i "s/#define HAVE_COPYSIGN 0/#define HAVE_COPYSIGN 1/" config.h
    sed  -i "s/#define HAVE_ERF 0/#define HAVE_ERF 1/" config.h
    sed  -i "s/#define HAVE_ISNAN 0/#define HAVE_ISNAN 1/" config.h
    sed  -i "s/#define HAVE_ISFINITE 0/#define HAVE_ISFINITE 1/" config.h
    sed  -i "s/#define HAVE_HYPOT 0/#define HAVE_HYPOT 1/" config.h
    sed  -i "s/#define HAVE_RINT 0/#define HAVE_RINT 1/" config.h
    sed  -i "s/#define HAVE_LRINT 0/#define HAVE_LRINT 1/" config.h
    sed  -i "s/#define HAVE_LRINTF 0/#define HAVE_LRINTF 1/" config.h
    sed  -i "s/#define HAVE_ROUND 0/#define HAVE_ROUND 1/" config.h
    sed  -i "s/#define HAVE_ROUNDF 0/#define HAVE_ROUNDF 1/" config.h
    sed  -i "s/#define HAVE_TRUNC 0/#define HAVE_TRUNC 1/" config.h
    sed  -i "s/#define HAVE_TRUNCF 0/#define HAVE_TRUNCF 1/" config.h
    sed  -i "s/#define HAVE_INET_ATON 0/#define HAVE_INET_ATON 1/" config.h

    make clean
    make -j12
    make install
}

build "armeabi-v7a" "21"
build "arm64-v8a" "21"
build "x86_64" "21"
build "x86" "16"
Copy the code

IO Type Determination

Android layer: Input IO scheme

Android’s messy File management system, File and Uri schemes exist together, the FileSystem API is being abandoned, some versions of Android do not fully support Uri schemes, and disabled partitioned storage on Android 10 have to be considered.

Option 1: Uri to File (not recommended)

• In earlier versions, it was possible to convert a File object into a File object by reading the absolute path of a File in a multimedia database, but Android blocked this (failed).
• Using file descriptors (restricted in Android 11)

    /**
     * 注意释放资源 
     */
    public static File convertUriToFile(Context context, Uri uri) throws FileNotFoundException {
        int fd = context.getContentResolver().openAssetFileDescriptor(uri, "r").getParcelFileDescriptor().getFd();
        File result = null;
        if(fd ! = -1) {
            String path = "/proc/self/fd/" + fd;
            result = new File(path);
        }
        return result;
    }
Copy the code

Scheme 2: Use file descriptors

    /** * Only used as Demo, try resource will release the resource, resulting in the return fd cannot be used * At the same time, note that the return value of afd.getStarToffSet () from assets or raw is different */
    public static int getFd(Context context, Uri uri) {
        try (AssetFileDescriptor afd = context.getContentResolver().openAssetFileDescriptor(uri, "r")) {
            return afd.getParcelFileDescriptor().getFd();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return -1;
    }
Copy the code

FFmpeg layer: Custom IO

Custom read_packet, write_packet, and seek methods

static int read_packet(void *opaque, uint8_t *buf, int buf_size) {
    int fd = *(static_cast<int *>(opaque));
    int ret = read(fd, buf, buf_size);
    if (ret == 0) {
        return AVERROR_EOF;
    }
    return (ret == - 1)?AVERROR(errno) : ret;
}

static int write_packet(void *opaque, uint8_t *buf, int buf_size) {
    int fd = *(static_cast<int *>(opaque));
    int ret = write(fd, buf, buf_size);
    return (ret == - 1)?AVERROR(errno) : ret;
}

static int64_t seek(void *opaque, int64_t offset, int whence) {
    int fd = *(static_cast<int *>(opaque));
    int64_t ret;
    if (whence == AVSEEK_SIZE) {
        struct stat64 st;
        ret = fstat64(fd, &st);
        return ret < 0 ? AVERROR(errno) : (S_ISFIFO(st.st_mode) ? 0 : st.st_size);
    }
    ret = lseek64(fd, offset, whence);
    return ret < 0 ? AVERROR(errno) : ret;
}
Copy the code

Create AVIOContext

AVIOContext *avioContext = avio_alloc_context(buffer, bufferSize, 0, &fd, &read_packet,
                                                  &write_packet,
                                                  &seek);
Copy the code

Set pb and flags

AVFormatContext *avFormatContext = avformat_alloc_context(); . avFormatContext->pb = avioContext; avFormatContext->flags |= AVFMT_FLAG_CUSTOM_IO;Copy the code

FFmpeg decoding process

1. Initialize AVIOContext:avio_alloc_context
2. Initialize AVFormatContext:avformat_alloc_context
3. To set custom I/O:avFormatContext->pb = avioContext; avFormatContext->flags |= AVFMT_FLAG_CUSTOM_IO;
4. Unseal the file and set avFormatContext:avformat_open_input
5. Looking for video tracks:avformat_find_stream_info
6. Get the video stream and decoder:av_find_best_stream
7. Create a decoder context:avcodec_alloc_context3
8. Copy the decoder information obtained in step 6 into AVCodecContext:avcodec_parameters_to_context
9. To open the decoder:avcodec_open2
10. Read the next frame from the stream:av_read_frame
11. Input raw data to the decoder:avcodec_send_packet
12. Get output data from the decoder:avcodec_receive_frame

Repeat 10- >11- >12 until the data is finished.

Fill AndroidBitmap

Transform the pixels memory area of the AndroidBitmap when FFmpeg outputs data from the decoder. Libyuv is used in the library for the corresponding clipping scaling.

1. Calculate hScale and wScale of length and width of original graph and output image respectively.
2. If hScale is equal to wScale, scale directly, otherwise go to the next step.
3. If the minimum value of hScale and wScale is greater than 2, scale the data first and then crop it; otherwise, scale the data first and then scale it.
4. Finally, yuV data is converted into RGB data to fill the target bitmap.

void drawBitmap(AVFrame *frame, int outWidth, int outHeight, uint8_t *data) {
    if(frame->format ! = AV_PIX_FMT_YUV420P) {LOGE("format is not 420 %d", frame->format);
        return;
    }
    if (frame->data[0] = =nullptr || frame->data[1] = =nullptr || frame->data[2] = =nullptr) {
        LOGE("format data is null");
        return;
    }
    int srcW = frame->width;
    int srcH = frame->height;
    if (srcW < 1 || srcH < 1) {
        LOGE("format width %d or height %d not right", srcW, srcH);
        return;
    }
    float wScale = srcW * 1.0 F / outWidth;
    float hScale = srcH * 1.0 F / outHeight;
    int cropWidth;
    int cropHeight;
    int cropX;
    int cropY;
    bool isCropScale = false;
    bool isScaleCrop = false;
    if(wScale ! = hScale) {float s = fmin(wScale, hScale);
        if (s >= 2.0 F) {
            srcW = frame->width / s;
            srcH = frame->height / s;
            if (srcH < outHeight) {
                srcH = outHeight;
            }
            if (srcW < outWidth) {
                srcW = outWidth;
            }
            isScaleCrop = true;
            cropWidth = outWidth;
            cropHeight = outHeight;
        } else {
            isCropScale = true;
            if (srcW > srcH) {
                cropWidth = srcH;
                cropHeight = srcH;
            } else {
                cropWidth = srcW;
                cropHeight = srcW;
            }
        }
        cropX = (srcW - cropWidth) / 2;
        cropY = (srcH - cropHeight) / 2;
        if ((cropX & 0b01) != 0) {
            cropX -= 1;
        }
        if ((cropY & 0b01) != 0) {
            cropY -= 1; }}int halfWidth = outWidth >> 1;
    int outWH = outWidth * outHeight;
    auto *temp = new uint8_t[outWH * 3 / 2];
    uint8_t *temp_u = temp + outWH;
    uint8_t *temp_v = temp + outWH * 5 / 4;

    if (isCropScale) {
        int hw = cropWidth >> 1;
        int wh = cropWidth * cropHeight;
        auto *crop = new uint8_t[wh * 3 / 2];
        uint8_t *crop_u = crop + wh;
        uint8_t *crop_v = crop + wh * 5 / 4;
        uint8_t *src_y = frame->data[0] + (frame->linesize[0] * cropY + cropX);
        uint8_t *src_u = frame->data[1] + frame->linesize[1] * (cropY / 2) + (cropX / 2);
        uint8_t *src_v = frame->data[2] + frame->linesize[2] * (cropY / 2) + (cropX / 2);
        int result = libyuv::I420Rotate(src_y, frame->linesize[0], src_u, frame->linesize[1], src_v,
                                        frame->linesize[2],
                                        crop, cropWidth,
                                        crop_u, hw,
                                        crop_v, hw,
                                        cropWidth, cropHeight, libyuv::kRotate0);
        result = libyuv::I420Scale(
                crop, cropWidth,
                crop_u, hw,
                crop_v, hw,
                cropWidth, cropHeight,
                temp, outWidth,
                temp_u, halfWidth,
                temp_v, halfWidth,
                outWidth, outHeight,
                libyuv::FilterModeEnum::kFilterNone
        );
        delete[]crop;
    } else if (isScaleCrop) {
        int hw = srcW >> 1;
        int wh = srcW * srcH;
        auto *scale = new uint8_t[wh * 3 / 2];
        uint8_t *scale_u = scale + wh;
        uint8_t *scale_v = scale + wh * 5 / 4;
        int result = libyuv::I420Scale(
                frame->data[0], frame->linesize[0],
                frame->data[1], frame->linesize[1],
                frame->data[2], frame->linesize[2],
                frame->width, frame->height,
                scale, srcW,
                scale_u, hw,
                scale_v, hw,
                srcW, srcH,
                libyuv::FilterModeEnum::kFilterNone
        );
        uint8_t *src_y = scale + (srcW * cropY + cropX);
        uint8_t *src_u = scale_u + hw * (cropY / 2) + (cropX / 2);
        uint8_t *src_v = scale_v + hw * (cropY / 2) + (cropX / 2);
        result = libyuv::I420Rotate(src_y, srcW, src_u, hw, src_v, hw,
                                    temp, outWidth,
                                    temp_u, halfWidth,
                                    temp_v, halfWidth,
                                    cropWidth, cropHeight, libyuv::kRotate0);
        delete[]scale;
    } else {
        libyuv::I420Scale(
                frame->data[0], frame->linesize[0],
                frame->data[1], frame->linesize[1],
                frame->data[2], frame->linesize[2],
                frame->width, frame->height,
                temp, outWidth,
                temp_u, halfWidth,
                temp_v, halfWidth,
                outWidth, outHeight,
                libyuv::FilterModeEnum::kFilterNone
        );
    }
    int linesize = outWidth * 4;
    libyuv::I420ToABGR(
            temp, outWidth,//Y
            temp_u, halfWidth,//U
            temp_v, halfWidth,// V
            data, linesize,  // RGBA
            outWidth, outHeight);
    delete[]temp;
}
Copy the code

Known issues

In rare cases, compiled FFmpeg libraries crash at ff_init_vlc_from_lengths, if you know the cause or the solution, please contact 😂.

#00 pc 0010e994 /data/app/com.demo-ld--z-yKfIs5fn0oXW5R-A==/lib/arm64/libffmpeg_core.so(ff_init_vlc_from_lengths) [arm64-v8a::]
Copy the code

Refer to the content

• Linux process with its file descriptor, /proc/self for the current process directory, lsof
• Ffmpeg AVIOContext custom IO and seek
• FFmpeg MEMORY IO mode (memory area as input or output)
• Trace back to the root to solve the mystery of noise and reach the perfect sound quality