RK3588 MPP video encoding problem

liwang54321 · December 8, 2022, 4:42pm

I use RK3588 hdmi in interface and use v4l2 API to capture BGR888 raw data and encode the data through mpp API, but the encoded data is not correct, does anyone know why?

#pragma once

#include <rockchip/rk_mpi.h>

#include <functional>
#include <string>
#include <thread>

struct FrameInfo {
    uint32_t height;
    uint32_t width;
    std::string format;
    uint8_t fps;
};

struct StreamInfo {
    std::string StreamType;
    uint32_t gop;
};

struct MppEncInfo {
    int32_t width;
    int32_t height;
    int32_t hor_stride;
    int32_t ver_stride;
    int32_t frame_size;
    int32_t header_size;
    int32_t mdinfo_size;
    int32_t bps;
    MppCodingType code_type;
    MppFrameFormat frame_format;

    MppEncRcMode rc_mode;
    // MppBufferGroup buf_grp;
    // MppBuffer frm_buf;
    // MppBuffer pkt_buf;
    // MppBuffer md_info;
    MppEncSeiMode sei_mode;
    MppEncHeaderMode header_mode;
};

struct StreamPackage {
    uint8_t* data;
    uint32_t len;
    bool is_eos;
};

class VideoEncoder {
public:
    VideoEncoder(const FrameInfo& frame_info, const StreamInfo& stream_info,
        int timeout = -1, bool is_camera_dma = false);
    ~VideoEncoder(void);

    bool Init(const std::function<void(uint8_t* data, uint32_t size)>& package_callback);

    bool PutFrame(uint8_t* data, uint32_t size, int dma_fd = -1);

private:
    static void EncRecvThread(VideoEncoder* self);
    bool SetMppEncCfg(void);

private:
    MppCtx ctx_ = nullptr;
    MppApi* api_ = nullptr;
    MppEncCfg cfg_ = nullptr;
    // MppEncRcCfg rc_cfg_;
    // MppEncPrepCfg prep_cfg_;
    // MppEncCodecCfg codec_cfg_;

    int timeout = -1;
    bool is_camera_dma_ = false;
    FrameInfo frame_info_;
    StreamInfo stream_info_;
    MppEncInfo enc_info_;

    bool is_running_ = false;
    std::thread recv_thread_;
    std::function<void(uint8_t* data, uint32_t size)> package_callback_;
};
#include "VideoEncoder.h"

#include <spdlog/spdlog.h>
#include <magic_enum.hpp>

#define MPP_ALIGN(x, a) (((x) + (a)-1) & ~((a)-1))

VideoEncoder::VideoEncoder(const FrameInfo& frame_info, const StreamInfo& stream_info,
    int timeout, bool is_camera_dma)
    : frame_info_(frame_info)
    , stream_info_(stream_info)
    , is_running_(true)
    , is_camera_dma_(is_camera_dma)
{
}

VideoEncoder::~VideoEncoder(void)
{
    if (ctx_ != nullptr) {
        mpp_destroy(ctx_);
    }
}

static inline MppCodingType AdaptStreamType(std::string_view stream_type)
{
    if (stream_type.compare("H265") == 0) {
        return MPP_VIDEO_CodingHEVC;
    } else if (stream_type.compare("H264") == 0) {
        return MPP_VIDEO_CodingAVC;
    }

    spdlog::error("Unkown support type {}", stream_type);
    return MPP_VIDEO_CodingUnused;
}

static inline MppFrameFormat AdaptFrameType(std::string_view stream_type)
{
    if (stream_type.compare("BGR24") == 0) {
        return MPP_FMT_BGR888;
    }

    spdlog::error("Unkonw Frame Type {}", stream_type);
    return MPP_FMT_BUTT;
}
static inline int GetFrameSize(MppFrameFormat frame_format, int32_t hor_stride, int32_t ver_stride)
{
    int32_t frame_size = 0;
    switch (frame_format & MPP_FRAME_FMT_MASK) {
    case MPP_FMT_YUV420SP:
    case MPP_FMT_YUV420P: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64) * 3 / 2;
    } break;

    case MPP_FMT_YUV422_YUYV:
    case MPP_FMT_YUV422_YVYU:
    case MPP_FMT_YUV422_UYVY:
    case MPP_FMT_YUV422_VYUY:
    case MPP_FMT_YUV422P:
    case MPP_FMT_YUV422SP: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64) * 2;
    } break;
    case MPP_FMT_RGB444:
    case MPP_FMT_BGR444:
    case MPP_FMT_RGB555:
    case MPP_FMT_BGR555:
    case MPP_FMT_RGB565:
    case MPP_FMT_BGR565:
    case MPP_FMT_RGB888:
    case MPP_FMT_BGR888:
    case MPP_FMT_RGB101010:
    case MPP_FMT_BGR101010:
    case MPP_FMT_ARGB8888:
    case MPP_FMT_ABGR8888:
    case MPP_FMT_BGRA8888:
    case MPP_FMT_RGBA8888: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64);
    } break;

    default: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64) * 4;
    } break;
    }
    return frame_size;
}

#define SZ_1K (1024)
#define SZ_2K (SZ_1K * 2)
#define SZ_4K (SZ_1K * 4)
static inline int GetHeaderSize(MppFrameFormat frame_format, uint32_t width, uint32_t height)
{
    int header_size = 0;
    if (MPP_FRAME_FMT_IS_FBC(frame_format)) {
        if ((frame_format & MPP_FRAME_FBC_MASK) == MPP_FRAME_FBC_AFBC_V1)
            header_size = MPP_ALIGN(MPP_ALIGN(width, 16) * MPP_ALIGN(height, 16) / 16, SZ_4K);
        else
            header_size = MPP_ALIGN(width, 16) * MPP_ALIGN(height, 16) / 16;
    } else {
        header_size = 0;
    }
    return header_size;
}

bool VideoEncoder::SetMppEncCfg(void)
{
    mpp_enc_cfg_set_s32(cfg_, "prep:width", enc_info_.width);
    mpp_enc_cfg_set_s32(cfg_, "prep:height", enc_info_.height);
    mpp_enc_cfg_set_s32(cfg_, "prep:hor_stride", enc_info_.hor_stride);
    mpp_enc_cfg_set_s32(cfg_, "prep:ver_stride", enc_info_.ver_stride);
    mpp_enc_cfg_set_s32(cfg_, "prep:format", enc_info_.frame_format);

    mpp_enc_cfg_set_s32(cfg_, "rc:mode", enc_info_.rc_mode);

    /* fix input / output frame rate */
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_in_flex", 0);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_in_num", frame_info_.fps);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_in_denorm", 1);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_out_flex", 0);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_out_num", frame_info_.fps);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_out_denorm", 1);
    mpp_enc_cfg_set_s32(cfg_, "rc:gop", stream_info_.gop ? stream_info_.gop : frame_info_.fps * 2);

    /* drop frame or not when bitrate overflow */
    mpp_enc_cfg_set_u32(cfg_, "rc:drop_mode", MPP_ENC_RC_DROP_FRM_DISABLED);
    mpp_enc_cfg_set_u32(cfg_, "rc:drop_thd", 20); /* 20% of max bps */
    mpp_enc_cfg_set_u32(cfg_, "rc:drop_gap", 1); /* Do not continuous drop frame */

    /* setup bitrate for different rc_mode */
    mpp_enc_cfg_set_s32(cfg_, "rc:bps_target", enc_info_.bps);
    switch (enc_info_.rc_mode) {
    case MPP_ENC_RC_MODE_FIXQP: {
        /* do not setup bitrate on FIXQP mode */
    } break;
    case MPP_ENC_RC_MODE_CBR: {
        /* CBR mode has narrow bound */
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_max", enc_info_.bps * 17 / 16);
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_min", enc_info_.bps * 15 / 16);
    } break;
    case MPP_ENC_RC_MODE_VBR:
    case MPP_ENC_RC_MODE_AVBR: {
        /* VBR mode has wide bound */
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_max", enc_info_.bps * 17 / 16);
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_min", enc_info_.bps * 1 / 16);
    } break;
    default: {
        /* default use CBR mode */
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_max", enc_info_.bps * 17 / 16);
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_min", enc_info_.bps * 15 / 16);
    } break;
    }

    /* setup qp for different codec and rc_mode */
    switch (enc_info_.code_type) {
    case MPP_VIDEO_CodingAVC:
    case MPP_VIDEO_CodingHEVC: {
        switch (enc_info_.rc_mode) {
        case MPP_ENC_RC_MODE_FIXQP: {
            RK_S32 fix_qp = 0;
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_init", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max_i", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min_i", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_ip", 0);
        } break;
        case MPP_ENC_RC_MODE_CBR:
        case MPP_ENC_RC_MODE_VBR:
        case MPP_ENC_RC_MODE_AVBR: {
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_init", -1);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max", 51);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min", 10);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max_i", 51);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min_i", 10);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_ip", 2);
        } break;
        default: {
            spdlog::error("unsupport encoder rc mode {}", enc_info_.rc_mode);
        } break;
        }
    } break;
    case MPP_VIDEO_CodingVP8: {
        /* vp8 only setup base qp range */
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_init", 40);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_max", 127);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_min", 0);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_max_i", 127);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_min_i", 0);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_ip", 6);
    } break;
    case MPP_VIDEO_CodingMJPEG: {
        /* jpeg use special codec config to control qtable */
        mpp_enc_cfg_set_s32(cfg_, "jpeg:q_factor", 80);
        mpp_enc_cfg_set_s32(cfg_, "jpeg:qf_max", 99);
        mpp_enc_cfg_set_s32(cfg_, "jpeg:qf_min", 1);
    } break;
    default: {
    } break;
    }

    /* setup codec  */
    mpp_enc_cfg_set_s32(cfg_, "codec:type", enc_info_.code_type);
    switch (enc_info_.code_type) {
    case MPP_VIDEO_CodingAVC: {
        RK_U32 constraint_set;
        /*
         * H.264 profile_idc parameter
         * 66  - Baseline profile
         * 77  - Main profile
         * 100 - High profile
         */
        mpp_enc_cfg_set_s32(cfg_, "h264:profile", 100);
        /*
         * H.264 level_idc parameter
         * 10 / 11 / 12 / 13    - qcif@15fps / cif@7.5fps / cif@15fps / cif@30fps
         * 20 / 21 / 22         - cif@30fps / half-D1@@25fps / D1@12.5fps
         * 30 / 31 / 32         - D1@25fps / 720p@30fps / 720p@60fps
         * 40 / 41 / 42         - 1080p@30fps / 1080p@30fps / 1080p@60fps
         * 50 / 51 / 52         - 4K@30fps
         */
        mpp_enc_cfg_set_s32(cfg_, "h264:level", 40);
        mpp_enc_cfg_set_s32(cfg_, "h264:cabac_en", 1);
        mpp_enc_cfg_set_s32(cfg_, "h264:cabac_idc", 0);
        mpp_enc_cfg_set_s32(cfg_, "h264:trans8x8", 1);

        // mpp_env_get_u32("constraint_set", &constraint_set, 0);
        // if (constraint_set & 0x3f0000)
        //     mpp_enc_cfg_set_s32(cfg_, "h264:constraint_set", constraint_set);
    } break;
    case MPP_VIDEO_CodingHEVC:
    case MPP_VIDEO_CodingMJPEG:
    case MPP_VIDEO_CodingVP8: {
    } break;
    default: {
        spdlog::error("unsupport encoder coding type {}", enc_info_.code_type);
    } break;
    }

    // p->split_mode = 0;
    // p->split_arg = 0;
    // p->split_out = 0;

    // mpp_env_get_u32("split_mode", &p->split_mode, MPP_ENC_SPLIT_NONE);
    // mpp_env_get_u32("split_arg", &p->split_arg, 0);
    // mpp_env_get_u32("split_out", &p->split_out, 0);

    // if (p->split_mode) {
    //     mpp_log_q(quiet, "%p split mode %d arg %d out %d\n", ctx,
    //         p->split_mode, p->split_arg, p->split_out);
    //     mpp_enc_cfg_set_s32(cfg_, "split:mode", p->split_mode);
    //     mpp_enc_cfg_set_s32(cfg_, "split:arg", p->split_arg);
    //     mpp_enc_cfg_set_s32(cfg_, "split:out", p->split_out);
    // }

    // mpp_env_get_u32("mirroring", &mirroring, 0);
    // mpp_env_get_u32("rotation", &rotation, 0);
    // mpp_env_get_u32("flip", &flip, 0);

    // mpp_enc_cfg_set_s32(cfg_, "prep:mirroring", mirroring);
    // mpp_enc_cfg_set_s32(cfg_, "prep:rotation", rotation);
    // mpp_enc_cfg_set_s32(cfg_, "prep:flip", flip);

    auto ret = api_->control(ctx_, MPP_ENC_SET_CFG, cfg_);
    if (ret) {
        spdlog::error("mpi control enc set cfg failed ret {}", magic_enum::enum_name(ret));
        return false;
    }
    return true;
}

bool VideoEncoder::Init(const std::function<void(uint8_t* data, uint32_t size)>& package_callback)
{
    MPP_RET ret = MPP_SUCCESS;
    enc_info_.width = frame_info_.width;
    enc_info_.height = frame_info_.height;
    enc_info_.code_type = AdaptStreamType(stream_info_.StreamType);
    enc_info_.frame_format = AdaptFrameType(frame_info_.format);
    enc_info_.hor_stride = MPP_ALIGN(frame_info_.width, 16);
    enc_info_.ver_stride = MPP_ALIGN(frame_info_.height, 16);
    enc_info_.frame_size = GetFrameSize(enc_info_.frame_format, enc_info_.hor_stride, enc_info_.ver_stride);
    enc_info_.header_size = GetHeaderSize(enc_info_.frame_format, frame_info_.width, frame_info_.height);
    enc_info_.mdinfo_size = (MPP_VIDEO_CodingHEVC == enc_info_.code_type) ? (MPP_ALIGN(enc_info_.hor_stride, 32) >> 5) * (MPP_ALIGN(enc_info_.ver_stride, 32) >> 5) * 16 : (MPP_ALIGN(enc_info_.hor_stride, 64) >> 6) * (MPP_ALIGN(enc_info_.ver_stride, 16) >> 4) * 16;
    enc_info_.bps = frame_info_.width * frame_info_.height / 8 * frame_info_.fps;
    enc_info_.rc_mode = MPP_ENC_RC_MODE_VBR;
    // auto ret = mpp_buffer_group_get_internal(&enc_info_.buf_grp, MPP_BUFFER_TYPE_DRM);
    // if (ret) {
    //     spdlog::error("failed to get mpp buffer group ret {}", ret);
    //     return false;
    // }

    // ret = mpp_buffer_get(enc_info_.buf_grp, &enc_info_.frm_buf, enc_info_.frame_size + enc_info_.header_size);
    // if (ret) {
    //     spdlog::error("failed to get buffer for input frame ret {}", ret);
    //     return false;
    // }

    // ret = mpp_buffer_get(enc_info_.buf_grp, &enc_info_.pkt_buf, enc_info_.frame_size);
    // if (ret) {
    //     spdlog::error("failed to get buffer for output packet ret {}", ret);
    //     return false;
    // }

    // ret = mpp_buffer_get(enc_info_.buf_grp, &enc_info_.md_info, enc_info_.mdinfo_size);
    // if (ret) {
    //     spdlog::error("failed to get buffer for motion info output packet ret {}", ret);
    //     return false;
    // }

    ret = mpp_check_support_format(MPP_CTX_ENC, enc_info_.code_type);
    if (ret == MPP_SUCCESS) {
        spdlog::info("Mpp Support {}", stream_info_.StreamType);
    } else {
        spdlog::error("Mpp Not Support {}", stream_info_.StreamType);
        return false;
    }

    ret = mpp_create(&ctx_, &api_);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Create mpp error {}", ret);
        return false;
    }

    spdlog::info("encoder start w {} h {} type {}",
        frame_info_.width, frame_info_.height, frame_info_.format);
    MppPollType timeout = MPP_POLL_NON_BLOCK;

    ret = api_->control(ctx_, MPP_SET_INPUT_TIMEOUT, &timeout);
    if (ret != MPP_SUCCESS) {
        spdlog::info("mpi control set input timeout {} ret {}", timeout, ret);
        return false;
    }
    timeout = MPP_POLL_BLOCK;
    ret = api_->control(ctx_, MPP_SET_OUTPUT_TIMEOUT, &timeout);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Mpp Set output time out error {}", ret);
        return false;
    }

    ret = mpp_init(ctx_, MPP_CTX_ENC, enc_info_.code_type);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Mpp Init error {}", ret);
        return false;
    }

    ret = mpp_enc_cfg_init(&cfg_);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Mpp enc_cfg_init error {}", ret);
        return false;
    }

    ret = api_->control(ctx_, MPP_ENC_GET_CFG, cfg_);
    if (ret) {
        spdlog::error("get enc cfg failed ret {}", ret);
        return false;
    }

    if(!SetMppEncCfg()) {
        return false;
    }

    package_callback_ = package_callback;
    recv_thread_ = std::thread(&VideoEncoder::EncRecvThread, this);

    return true;
}

#include <time.h>

static inline RK_S64 mpp_time()
{
    struct timespec time = {0, 0};
    clock_gettime(CLOCK_MONOTONIC, &time);
    return (RK_S64)time.tv_sec * 1000000 + (RK_S64)time.tv_nsec / 1000;
}

void VideoEncoder::EncRecvThread(VideoEncoder* self)
{
    MppPacket packet = NULL;
    RK_S64 last_pkt_time = 0;
    RK_S64 first_pkt_time = 0;
    RK_U32 eoi = 1;
    // std::shared_ptr<StreamPackage> stream_package;
    while (self->is_running_) {
        auto ret = self->api_->encode_get_packet(self->ctx_, &packet);
        if (ret || NULL == packet) {
            spdlog::error("Get Package error, {}", ret);
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
            continue;
        }
        last_pkt_time = mpp_time();

        auto data = (uint8_t*)mpp_packet_get_pos(packet);
        auto len = mpp_packet_get_length(packet);
        
        if(first_pkt_time == 0)
            first_pkt_time = mpp_time();

        auto pkt_eos = mpp_packet_get_eos(packet);
        /* for low delay partition encoding */
        if (mpp_packet_is_partition(packet)) {
            eoi = mpp_packet_is_eoi(packet);
        }

        self->package_callback_(data, len);

        ret = mpp_packet_deinit(&packet);
        assert(ret == MPP_SUCCESS);
    }
}

bool VideoEncoder::PutFrame(uint8_t* data, uint32_t size, int dma_fd)
{   
    MppBuffer cam_buf = NULL;
    MppFrame frame = nullptr;

    auto ret = mpp_frame_init(&frame);
    if (ret) {
        spdlog::error("mpp_frame_init failed {}", ret);
        return false;
    }

    mpp_frame_set_width(frame, enc_info_.width);
    mpp_frame_set_height(frame, enc_info_.height);
    mpp_frame_set_hor_stride(frame, enc_info_.hor_stride);
    mpp_frame_set_ver_stride(frame, enc_info_.ver_stride);
    mpp_frame_set_fmt(frame, enc_info_.frame_format);
    mpp_frame_set_eos(frame, 0);


    MppBufferInfo info;
    memset(&info, 0, sizeof(MppBufferInfo));
    info.type = MPP_BUFFER_TYPE_EXT_DMA;
    info.fd =  dma_fd;
    info.size = size & 0x07ffffff;
    info.index = (size & 0xf8000000) >> 27;
    mpp_buffer_import(&cam_buf, &info);

    mpp_frame_set_buffer(frame, cam_buf);

    ret = api_->encode_put_frame(ctx_, frame);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Encode frame error {}", ret);
        return false;
    }

    return true;
}

NovemberRain · January 23, 2024, 10:04am

Hello!

Try to change hor_stride to this for BGR24:

enc_info_.hor_stride = MPP_ALIGN(frame_info_.width * 3, 16);

liwang54321 · January 26, 2024, 4:07pm

Hi @NovemberRain
Thank your support, the problem has been solved