dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add AMD support via ort alternative to trt

Browse Source
This commit is contained in:
dtourolle
2026-06-28 11:50:05 +02:00
parent a3ba53ddf7
commit 0ee131a692
27 changed files with 1357 additions and 977 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ffmpeg_decoder.hpp
+116 -41
View File
@@ -4,8 +4,10 @@ extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libavutil/avutil.h>
#include <libavutil/hwcontext.h>
#include <libavutil/imgutils.h>
#include <libavutil/opt.h>
#include <libavutil/pixdesc.h>
#include <libswscale/swscale.h>
}
@@ -14,13 +16,20 @@ extern "C" {
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>
// ── FFmpegDecoder ─────────────────────────────────────────────────────────────
// Seek-and-decode video reader backed by FFmpeg.
//
// Hardware decode priority: NVDEC (_cuvid variants) → CPU software.
// _cuvid decoders output NV12 to system memory directly — no explicit GPU
// frame transfer is needed. swscale converts NV12/YUV → BGR24 for the rest
// Hardware decode is selected at runtime via the generic hwaccel API
// (av_hwdevice_ctx_create): the decoder probes the device types the local
// build supports, in priority order CUDA (NVIDIA) → VAAPI (AMD/Intel) →
// software. This works across GPU vendors without vendor-specific decoder
// names.
//
// Hardware decoders output frames in GPU memory (e.g. AV_PIX_FMT_CUDA,
// AV_PIX_FMT_VAAPI); av_hwframe_transfer_data copies them to a system-memory
// frame (typically NV12), then swscale converts NV12/YUV → BGR24 for the rest
// of the pipeline.
//
// Non-copyable; wrap in unique_ptr if you need to move it.
@@ -40,26 +49,9 @@ struct FFmpegDecoder {
AVStream* stream = fmt_ctx_->streams[stream_idx_];
AVCodecID cid = stream->codecpar->codec_id;
// Try NVDEC first; fall back to software on any failure
if (use_hw) {
if (const AVCodec* hwc = hw_codec_for(cid)) {
codec_ctx_ = avcodec_alloc_context3(hwc);
avcodec_parameters_to_context(codec_ctx_, stream->codecpar);
codec_ctx_->thread_count = 1;
AVDictionary* opts = nullptr;
av_dict_set(&opts, "gpu", "0", 0);
if (avcodec_open2(codec_ctx_, hwc, &opts) >= 0) {
hw_active_ = true;
std::cerr << "[FFmpegDecoder] " << path
<< " codec=" << hwc->name << " (NVDEC)\n";
} else {
avcodec_free_context(&codec_ctx_);
std::cerr << "[FFmpegDecoder] NVDEC init failed, falling back to CPU\n";
}
av_dict_free(&opts);
}
}
// Try hardware backends in priority order; fall back to software.
if (use_hw)
try_open_hw(stream, cid);
if (!hw_active_) {
const AVCodec* swc = avcodec_find_decoder(cid);
@@ -93,6 +85,7 @@ struct FFmpegDecoder {
av_frame_free(&tmp_frame_);
av_packet_free(&pkt_);
avcodec_free_context(&codec_ctx_);
if (hw_device_ctx_) av_buffer_unref(&hw_device_ctx_);
avformat_close_input(&fmt_ctx_);
}
@@ -112,6 +105,10 @@ struct FFmpegDecoder {
bool hw_active() const { return hw_active_; }
const char* codec_name() const { return codec_ctx_ ? codec_ctx_->codec->name : "unknown"; }
// Human-readable backend: "CUDA", "VAAPI", … or "CPU".
const char* hw_backend() const {
return hw_active_ ? av_hwdevice_get_type_name(hw_type_) : "CPU";
}
// Decode the frame at target_sec and return it as BGR cv::Mat.
// Returns an empty Mat at EOF.
@@ -119,7 +116,7 @@ struct FFmpegDecoder {
// Smart seek: if the target is within max_forward_sec_ ahead of the last
// decoded position, decode forward (no seek, no flush). This is dramatically
// faster for sequential sampling because avcodec_flush_buffers + re-init on
// every call is the main bottleneck — especially with NVDEC.
// every call is the main bottleneck — especially with GPU decode.
cv::Mat read_at(double target_sec) {
AVStream* stream = fmt_ctx_->streams[stream_idx_];
int64_t tgt_pts = to_stream_pts(target_sec);
@@ -139,7 +136,7 @@ struct FFmpegDecoder {
}
// Decode forward until we reach or pass target_pts.
// Convert to BGR and unref the AVFrame immediately so NVDEC surfaces
// Convert to BGR and unref the AVFrame immediately so GPU surfaces
// are returned to the pool — holding them causes surface exhaustion
// at higher sample rates.
cv::Mat out;
@@ -161,7 +158,7 @@ struct FFmpegDecoder {
last_pts_ = pts;
if (pts >= tgt_pts)
out = to_bgr(frame_);
av_frame_unref(frame_); // release NVDEC surface immediately
av_frame_unref(frame_); // release GPU surface immediately
if (!out.empty()) break;
}
}
@@ -171,12 +168,15 @@ struct FFmpegDecoder {
private:
AVFormatContext* fmt_ctx_ = nullptr;
AVCodecContext* codec_ctx_ = nullptr;
AVBufferRef* hw_device_ctx_ = nullptr;
AVFrame* frame_ = nullptr;
AVFrame* tmp_frame_ = nullptr;
AVPacket* pkt_ = nullptr;
SwsContext* sws_ctx_ = nullptr;
int stream_idx_ = -1;
bool hw_active_ = false;
AVHWDeviceType hw_type_ = AV_HWDEVICE_TYPE_NONE;
AVPixelFormat hw_pix_fmt_ = AV_PIX_FMT_NONE;
int64_t last_pts_ = AV_NOPTS_VALUE;
int64_t max_forward_pts_ = AV_NOPTS_VALUE; // set after codec opens
@@ -186,26 +186,101 @@ private:
AV_TIME_BASE_Q, s->time_base);
}
static const AVCodec* hw_codec_for(AVCodecID id) {
const char* name = nullptr;
switch (id) {
case AV_CODEC_ID_H264: name = "h264_cuvid"; break;
case AV_CODEC_ID_HEVC: name = "hevc_cuvid"; break;
case AV_CODEC_ID_AV1: name = "av1_cuvid"; break;
case AV_CODEC_ID_MPEG2VIDEO: name = "mpeg2_cuvid"; break;
case AV_CODEC_ID_MPEG4: name = "mpeg4_cuvid"; break;
case AV_CODEC_ID_VC1: name = "vc1_cuvid"; break;
default: return nullptr;
// get_format callback: tell the decoder we want the hardware surface
// format negotiated for this device. The chosen format is stashed on the
// codec context's opaque pointer so this static callback can read it.
static AVPixelFormat get_hw_format(AVCodecContext* ctx,
const AVPixelFormat* fmts) {
auto want = *static_cast<const AVPixelFormat*>(ctx->opaque);
for (const AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; ++p)
if (*p == want) return *p;
std::cerr << "[FFmpegDecoder] hw surface format unavailable, "
"decoder will fall back to software output\n";
return fmts[0];
}
// Probe hardware device types in priority order and open the first that
// works for this codec. Detection is fully at runtime: only device types
// compiled into the local FFmpeg are returned by av_hwdevice_iterate_types,
// and av_hwdevice_ctx_create only succeeds if a usable device is present.
void try_open_hw(AVStream* stream, AVCodecID cid) {
static const AVHWDeviceType kPriority[] = {
AV_HWDEVICE_TYPE_CUDA, // NVIDIA
AV_HWDEVICE_TYPE_VAAPI, // AMD / Intel (Linux)
};
const std::vector<AVHWDeviceType> available = available_hw_types();
const AVCodec* dec = avcodec_find_decoder(cid);
if (!dec) return;
for (AVHWDeviceType type : kPriority) {
bool present = false;
for (AVHWDeviceType a : available) present |= (a == type);
if (!present) continue;
// Find the hw pixel format this decoder advertises for this device.
AVPixelFormat pix = hw_pix_fmt_for(dec, type);
if (pix == AV_PIX_FMT_NONE) continue;
AVBufferRef* dev_ctx = nullptr;
if (av_hwdevice_ctx_create(&dev_ctx, type, nullptr, nullptr, 0) < 0)
continue; // no usable device of this type on the machine
codec_ctx_ = avcodec_alloc_context3(dec);
avcodec_parameters_to_context(codec_ctx_, stream->codecpar);
codec_ctx_->thread_count = 1;
codec_ctx_->hw_device_ctx = av_buffer_ref(dev_ctx);
hw_pix_fmt_ = pix;
codec_ctx_->opaque = &hw_pix_fmt_;
codec_ctx_->get_format = get_hw_format;
if (avcodec_open2(codec_ctx_, dec, nullptr) >= 0) {
hw_active_ = true;
hw_type_ = type;
hw_device_ctx_ = dev_ctx;
std::cerr << "[FFmpegDecoder] codec=" << dec->name
<< " hwaccel=" << av_hwdevice_get_type_name(type)
<< "\n";
return;
}
// This backend failed to open; tear down and try the next.
avcodec_free_context(&codec_ctx_);
av_buffer_unref(&dev_ctx);
hw_pix_fmt_ = AV_PIX_FMT_NONE;
std::cerr << "[FFmpegDecoder] "
<< av_hwdevice_get_type_name(type)
<< " init failed, trying next backend\n";
}
return avcodec_find_decoder_by_name(name);
}
static std::vector<AVHWDeviceType> available_hw_types() {
std::vector<AVHWDeviceType> types;
AVHWDeviceType t = AV_HWDEVICE_TYPE_NONE;
while ((t = av_hwdevice_iterate_types(t)) != AV_HWDEVICE_TYPE_NONE)
types.push_back(t);
return types;
}
// Look up the hw-surface pixel format the decoder exposes for a device type
// (e.g. AV_PIX_FMT_CUDA for CUDA, AV_PIX_FMT_VAAPI for VAAPI).
static AVPixelFormat hw_pix_fmt_for(const AVCodec* dec, AVHWDeviceType type) {
for (int i = 0;; ++i) {
const AVCodecHWConfig* cfg = avcodec_get_hw_config(dec, i);
if (!cfg) break;
if ((cfg->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) &&
cfg->device_type == type)
return cfg->pix_fmt;
}
return AV_PIX_FMT_NONE;
}
cv::Mat to_bgr(AVFrame* src) {
// _cuvid decoders output NV12 to system memory.
// Generic hwaccel would output AV_PIX_FMT_CUDA and need a transfer.
// Hardware decoders hand back GPU surfaces; transfer to system memory.
AVFrame* sw = src;
if (src->format == AV_PIX_FMT_CUDA) {
tmp_frame_->format = AV_PIX_FMT_NV12;
if (src->format == hw_pix_fmt_ && hw_pix_fmt_ != AV_PIX_FMT_NONE) {
av_frame_unref(tmp_frame_);
if (av_hwframe_transfer_data(tmp_frame_, src, 0) < 0) return {};
av_frame_copy_props(tmp_frame_, src);
sw = tmp_frame_;