dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: scene-actor-extraction pipeline

Browse Source 
Source (KPN++ pipeline nodes, ArcFace embedders, SCRFD/YuNet detectors,
gallery builder), build scripts, and eval artifacts.

- external/KPN as a git submodule (gitea.tourolle.paris/dtourolle/KPN)
- ONNX models tracked via Git LFS (models/*.onnx)
- generated outputs, TensorRT engines, reference repos, and media ignored
This commit is contained in:
dtourolle
2026-06-12 15:29:01 +02:00
commit d753062c6c
50 changed files with 10100 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ffmpeg_decoder.hpp
+231
View File
@@ -0,0 +1,231 @@
#pragma once
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libavutil/opt.h>
#include <libswscale/swscale.h>
}
#include <opencv2/core.hpp>
#include <iostream>
#include <stdexcept>
#include <string>
// ── FFmpegDecoder ─────────────────────────────────────────────────────────────
// Seek-and-decode video reader backed by FFmpeg.
//
// Hardware decode priority: NVDEC (_cuvid variants) → CPU software.
// _cuvid decoders output NV12 to system memory directly — no explicit GPU
// frame transfer is needed. swscale converts NV12/YUV → BGR24 for the rest
// of the pipeline.
//
// Non-copyable; wrap in unique_ptr if you need to move it.
struct FFmpegDecoder {
explicit FFmpegDecoder(const std::string& path, bool use_hw = true) {
if (avformat_open_input(&fmt_ctx_, path.c_str(), nullptr, nullptr) < 0)
throw std::runtime_error("[FFmpegDecoder] cannot open: " + path);
if (avformat_find_stream_info(fmt_ctx_, nullptr) < 0)
throw std::runtime_error("[FFmpegDecoder] stream info failed");
stream_idx_ = av_find_best_stream(
fmt_ctx_, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
if (stream_idx_ < 0)
throw std::runtime_error("[FFmpegDecoder] no video stream");
AVStream* stream = fmt_ctx_->streams[stream_idx_];
AVCodecID cid = stream->codecpar->codec_id;
// Try NVDEC first; fall back to software on any failure
if (use_hw) {
if (const AVCodec* hwc = hw_codec_for(cid)) {
codec_ctx_ = avcodec_alloc_context3(hwc);
avcodec_parameters_to_context(codec_ctx_, stream->codecpar);
codec_ctx_->thread_count = 1;
AVDictionary* opts = nullptr;
av_dict_set(&opts, "gpu", "0", 0);
if (avcodec_open2(codec_ctx_, hwc, &opts) >= 0) {
hw_active_ = true;
std::cerr << "[FFmpegDecoder] " << path
<< " codec=" << hwc->name << " (NVDEC)\n";
} else {
avcodec_free_context(&codec_ctx_);
std::cerr << "[FFmpegDecoder] NVDEC init failed, falling back to CPU\n";
}
av_dict_free(&opts);
}
}
if (!hw_active_) {
const AVCodec* swc = avcodec_find_decoder(cid);
if (!swc) throw std::runtime_error("[FFmpegDecoder] no software decoder");
codec_ctx_ = avcodec_alloc_context3(swc);
avcodec_parameters_to_context(codec_ctx_, stream->codecpar);
codec_ctx_->thread_count = 0; // auto: use all cores
if (avcodec_open2(codec_ctx_, swc, nullptr) < 0)
throw std::runtime_error("[FFmpegDecoder] cannot open codec");
std::cerr << "[FFmpegDecoder] " << path
<< " codec=" << swc->name << " (CPU)\n";
}
frame_ = av_frame_alloc();
tmp_frame_= av_frame_alloc();
pkt_ = av_packet_alloc();
// Decode forward (no seek) when target is within this many pts units.
// 5 seconds covers typical H.264/H.265 GOP sizes.
max_forward_pts_ = to_stream_pts(5.0);
double total = duration_sec();
double vfps = fps();
std::cerr << "[FFmpegDecoder] duration=" << total
<< "s fps=" << vfps << "\n";
}
~FFmpegDecoder() {
if (sws_ctx_) sws_freeContext(sws_ctx_);
av_frame_free(&frame_);
av_frame_free(&tmp_frame_);
av_packet_free(&pkt_);
avcodec_free_context(&codec_ctx_);
avformat_close_input(&fmt_ctx_);
}
FFmpegDecoder(const FFmpegDecoder&) = delete;
FFmpegDecoder& operator=(const FFmpegDecoder&) = delete;
double duration_sec() const {
if (!fmt_ctx_ || fmt_ctx_->duration == AV_NOPTS_VALUE) return 0.0;
return static_cast<double>(fmt_ctx_->duration) / AV_TIME_BASE;
}
double fps() const {
AVStream* s = fmt_ctx_->streams[stream_idx_];
if (s->avg_frame_rate.den == 0) return 25.0;
return av_q2d(s->avg_frame_rate);
}
bool hw_active() const { return hw_active_; }
const char* codec_name() const { return codec_ctx_ ? codec_ctx_->codec->name : "unknown"; }
// Decode the frame at target_sec and return it as BGR cv::Mat.
// Returns an empty Mat at EOF.
//
// Smart seek: if the target is within max_forward_sec_ ahead of the last
// decoded position, decode forward (no seek, no flush). This is dramatically
// faster for sequential sampling because avcodec_flush_buffers + re-init on
// every call is the main bottleneck — especially with NVDEC.
cv::Mat read_at(double target_sec) {
AVStream* stream = fmt_ctx_->streams[stream_idx_];
int64_t tgt_pts = to_stream_pts(target_sec);
// Decide: seek or decode forward?
bool need_seek = (last_pts_ == AV_NOPTS_VALUE) ||
(tgt_pts < last_pts_) ||
(tgt_pts - last_pts_ > max_forward_pts_);
if (need_seek) {
if (av_seek_frame(fmt_ctx_, stream_idx_, tgt_pts, AVSEEK_FLAG_BACKWARD) < 0)
av_seek_frame(fmt_ctx_, -1,
static_cast<int64_t>(target_sec * AV_TIME_BASE),
AVSEEK_FLAG_BACKWARD);
avcodec_flush_buffers(codec_ctx_);
last_pts_ = AV_NOPTS_VALUE;
}
// Decode forward until we reach or pass target_pts.
// Convert to BGR and unref the AVFrame immediately so NVDEC surfaces
// are returned to the pool — holding them causes surface exhaustion
// at higher sample rates.
cv::Mat out;
while (out.empty()) {
int ret = av_read_frame(fmt_ctx_, pkt_);
if (ret == AVERROR_EOF || ret < 0) break;
if (pkt_->stream_index != stream_idx_) {
av_packet_unref(pkt_);
continue;
}
avcodec_send_packet(codec_ctx_, pkt_);
av_packet_unref(pkt_);
while (avcodec_receive_frame(codec_ctx_, frame_) == 0) {
int64_t pts = frame_->best_effort_timestamp;
if (pts == AV_NOPTS_VALUE) pts = frame_->pts;
last_pts_ = pts;
if (pts >= tgt_pts)
out = to_bgr(frame_);
av_frame_unref(frame_); // release NVDEC surface immediately
if (!out.empty()) break;
}
}
return out;
}
private:
AVFormatContext* fmt_ctx_ = nullptr;
AVCodecContext* codec_ctx_ = nullptr;
AVFrame* frame_ = nullptr;
AVFrame* tmp_frame_ = nullptr;
AVPacket* pkt_ = nullptr;
SwsContext* sws_ctx_ = nullptr;
int stream_idx_ = -1;
bool hw_active_ = false;
int64_t last_pts_ = AV_NOPTS_VALUE;
int64_t max_forward_pts_ = AV_NOPTS_VALUE; // set after codec opens
int64_t to_stream_pts(double sec) const {
AVStream* s = fmt_ctx_->streams[stream_idx_];
return av_rescale_q(static_cast<int64_t>(sec * AV_TIME_BASE),
AV_TIME_BASE_Q, s->time_base);
}
static const AVCodec* hw_codec_for(AVCodecID id) {
const char* name = nullptr;
switch (id) {
case AV_CODEC_ID_H264: name = "h264_cuvid"; break;
case AV_CODEC_ID_HEVC: name = "hevc_cuvid"; break;
case AV_CODEC_ID_AV1: name = "av1_cuvid"; break;
case AV_CODEC_ID_MPEG2VIDEO: name = "mpeg2_cuvid"; break;
case AV_CODEC_ID_MPEG4: name = "mpeg4_cuvid"; break;
case AV_CODEC_ID_VC1: name = "vc1_cuvid"; break;
default: return nullptr;
}
return avcodec_find_decoder_by_name(name);
}
cv::Mat to_bgr(AVFrame* src) {
// _cuvid decoders output NV12 to system memory.
// Generic hwaccel would output AV_PIX_FMT_CUDA and need a transfer.
AVFrame* sw = src;
if (src->format == AV_PIX_FMT_CUDA) {
tmp_frame_->format = AV_PIX_FMT_NV12;
if (av_hwframe_transfer_data(tmp_frame_, src, 0) < 0) return {};
av_frame_copy_props(tmp_frame_, src);
sw = tmp_frame_;
}
const int w = sw->width;
const int h = sw->height;
sws_ctx_ = sws_getCachedContext(sws_ctx_,
w, h, static_cast<AVPixelFormat>(sw->format),
w, h, AV_PIX_FMT_BGR24,
SWS_BILINEAR, nullptr, nullptr, nullptr);
if (!sws_ctx_) return {};
cv::Mat out(h, w, CV_8UC3);
uint8_t* dst_data[1] = { out.data };
int dst_linesize[1] = { static_cast<int>(out.step) };
sws_scale(sws_ctx_,
sw->data, sw->linesize, 0, h,
dst_data, dst_linesize);
return out;
}
};