From 1d48d6af90b926d75d26831a2898f9e5fc378b81 Mon Sep 17 00:00:00 2001 From: salhiraid <134926431+salhiraid@users.noreply.github.com> Date: Wed, 15 Apr 2026 17:00:55 +0200 Subject: [PATCH 1/4] Integrate BEV rendering via BirdEyeViewTracker subclass --- deploy/TensorRT/cpp/BEV_INTEGRATION.md | 34 +++ deploy/TensorRT/cpp/include/BYTETracker.h | 39 +-- .../cpp/include/BirdEyeViewRenderer.h | 73 +++++ .../TensorRT/cpp/include/BirdEyeViewTracker.h | 31 ++ deploy/TensorRT/cpp/include/STrack.h | 14 +- deploy/TensorRT/cpp/src/BYTETracker.cpp | 22 +- .../TensorRT/cpp/src/BirdEyeViewRenderer.cpp | 264 ++++++++++++++++++ .../TensorRT/cpp/src/BirdEyeViewTracker.cpp | 56 ++++ deploy/TensorRT/cpp/src/STrack.cpp | 30 +- deploy/TensorRT/cpp/src/bytetrack.cpp | 4 +- 10 files changed, 523 insertions(+), 44 deletions(-) create mode 100644 deploy/TensorRT/cpp/BEV_INTEGRATION.md create mode 100644 deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h create mode 100644 deploy/TensorRT/cpp/include/BirdEyeViewTracker.h create mode 100644 deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp create mode 100644 deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp diff --git a/deploy/TensorRT/cpp/BEV_INTEGRATION.md b/deploy/TensorRT/cpp/BEV_INTEGRATION.md new file mode 100644 index 000000000..8d419f542 --- /dev/null +++ b/deploy/TensorRT/cpp/BEV_INTEGRATION.md @@ -0,0 +1,34 @@ +# BirdEyeViewTracker integration example + +The cleanest integration is to use `BirdEyeViewTracker`, a child class of `BYTETracker`. +It calls the original ByteTrack update logic, then renders **both tracked and lost tracks** +into BEV for each frame. + +```cpp +#include "BirdEyeViewTracker.h" + +// once before the frame loop +BirdEyeViewTracker tracker( + fps, + 30, + "bev_outputs", // output directory + -30.0f, // configurable y min [m] + 60.0f, // configurable y max [m] + 1000, + 1000); + +// in the frame loop (same call pattern as BYTETracker) +vector output_stracks = tracker.update(objects); +``` + +## Passing class and ground-point (`gp`) to ByteTrack tracks + +`Object` now includes: + +- `label` (class id) +- `gp` (`cv::Point3f`) for world coordinates `(x, y, z)` where `z = 0` + +When detections are converted to `STrack`, class and `gp` are copied, so tracked/lost states +carry BEV information without changing ByteTrack association/matching behavior. + +If your detector does not yet provide `gp`, set it before calling `update(...)`. diff --git a/deploy/TensorRT/cpp/include/BYTETracker.h b/deploy/TensorRT/cpp/include/BYTETracker.h index 112480050..df8278fdb 100644 --- a/deploy/TensorRT/cpp/include/BYTETracker.h +++ b/deploy/TensorRT/cpp/include/BYTETracker.h @@ -2,23 +2,26 @@ #include "STrack.h" -struct Object -{ - cv::Rect_ rect; - int label; - float prob; -}; - -class BYTETracker -{ -public: - BYTETracker(int frame_rate = 30, int track_buffer = 30); - ~BYTETracker(); - - vector update(const vector& objects); - Scalar get_color(int idx); - -private: +struct Object +{ + cv::Rect_ rect; + int label; + float prob; + cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F); +}; + +class BYTETracker +{ +public: + BYTETracker(int frame_rate = 30, int track_buffer = 30); + virtual ~BYTETracker(); + + virtual vector update(const vector& objects); + Scalar get_color(int idx); + const vector& get_tracked_stracks() const; + const vector& get_lost_stracks() const; + +private: vector joint_stracks(vector &tlista, vector &tlistb); vector joint_stracks(vector &tlista, vector &tlistb); @@ -46,4 +49,4 @@ class BYTETracker vector lost_stracks; vector removed_stracks; byte_kalman::KalmanFilter kalman_filter; -}; \ No newline at end of file +}; diff --git a/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h b/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h new file mode 100644 index 000000000..dbee96443 --- /dev/null +++ b/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h @@ -0,0 +1,73 @@ +#pragma once + +#include + +#include +#include +#include + +#include "STrack.h" + +struct BEVTrackPoint { + int track_id = -1; + int class_id = -1; + std::string class_label; + cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F); + bool is_active = true; + int track_state = TrackState::Tracked; +}; + +struct BEVRenderedTrackInfo { + int frame_index = -1; + int track_id = -1; + int class_id = -1; + std::string class_label; + cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F); + cv::Point2i bev_pixel = cv::Point2i(-1, -1); + bool inside_bev = false; + int track_state = TrackState::Tracked; +}; + +class BirdEyeViewRenderer { +public: + BirdEyeViewRenderer(const std::string& output_dir, + float y_min_m, + float y_max_m, + int canvas_width = 900, + int canvas_height = 900, + int padding_px = 50, + int grid_step_m = 10); + + void renderFrame(const std::vector& tracks, int frame_index); + + cv::Point2i worldToImage(float x_m, float y_m, bool* inside = nullptr) const; + + const std::vector& latestRenderedTracks() const; + +private: + struct Range { + float min_x; + float max_x; + float min_y; + float max_y; + }; + + bool ensureOutputDirectory(const std::string& dir) const; + std::string frameOutputPath(int frame_index) const; + cv::Mat buildCanvas() const; + void drawGridAndAxes(cv::Mat& canvas) const; + void drawTrackPoint(cv::Mat& canvas, const BEVTrackPoint& track, int frame_index); + cv::Scalar getTrackColor(int track_id); + std::string resolveClassText(const BEVTrackPoint& track) const; + +private: + std::string output_dir_; + Range world_range_; + int canvas_width_; + int canvas_height_; + int padding_px_; + int grid_step_m_; + + std::unordered_map track_color_table_; + std::vector latest_rendered_tracks_; +}; diff --git a/deploy/TensorRT/cpp/include/BirdEyeViewTracker.h b/deploy/TensorRT/cpp/include/BirdEyeViewTracker.h new file mode 100644 index 000000000..68e2737e6 --- /dev/null +++ b/deploy/TensorRT/cpp/include/BirdEyeViewTracker.h @@ -0,0 +1,31 @@ +#pragma once + +#include "BYTETracker.h" +#include "BirdEyeViewRenderer.h" + +class BirdEyeViewTracker : public BYTETracker { +public: + BirdEyeViewTracker(int frame_rate, + int track_buffer, + const std::string& bev_output_dir, + float y_min_m, + float y_max_m, + int canvas_width = 900, + int canvas_height = 900, + int padding_px = 50, + int grid_step_m = 10); + + vector update(const vector& objects) override; + + const BirdEyeViewRenderer& renderer() const; + BirdEyeViewRenderer& renderer(); + +private: + void appendTracksToBevInput(const vector& tracks, + int expected_state, + vector& bev_tracks) const; + +private: + BirdEyeViewRenderer bev_renderer_; + int bev_frame_index_; +}; diff --git a/deploy/TensorRT/cpp/include/STrack.h b/deploy/TensorRT/cpp/include/STrack.h index 4bd9d000c..f307cb128 100644 --- a/deploy/TensorRT/cpp/include/STrack.h +++ b/deploy/TensorRT/cpp/include/STrack.h @@ -10,8 +10,8 @@ enum TrackState { New = 0, Tracked, Lost, Removed }; class STrack { -public: - STrack(vector tlwh_, float score); +public: + STrack(vector tlwh_, float score, int class_id = -1, cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F)); ~STrack(); vector static tlbr_to_tlwh(vector &tlbr); @@ -43,8 +43,10 @@ class STrack KAL_MEAN mean; KAL_COVA covariance; - float score; - -private: + float score; + int class_id; + cv::Point3f gp; + +private: byte_kalman::KalmanFilter kalman_filter; -}; \ No newline at end of file +}; diff --git a/deploy/TensorRT/cpp/src/BYTETracker.cpp b/deploy/TensorRT/cpp/src/BYTETracker.cpp index 7e036e996..c3191631c 100644 --- a/deploy/TensorRT/cpp/src/BYTETracker.cpp +++ b/deploy/TensorRT/cpp/src/BYTETracker.cpp @@ -12,9 +12,9 @@ BYTETracker::BYTETracker(int frame_rate, int track_buffer) cout << "Init ByteTrack!" << endl; } -BYTETracker::~BYTETracker() -{ -} +BYTETracker::~BYTETracker() +{ +} vector BYTETracker::update(const vector& objects) { @@ -51,7 +51,7 @@ vector BYTETracker::update(const vector& objects) float score = objects[i].prob; - STrack strack(STrack::tlbr_to_tlwh(tlbr_), score); + STrack strack(STrack::tlbr_to_tlwh(tlbr_), score, objects[i].label, objects[i].gp); if (score >= track_thresh) { detections.push_back(strack); @@ -237,5 +237,15 @@ vector BYTETracker::update(const vector& objects) output_stracks.push_back(this->tracked_stracks[i]); } } - return output_stracks; -} \ No newline at end of file + return output_stracks; +} + +const vector& BYTETracker::get_tracked_stracks() const +{ + return tracked_stracks; +} + +const vector& BYTETracker::get_lost_stracks() const +{ + return lost_stracks; +} diff --git a/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp b/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp new file mode 100644 index 000000000..d4a5fca0c --- /dev/null +++ b/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp @@ -0,0 +1,264 @@ +#include "BirdEyeViewRenderer.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { +cv::Scalar hsvToBgr(float h_deg, float s, float v) { + cv::Mat hsv(1, 1, CV_32FC3, cv::Scalar(h_deg, s, v)); + cv::Mat bgr; + cv::cvtColor(hsv, bgr, cv::COLOR_HSV2BGR); + const cv::Vec3f color = bgr.at(0, 0); + return cv::Scalar(color[0] * 255.0F, color[1] * 255.0F, color[2] * 255.0F); +} + +float clamp01(float v) { + return std::max(0.0F, std::min(1.0F, v)); +} +} // namespace + +BirdEyeViewRenderer::BirdEyeViewRenderer(const std::string& output_dir, + float y_min_m, + float y_max_m, + int canvas_width, + int canvas_height, + int padding_px, + int grid_step_m) + : output_dir_(output_dir), + canvas_width_(canvas_width), + canvas_height_(canvas_height), + padding_px_(padding_px), + grid_step_m_(grid_step_m) { + world_range_.min_x = -40.0F; + world_range_.max_x = 50.0F; + world_range_.min_y = std::min(y_min_m, y_max_m); + world_range_.max_y = std::max(y_min_m, y_max_m); + + ensureOutputDirectory(output_dir_); +} + +void BirdEyeViewRenderer::renderFrame(const std::vector& tracks, int frame_index) { + cv::Mat canvas = buildCanvas(); + latest_rendered_tracks_.clear(); + latest_rendered_tracks_.reserve(tracks.size()); + + for (size_t i = 0; i < tracks.size(); ++i) { + drawTrackPoint(canvas, tracks[i], frame_index); + } + + const std::string output_path = frameOutputPath(frame_index); + cv::imwrite(output_path, canvas); +} + +cv::Point2i BirdEyeViewRenderer::worldToImage(float x_m, float y_m, bool* inside) const { + const float x_span = world_range_.max_x - world_range_.min_x; + const float y_span = world_range_.max_y - world_range_.min_y; + const float usable_w = static_cast(canvas_width_ - 2 * padding_px_); + const float usable_h = static_cast(canvas_height_ - 2 * padding_px_); + + const float nx = (x_m - world_range_.min_x) / x_span; + const float ny = (y_m - world_range_.min_y) / y_span; + + const int px = static_cast(std::round(padding_px_ + clamp01(nx) * usable_w)); + const int py = static_cast(std::round(canvas_height_ - padding_px_ - clamp01(ny) * usable_h)); + + const bool is_inside = (x_m >= world_range_.min_x && x_m <= world_range_.max_x && y_m >= world_range_.min_y && + y_m <= world_range_.max_y); + if (inside != nullptr) { + *inside = is_inside; + } + + return cv::Point2i(px, py); +} + +const std::vector& BirdEyeViewRenderer::latestRenderedTracks() const { + return latest_rendered_tracks_; +} + +bool BirdEyeViewRenderer::ensureOutputDirectory(const std::string& dir) const { + if (dir.empty()) { + return false; + } + + std::string current; + current.reserve(dir.size()); + for (size_t i = 0; i < dir.size(); ++i) { + current.push_back(dir[i]); + if (dir[i] == '/' || i + 1 == dir.size()) { + if (current.empty() || current == "/") { + continue; + } + if (mkdir(current.c_str(), 0755) != 0 && errno != EEXIST) { + return false; + } + } + } + return true; +} + +std::string BirdEyeViewRenderer::frameOutputPath(int frame_index) const { + std::ostringstream oss; + oss << output_dir_ << "/bev_" << std::setfill('0') << std::setw(6) << frame_index << ".jpg"; + return oss.str(); +} + +cv::Mat BirdEyeViewRenderer::buildCanvas() const { + cv::Mat canvas(canvas_height_, canvas_width_, CV_8UC3, cv::Scalar(245, 245, 245)); + drawGridAndAxes(canvas); + return canvas; +} + +void BirdEyeViewRenderer::drawGridAndAxes(cv::Mat& canvas) const { + const cv::Point2i top_left(padding_px_, padding_px_); + const cv::Point2i bottom_right(canvas_width_ - padding_px_, canvas_height_ - padding_px_); + + cv::rectangle(canvas, top_left, bottom_right, cv::Scalar(190, 190, 190), 1, cv::LINE_AA); + + for (int x = static_cast(std::ceil(world_range_.min_x)); x <= static_cast(std::floor(world_range_.max_x)); + x += grid_step_m_) { + bool inside = false; + const int px = worldToImage(static_cast(x), world_range_.min_y, &inside).x; + if (!inside && (x < world_range_.min_x || x > world_range_.max_x)) { + continue; + } + cv::line(canvas, + cv::Point(px, padding_px_), + cv::Point(px, canvas_height_ - padding_px_), + cv::Scalar(220, 220, 220), + 1, + cv::LINE_AA); + cv::putText(canvas, + std::to_string(x) + "m", + cv::Point(px + 2, canvas_height_ - padding_px_ + 18), + cv::FONT_HERSHEY_SIMPLEX, + 0.35, + cv::Scalar(100, 100, 100), + 1, + cv::LINE_AA); + } + + for (int y = static_cast(std::ceil(world_range_.min_y)); y <= static_cast(std::floor(world_range_.max_y)); + y += grid_step_m_) { + bool inside = false; + const int py = worldToImage(world_range_.min_x, static_cast(y), &inside).y; + if (!inside && (y < world_range_.min_y || y > world_range_.max_y)) { + continue; + } + cv::line(canvas, + cv::Point(padding_px_, py), + cv::Point(canvas_width_ - padding_px_, py), + cv::Scalar(220, 220, 220), + 1, + cv::LINE_AA); + cv::putText(canvas, + std::to_string(y) + "m", + cv::Point(8, py - 2), + cv::FONT_HERSHEY_SIMPLEX, + 0.35, + cv::Scalar(100, 100, 100), + 1, + cv::LINE_AA); + } + + cv::putText(canvas, + "BEV map (meters)", + cv::Point(padding_px_, 25), + cv::FONT_HERSHEY_SIMPLEX, + 0.6, + cv::Scalar(60, 60, 60), + 2, + cv::LINE_AA); + + cv::putText(canvas, + "x range: [-40, 50] m", + cv::Point(canvas_width_ - 210, 25), + cv::FONT_HERSHEY_SIMPLEX, + 0.45, + cv::Scalar(80, 80, 80), + 1, + cv::LINE_AA); +} + +void BirdEyeViewRenderer::drawTrackPoint(cv::Mat& canvas, const BEVTrackPoint& track, int frame_index) { + bool inside = false; + const cv::Point2i pixel = worldToImage(track.gp.x, track.gp.y, &inside); + const cv::Scalar color = getTrackColor(track.track_id); + + BEVRenderedTrackInfo rendered; + rendered.frame_index = frame_index; + rendered.track_id = track.track_id; + rendered.class_id = track.class_id; + rendered.class_label = track.class_label; + rendered.gp = track.gp; + rendered.bev_pixel = pixel; + rendered.inside_bev = inside; + rendered.track_state = track.track_state; + latest_rendered_tracks_.push_back(rendered); + + if (!inside) { + return; + } + + if (track.track_state == TrackState::Lost) { + cv::drawMarker(canvas, pixel, color, cv::MARKER_TILTED_CROSS, 14, 2, cv::LINE_AA); + } else { + cv::circle(canvas, pixel, 5, color, cv::FILLED, cv::LINE_AA); + cv::circle(canvas, pixel, 9, color, 1, cv::LINE_AA); + } + + const std::string class_text = resolveClassText(track); + std::ostringstream oss; + oss << "id:" << track.track_id; + if (!class_text.empty()) { + oss << " cls:" << class_text; + } + if (track.track_state == TrackState::Lost) { + oss << " LOST"; + } + + cv::putText(canvas, + oss.str(), + cv::Point(pixel.x + 8, pixel.y - 8), + cv::FONT_HERSHEY_SIMPLEX, + 0.42, + color, + 2, + cv::LINE_AA); + cv::putText(canvas, + oss.str(), + cv::Point(pixel.x + 8, pixel.y - 8), + cv::FONT_HERSHEY_SIMPLEX, + 0.42, + cv::Scalar(255, 255, 255), + 1, + cv::LINE_AA); +} + +cv::Scalar BirdEyeViewRenderer::getTrackColor(int track_id) { + const std::unordered_map::const_iterator it = track_color_table_.find(track_id); + if (it != track_color_table_.end()) { + return it->second; + } + + const int hue_seed = (track_id * 37) % 180; + const cv::Scalar color = hsvToBgr(static_cast(hue_seed), 0.85F, 0.95F); + track_color_table_[track_id] = color; + return color; +} + +std::string BirdEyeViewRenderer::resolveClassText(const BEVTrackPoint& track) const { + if (!track.class_label.empty()) { + return track.class_label; + } + if (track.class_id >= 0) { + return std::to_string(track.class_id); + } + return ""; +} diff --git a/deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp b/deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp new file mode 100644 index 000000000..67d481cc8 --- /dev/null +++ b/deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp @@ -0,0 +1,56 @@ +#include "BirdEyeViewTracker.h" + +BirdEyeViewTracker::BirdEyeViewTracker(int frame_rate, + int track_buffer, + const std::string& bev_output_dir, + float y_min_m, + float y_max_m, + int canvas_width, + int canvas_height, + int padding_px, + int grid_step_m) + : BYTETracker(frame_rate, track_buffer), + bev_renderer_(bev_output_dir, y_min_m, y_max_m, canvas_width, canvas_height, padding_px, grid_step_m), + bev_frame_index_(0) { +} + +vector BirdEyeViewTracker::update(const vector& objects) { + vector output_tracks = BYTETracker::update(objects); + + vector bev_tracks; + bev_tracks.reserve(get_tracked_stracks().size() + get_lost_stracks().size()); + + appendTracksToBevInput(get_tracked_stracks(), TrackState::Tracked, bev_tracks); + appendTracksToBevInput(get_lost_stracks(), TrackState::Lost, bev_tracks); + + ++bev_frame_index_; + bev_renderer_.renderFrame(bev_tracks, bev_frame_index_); + + return output_tracks; +} + +const BirdEyeViewRenderer& BirdEyeViewTracker::renderer() const { + return bev_renderer_; +} + +BirdEyeViewRenderer& BirdEyeViewTracker::renderer() { + return bev_renderer_; +} + +void BirdEyeViewTracker::appendTracksToBevInput(const vector& tracks, + int expected_state, + vector& bev_tracks) const { + for (size_t i = 0; i < tracks.size(); ++i) { + if (tracks[i].state != expected_state) { + continue; + } + + BEVTrackPoint point; + point.track_id = tracks[i].track_id; + point.class_id = tracks[i].class_id; + point.gp = tracks[i].gp; + point.is_active = tracks[i].is_activated; + point.track_state = tracks[i].state; + bev_tracks.push_back(point); + } +} diff --git a/deploy/TensorRT/cpp/src/STrack.cpp b/deploy/TensorRT/cpp/src/STrack.cpp index 0e8653e8f..5b1e092fb 100644 --- a/deploy/TensorRT/cpp/src/STrack.cpp +++ b/deploy/TensorRT/cpp/src/STrack.cpp @@ -1,7 +1,7 @@ #include "STrack.h" -STrack::STrack(vector tlwh_, float score) -{ +STrack::STrack(vector tlwh_, float score, int class_id, cv::Point3f gp) +{ _tlwh.resize(4); _tlwh.assign(tlwh_.begin(), tlwh_.end()); @@ -16,9 +16,11 @@ STrack::STrack(vector tlwh_, float score) static_tlbr(); frame_id = 0; tracklet_len = 0; - this->score = score; - start_frame = 0; -} + this->score = score; + this->class_id = class_id; + this->gp = gp; + start_frame = 0; +} STrack::~STrack() { @@ -77,10 +79,12 @@ void STrack::re_activate(STrack &new_track, int frame_id, bool new_id) this->state = TrackState::Tracked; this->is_activated = true; this->frame_id = frame_id; - this->score = new_track.score; - if (new_id) - this->track_id = next_id(); -} + this->score = new_track.score; + this->class_id = new_track.class_id; + this->gp = new_track.gp; + if (new_id) + this->track_id = next_id(); +} void STrack::update(STrack &new_track, int frame_id) { @@ -104,8 +108,10 @@ void STrack::update(STrack &new_track, int frame_id) this->state = TrackState::Tracked; this->is_activated = true; - this->score = new_track.score; -} + this->score = new_track.score; + this->class_id = new_track.class_id; + this->gp = new_track.gp; +} void STrack::static_tlwh() { @@ -191,4 +197,4 @@ void STrack::multi_predict(vector &stracks, byte_kalman::KalmanFilter & stracks[i]->static_tlwh(); stracks[i]->static_tlbr(); } -} \ No newline at end of file +} diff --git a/deploy/TensorRT/cpp/src/bytetrack.cpp b/deploy/TensorRT/cpp/src/bytetrack.cpp index 5b99eed8f..67e953ca6 100644 --- a/deploy/TensorRT/cpp/src/bytetrack.cpp +++ b/deploy/TensorRT/cpp/src/bytetrack.cpp @@ -9,7 +9,7 @@ #include "NvInfer.h" #include "cuda_runtime_api.h" #include "logging.h" -#include "BYTETracker.h" +#include "BirdEyeViewTracker.h" #define CHECK(status) \ do\ @@ -444,7 +444,7 @@ int main(int argc, char** argv) { VideoWriter writer("demo.mp4", VideoWriter::fourcc('m', 'p', '4', 'v'), fps, Size(img_w, img_h)); Mat img; - BYTETracker tracker(fps, 30); + BirdEyeViewTracker tracker(fps, 30, "bev_outputs", -30.0F, 60.0F, 1000, 1000); int num_frames = 0; int total_ms = 0; while (true) From bc5dbd36945b0db257ab6cce8d1f293c12b8464c Mon Sep 17 00:00:00 2001 From: salhiraid <134926431+salhiraid@users.noreply.github.com> Date: Wed, 15 Apr 2026 17:29:00 +0200 Subject: [PATCH 2/4] Make BEV renderer consume tracked/lost STrack vectors directly --- deploy/TensorRT/cpp/BEV_INTEGRATION.md | 26 +++++++++---------- .../cpp/include/BirdEyeViewRenderer.h | 18 ++++--------- .../TensorRT/cpp/include/BirdEyeViewTracker.h | 5 ---- .../TensorRT/cpp/src/BirdEyeViewRenderer.cpp | 21 ++++++++------- .../TensorRT/cpp/src/BirdEyeViewTracker.cpp | 26 +------------------ 5 files changed, 29 insertions(+), 67 deletions(-) diff --git a/deploy/TensorRT/cpp/BEV_INTEGRATION.md b/deploy/TensorRT/cpp/BEV_INTEGRATION.md index 8d419f542..dcb163e41 100644 --- a/deploy/TensorRT/cpp/BEV_INTEGRATION.md +++ b/deploy/TensorRT/cpp/BEV_INTEGRATION.md @@ -1,13 +1,16 @@ # BirdEyeViewTracker integration example -The cleanest integration is to use `BirdEyeViewTracker`, a child class of `BYTETracker`. -It calls the original ByteTrack update logic, then renders **both tracked and lost tracks** -into BEV for each frame. +`BirdEyeViewTracker` is a child class of `BYTETracker` and now feeds BEV rendering using +ByteTrack state containers directly: + +- `vector tracked_stracks` +- `vector lost_stracks` + +No object-level BEV adapter is needed. ```cpp #include "BirdEyeViewTracker.h" -// once before the frame loop BirdEyeViewTracker tracker( fps, 30, @@ -17,18 +20,13 @@ BirdEyeViewTracker tracker( 1000, 1000); -// in the frame loop (same call pattern as BYTETracker) vector output_stracks = tracker.update(objects); ``` -## Passing class and ground-point (`gp`) to ByteTrack tracks - -`Object` now includes: +Internally, after each `update(...)`, the subclass calls: -- `label` (class id) -- `gp` (`cv::Point3f`) for world coordinates `(x, y, z)` where `z = 0` - -When detections are converted to `STrack`, class and `gp` are copied, so tracked/lost states -carry BEV information without changing ByteTrack association/matching behavior. +```cpp +bev_renderer_.renderFrame(get_tracked_stracks(), get_lost_stracks(), bev_frame_index_); +``` -If your detector does not yet provide `gp`, set it before calling `update(...)`. +So BEV rendering operates directly on `STrack` instances (tracked + lost). diff --git a/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h b/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h index dbee96443..2da9127d4 100644 --- a/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h +++ b/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h @@ -8,20 +8,10 @@ #include "STrack.h" -struct BEVTrackPoint { - int track_id = -1; - int class_id = -1; - std::string class_label; - cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F); - bool is_active = true; - int track_state = TrackState::Tracked; -}; - struct BEVRenderedTrackInfo { int frame_index = -1; int track_id = -1; int class_id = -1; - std::string class_label; cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F); cv::Point2i bev_pixel = cv::Point2i(-1, -1); bool inside_bev = false; @@ -38,7 +28,9 @@ class BirdEyeViewRenderer { int padding_px = 50, int grid_step_m = 10); - void renderFrame(const std::vector& tracks, int frame_index); + void renderFrame(const std::vector& tracked_stracks, + const std::vector& lost_stracks, + int frame_index); cv::Point2i worldToImage(float x_m, float y_m, bool* inside = nullptr) const; @@ -56,9 +48,9 @@ class BirdEyeViewRenderer { std::string frameOutputPath(int frame_index) const; cv::Mat buildCanvas() const; void drawGridAndAxes(cv::Mat& canvas) const; - void drawTrackPoint(cv::Mat& canvas, const BEVTrackPoint& track, int frame_index); + void drawTrackPoint(cv::Mat& canvas, const STrack& track, int frame_index); cv::Scalar getTrackColor(int track_id); - std::string resolveClassText(const BEVTrackPoint& track) const; + std::string resolveClassText(const STrack& track) const; private: std::string output_dir_; diff --git a/deploy/TensorRT/cpp/include/BirdEyeViewTracker.h b/deploy/TensorRT/cpp/include/BirdEyeViewTracker.h index 68e2737e6..ae7c375f6 100644 --- a/deploy/TensorRT/cpp/include/BirdEyeViewTracker.h +++ b/deploy/TensorRT/cpp/include/BirdEyeViewTracker.h @@ -20,11 +20,6 @@ class BirdEyeViewTracker : public BYTETracker { const BirdEyeViewRenderer& renderer() const; BirdEyeViewRenderer& renderer(); -private: - void appendTracksToBevInput(const vector& tracks, - int expected_state, - vector& bev_tracks) const; - private: BirdEyeViewRenderer bev_renderer_; int bev_frame_index_; diff --git a/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp b/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp index d4a5fca0c..99fef1805 100644 --- a/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp +++ b/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp @@ -44,13 +44,18 @@ BirdEyeViewRenderer::BirdEyeViewRenderer(const std::string& output_dir, ensureOutputDirectory(output_dir_); } -void BirdEyeViewRenderer::renderFrame(const std::vector& tracks, int frame_index) { +void BirdEyeViewRenderer::renderFrame(const std::vector& tracked_stracks, + const std::vector& lost_stracks, + int frame_index) { cv::Mat canvas = buildCanvas(); latest_rendered_tracks_.clear(); - latest_rendered_tracks_.reserve(tracks.size()); + latest_rendered_tracks_.reserve(tracked_stracks.size() + lost_stracks.size()); - for (size_t i = 0; i < tracks.size(); ++i) { - drawTrackPoint(canvas, tracks[i], frame_index); + for (size_t i = 0; i < tracked_stracks.size(); ++i) { + drawTrackPoint(canvas, tracked_stracks[i], frame_index); + } + for (size_t i = 0; i < lost_stracks.size(); ++i) { + drawTrackPoint(canvas, lost_stracks[i], frame_index); } const std::string output_path = frameOutputPath(frame_index); @@ -186,7 +191,7 @@ void BirdEyeViewRenderer::drawGridAndAxes(cv::Mat& canvas) const { cv::LINE_AA); } -void BirdEyeViewRenderer::drawTrackPoint(cv::Mat& canvas, const BEVTrackPoint& track, int frame_index) { +void BirdEyeViewRenderer::drawTrackPoint(cv::Mat& canvas, const STrack& track, int frame_index) { bool inside = false; const cv::Point2i pixel = worldToImage(track.gp.x, track.gp.y, &inside); const cv::Scalar color = getTrackColor(track.track_id); @@ -195,7 +200,6 @@ void BirdEyeViewRenderer::drawTrackPoint(cv::Mat& canvas, const BEVTrackPoint& t rendered.frame_index = frame_index; rendered.track_id = track.track_id; rendered.class_id = track.class_id; - rendered.class_label = track.class_label; rendered.gp = track.gp; rendered.bev_pixel = pixel; rendered.inside_bev = inside; @@ -253,10 +257,7 @@ cv::Scalar BirdEyeViewRenderer::getTrackColor(int track_id) { return color; } -std::string BirdEyeViewRenderer::resolveClassText(const BEVTrackPoint& track) const { - if (!track.class_label.empty()) { - return track.class_label; - } +std::string BirdEyeViewRenderer::resolveClassText(const STrack& track) const { if (track.class_id >= 0) { return std::to_string(track.class_id); } diff --git a/deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp b/deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp index 67d481cc8..6f37b5c3b 100644 --- a/deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp +++ b/deploy/TensorRT/cpp/src/BirdEyeViewTracker.cpp @@ -17,14 +17,8 @@ BirdEyeViewTracker::BirdEyeViewTracker(int frame_rate, vector BirdEyeViewTracker::update(const vector& objects) { vector output_tracks = BYTETracker::update(objects); - vector bev_tracks; - bev_tracks.reserve(get_tracked_stracks().size() + get_lost_stracks().size()); - - appendTracksToBevInput(get_tracked_stracks(), TrackState::Tracked, bev_tracks); - appendTracksToBevInput(get_lost_stracks(), TrackState::Lost, bev_tracks); - ++bev_frame_index_; - bev_renderer_.renderFrame(bev_tracks, bev_frame_index_); + bev_renderer_.renderFrame(get_tracked_stracks(), get_lost_stracks(), bev_frame_index_); return output_tracks; } @@ -36,21 +30,3 @@ const BirdEyeViewRenderer& BirdEyeViewTracker::renderer() const { BirdEyeViewRenderer& BirdEyeViewTracker::renderer() { return bev_renderer_; } - -void BirdEyeViewTracker::appendTracksToBevInput(const vector& tracks, - int expected_state, - vector& bev_tracks) const { - for (size_t i = 0; i < tracks.size(); ++i) { - if (tracks[i].state != expected_state) { - continue; - } - - BEVTrackPoint point; - point.track_id = tracks[i].track_id; - point.class_id = tracks[i].class_id; - point.gp = tracks[i].gp; - point.is_active = tracks[i].is_activated; - point.track_state = tracks[i].state; - bev_tracks.push_back(point); - } -} From 17996be16c7f4823868475f3447a6f62ab8c87c2 Mon Sep 17 00:00:00 2001 From: salhiraid <134926431+salhiraid@users.noreply.github.com> Date: Wed, 15 Apr 2026 17:33:16 +0200 Subject: [PATCH 3/4] Switch BEV ground-point handling to STrack gp vector --- deploy/TensorRT/cpp/BEV_INTEGRATION.md | 17 +++++++++++------ deploy/TensorRT/cpp/include/BYTETracker.h | 1 - .../TensorRT/cpp/include/BirdEyeViewRenderer.h | 2 +- deploy/TensorRT/cpp/include/STrack.h | 4 ++-- deploy/TensorRT/cpp/src/BYTETracker.cpp | 2 +- deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp | 4 +++- deploy/TensorRT/cpp/src/STrack.cpp | 4 +++- 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/deploy/TensorRT/cpp/BEV_INTEGRATION.md b/deploy/TensorRT/cpp/BEV_INTEGRATION.md index dcb163e41..e2b6935b6 100644 --- a/deploy/TensorRT/cpp/BEV_INTEGRATION.md +++ b/deploy/TensorRT/cpp/BEV_INTEGRATION.md @@ -1,12 +1,11 @@ # BirdEyeViewTracker integration example -`BirdEyeViewTracker` is a child class of `BYTETracker` and now feeds BEV rendering using -ByteTrack state containers directly: +`BirdEyeViewTracker` is a child class of `BYTETracker` and feeds BEV rendering directly with: - `vector tracked_stracks` - `vector lost_stracks` -No object-level BEV adapter is needed. +No object-level BEV input is used. ```cpp #include "BirdEyeViewTracker.h" @@ -23,10 +22,16 @@ BirdEyeViewTracker tracker( vector output_stracks = tracker.update(objects); ``` -Internally, after each `update(...)`, the subclass calls: +## Ground point format in `STrack` + +The BEV renderer reads `STrack::gp` directly, where `gp` is a `std::vector` +of size 2 containing world coordinates: + +- `gp[0]` -> `x` (meters) +- `gp[1]` -> `y` (meters) + +Internally after each update: ```cpp bev_renderer_.renderFrame(get_tracked_stracks(), get_lost_stracks(), bev_frame_index_); ``` - -So BEV rendering operates directly on `STrack` instances (tracked + lost). diff --git a/deploy/TensorRT/cpp/include/BYTETracker.h b/deploy/TensorRT/cpp/include/BYTETracker.h index df8278fdb..24d1cc9a6 100644 --- a/deploy/TensorRT/cpp/include/BYTETracker.h +++ b/deploy/TensorRT/cpp/include/BYTETracker.h @@ -7,7 +7,6 @@ struct Object cv::Rect_ rect; int label; float prob; - cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F); }; class BYTETracker diff --git a/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h b/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h index 2da9127d4..45579ff44 100644 --- a/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h +++ b/deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h @@ -12,7 +12,7 @@ struct BEVRenderedTrackInfo { int frame_index = -1; int track_id = -1; int class_id = -1; - cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F); + std::vector gp = std::vector(2, 0.0F); cv::Point2i bev_pixel = cv::Point2i(-1, -1); bool inside_bev = false; int track_state = TrackState::Tracked; diff --git a/deploy/TensorRT/cpp/include/STrack.h b/deploy/TensorRT/cpp/include/STrack.h index f307cb128..eda3335c6 100644 --- a/deploy/TensorRT/cpp/include/STrack.h +++ b/deploy/TensorRT/cpp/include/STrack.h @@ -11,7 +11,7 @@ enum TrackState { New = 0, Tracked, Lost, Removed }; class STrack { public: - STrack(vector tlwh_, float score, int class_id = -1, cv::Point3f gp = cv::Point3f(0.0F, 0.0F, 0.0F)); + STrack(vector tlwh_, float score, int class_id = -1, vector gp = vector(2, 0.0F)); ~STrack(); vector static tlbr_to_tlwh(vector &tlbr); @@ -45,7 +45,7 @@ class STrack KAL_COVA covariance; float score; int class_id; - cv::Point3f gp; + vector gp; private: byte_kalman::KalmanFilter kalman_filter; diff --git a/deploy/TensorRT/cpp/src/BYTETracker.cpp b/deploy/TensorRT/cpp/src/BYTETracker.cpp index c3191631c..1d4a6d41d 100644 --- a/deploy/TensorRT/cpp/src/BYTETracker.cpp +++ b/deploy/TensorRT/cpp/src/BYTETracker.cpp @@ -51,7 +51,7 @@ vector BYTETracker::update(const vector& objects) float score = objects[i].prob; - STrack strack(STrack::tlbr_to_tlwh(tlbr_), score, objects[i].label, objects[i].gp); + STrack strack(STrack::tlbr_to_tlwh(tlbr_), score, objects[i].label); if (score >= track_thresh) { detections.push_back(strack); diff --git a/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp b/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp index 99fef1805..d2d7b6e1c 100644 --- a/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp +++ b/deploy/TensorRT/cpp/src/BirdEyeViewRenderer.cpp @@ -192,8 +192,10 @@ void BirdEyeViewRenderer::drawGridAndAxes(cv::Mat& canvas) const { } void BirdEyeViewRenderer::drawTrackPoint(cv::Mat& canvas, const STrack& track, int frame_index) { + const float gp_x = track.gp.size() > 0 ? track.gp[0] : 0.0F; + const float gp_y = track.gp.size() > 1 ? track.gp[1] : 0.0F; bool inside = false; - const cv::Point2i pixel = worldToImage(track.gp.x, track.gp.y, &inside); + const cv::Point2i pixel = worldToImage(gp_x, gp_y, &inside); const cv::Scalar color = getTrackColor(track.track_id); BEVRenderedTrackInfo rendered; diff --git a/deploy/TensorRT/cpp/src/STrack.cpp b/deploy/TensorRT/cpp/src/STrack.cpp index 5b1e092fb..dd93ef2b1 100644 --- a/deploy/TensorRT/cpp/src/STrack.cpp +++ b/deploy/TensorRT/cpp/src/STrack.cpp @@ -1,6 +1,6 @@ #include "STrack.h" -STrack::STrack(vector tlwh_, float score, int class_id, cv::Point3f gp) +STrack::STrack(vector tlwh_, float score, int class_id, vector gp) { _tlwh.resize(4); _tlwh.assign(tlwh_.begin(), tlwh_.end()); @@ -19,6 +19,8 @@ STrack::STrack(vector tlwh_, float score, int class_id, cv::Point3f gp) this->score = score; this->class_id = class_id; this->gp = gp; + if (this->gp.size() < 2) + this->gp.assign(2, 0.0F); start_frame = 0; } From ad5b62207dd0228ad3b9a5787c0fa04e8a2b9376 Mon Sep 17 00:00:00 2001 From: salhiraid <134926431+salhiraid@users.noreply.github.com> Date: Wed, 15 Apr 2026 17:38:25 +0200 Subject: [PATCH 4/4] Call BEV mapping at end of BYTETracker update --- deploy/TensorRT/cpp/BEV_INTEGRATION.md | 44 ++++++++++------------- deploy/TensorRT/cpp/include/BYTETracker.h | 26 ++++++++++---- deploy/TensorRT/cpp/src/BYTETracker.cpp | 42 ++++++++++++++++------ deploy/TensorRT/cpp/src/bytetrack.cpp | 5 +-- 4 files changed, 72 insertions(+), 45 deletions(-) diff --git a/deploy/TensorRT/cpp/BEV_INTEGRATION.md b/deploy/TensorRT/cpp/BEV_INTEGRATION.md index e2b6935b6..0e44f9f11 100644 --- a/deploy/TensorRT/cpp/BEV_INTEGRATION.md +++ b/deploy/TensorRT/cpp/BEV_INTEGRATION.md @@ -1,37 +1,31 @@ -# BirdEyeViewTracker integration example +# BEV mapping call location in `BYTETracker::update` -`BirdEyeViewTracker` is a child class of `BYTETracker` and feeds BEV rendering directly with: +The BEV mapping is now called **inside `BYTETracker::update`**, right after: -- `vector tracked_stracks` -- `vector lost_stracks` +```cpp +this->lost_stracks.assign(resb.begin(), resb.end()); +``` -No object-level BEV input is used. +Then: ```cpp -#include "BirdEyeViewTracker.h" - -BirdEyeViewTracker tracker( - fps, - 30, - "bev_outputs", // output directory - -30.0f, // configurable y min [m] - 60.0f, // configurable y max [m] - 1000, - 1000); - -vector output_stracks = tracker.update(objects); +if (bev_renderer.get() != nullptr) +{ + bev_frame_index++; + bev_renderer->renderFrame(this->tracked_stracks, this->lost_stracks, bev_frame_index); +} ``` -## Ground point format in `STrack` +So each frame maps directly from tracker state vectors: -The BEV renderer reads `STrack::gp` directly, where `gp` is a `std::vector` -of size 2 containing world coordinates: - -- `gp[0]` -> `x` (meters) -- `gp[1]` -> `y` (meters) +- `vector tracked_stracks` +- `vector lost_stracks` -Internally after each update: +## Enable mapping from `bytetrack.cpp` ```cpp -bev_renderer_.renderFrame(get_tracked_stracks(), get_lost_stracks(), bev_frame_index_); +BYTETracker tracker(fps, 30); +tracker.enable_bev_mapping("bev_outputs", -30.0f, 60.0f, 1000, 1000); ``` + +`STrack::gp` is used directly and expected as `std::vector{x, y}`. diff --git a/deploy/TensorRT/cpp/include/BYTETracker.h b/deploy/TensorRT/cpp/include/BYTETracker.h index 24d1cc9a6..3e0f25651 100644 --- a/deploy/TensorRT/cpp/include/BYTETracker.h +++ b/deploy/TensorRT/cpp/include/BYTETracker.h @@ -1,6 +1,9 @@ -#pragma once - -#include "STrack.h" +#pragma once + +#include "STrack.h" +#include + +class BirdEyeViewRenderer; struct Object { @@ -19,6 +22,13 @@ class BYTETracker Scalar get_color(int idx); const vector& get_tracked_stracks() const; const vector& get_lost_stracks() const; + void enable_bev_mapping(const string& output_dir, + float y_min_m, + float y_max_m, + int canvas_width = 900, + int canvas_height = 900, + int padding_px = 50, + int grid_step_m = 10); private: vector joint_stracks(vector &tlista, vector &tlistb); @@ -44,8 +54,10 @@ class BYTETracker int frame_id; int max_time_lost; - vector tracked_stracks; - vector lost_stracks; - vector removed_stracks; - byte_kalman::KalmanFilter kalman_filter; + vector tracked_stracks; + vector lost_stracks; + vector removed_stracks; + byte_kalman::KalmanFilter kalman_filter; + std::unique_ptr bev_renderer; + int bev_frame_index; }; diff --git a/deploy/TensorRT/cpp/src/BYTETracker.cpp b/deploy/TensorRT/cpp/src/BYTETracker.cpp index 1d4a6d41d..8f81af180 100644 --- a/deploy/TensorRT/cpp/src/BYTETracker.cpp +++ b/deploy/TensorRT/cpp/src/BYTETracker.cpp @@ -1,5 +1,6 @@ -#include "BYTETracker.h" -#include +#include "BYTETracker.h" +#include "BirdEyeViewRenderer.h" +#include BYTETracker::BYTETracker(int frame_rate, int track_buffer) { @@ -7,10 +8,11 @@ BYTETracker::BYTETracker(int frame_rate, int track_buffer) high_thresh = 0.6; match_thresh = 0.8; - frame_id = 0; - max_time_lost = int(frame_rate / 30.0 * track_buffer); - cout << "Init ByteTrack!" << endl; -} + frame_id = 0; + bev_frame_index = 0; + max_time_lost = int(frame_rate / 30.0 * track_buffer); + cout << "Init ByteTrack!" << endl; +} BYTETracker::~BYTETracker() { @@ -227,11 +229,17 @@ vector BYTETracker::update(const vector& objects) this->tracked_stracks.clear(); this->tracked_stracks.assign(resa.begin(), resa.end()); - this->lost_stracks.clear(); - this->lost_stracks.assign(resb.begin(), resb.end()); - - for (int i = 0; i < this->tracked_stracks.size(); i++) - { + this->lost_stracks.clear(); + this->lost_stracks.assign(resb.begin(), resb.end()); + + if (bev_renderer.get() != nullptr) + { + bev_frame_index++; + bev_renderer->renderFrame(this->tracked_stracks, this->lost_stracks, bev_frame_index); + } + + for (int i = 0; i < this->tracked_stracks.size(); i++) + { if (this->tracked_stracks[i].is_activated) { output_stracks.push_back(this->tracked_stracks[i]); @@ -249,3 +257,15 @@ const vector& BYTETracker::get_lost_stracks() const { return lost_stracks; } + +void BYTETracker::enable_bev_mapping(const string& output_dir, + float y_min_m, + float y_max_m, + int canvas_width, + int canvas_height, + int padding_px, + int grid_step_m) +{ + bev_renderer.reset(new BirdEyeViewRenderer(output_dir, y_min_m, y_max_m, canvas_width, canvas_height, padding_px, grid_step_m)); + bev_frame_index = 0; +} diff --git a/deploy/TensorRT/cpp/src/bytetrack.cpp b/deploy/TensorRT/cpp/src/bytetrack.cpp index 67e953ca6..fe0f32bea 100644 --- a/deploy/TensorRT/cpp/src/bytetrack.cpp +++ b/deploy/TensorRT/cpp/src/bytetrack.cpp @@ -9,7 +9,7 @@ #include "NvInfer.h" #include "cuda_runtime_api.h" #include "logging.h" -#include "BirdEyeViewTracker.h" +#include "BYTETracker.h" #define CHECK(status) \ do\ @@ -444,7 +444,8 @@ int main(int argc, char** argv) { VideoWriter writer("demo.mp4", VideoWriter::fourcc('m', 'p', '4', 'v'), fps, Size(img_w, img_h)); Mat img; - BirdEyeViewTracker tracker(fps, 30, "bev_outputs", -30.0F, 60.0F, 1000, 1000); + BYTETracker tracker(fps, 30); + tracker.enable_bev_mapping("bev_outputs", -30.0F, 60.0F, 1000, 1000); int num_frames = 0; int total_ms = 0; while (true)