Add Bird's-Eye-View rendering and tracker integration for ByteTrack

deploy/TensorRT/cpp/BEV_INTEGRATION.md

@@ -0,0 +1,31 @@
+# BEV mapping call location in `BYTETracker::update`
+
+The BEV mapping is now called **inside `BYTETracker::update`**, right after:
+
+```cpp
+this->lost_stracks.assign(resb.begin(), resb.end());
+```
+
+Then:
+
+```cpp
+if (bev_renderer.get() != nullptr)
+{
+    bev_frame_index++;
+    bev_renderer->renderFrame(this->tracked_stracks, this->lost_stracks, bev_frame_index);
+}
+```
+
+So each frame maps directly from tracker state vectors:
+
+- `vector<STrack> tracked_stracks`
+- `vector<STrack> lost_stracks`
+
+## Enable mapping from `bytetrack.cpp`
+
+```cpp
+BYTETracker tracker(fps, 30);
+tracker.enable_bev_mapping("bev_outputs", -30.0f, 60.0f, 1000, 1000);
+```
+
+`STrack::gp` is used directly and expected as `std::vector<float>{x, y}`.

deploy/TensorRT/cpp/include/BYTETracker.h

@@ -1,24 +1,36 @@
-#pragma once
-
-#include "STrack.h"
-
-struct Object
-{
-    cv::Rect_<float> rect;
-    int label;
-    float prob;
-};
-
-class BYTETracker
-{
-public:
-	BYTETracker(int frame_rate = 30, int track_buffer = 30);
-	~BYTETracker();
-
-	vector<STrack> update(const vector<Object>& objects);
-	Scalar get_color(int idx);
-
-private:
+#pragma once
+
+#include "STrack.h"
+#include <memory>
+
+class BirdEyeViewRenderer;
+
+struct Object
+{
+    cv::Rect_<float> rect;
+    int label;
+    float prob;
+};
+
+class BYTETracker
+{
+public:
+	BYTETracker(int frame_rate = 30, int track_buffer = 30);
+	virtual ~BYTETracker();
+
+	virtual vector<STrack> update(const vector<Object>& objects);
+	Scalar get_color(int idx);
+	const vector<STrack>& get_tracked_stracks() const;
+	const vector<STrack>& get_lost_stracks() const;
+	void enable_bev_mapping(const string& output_dir,
+	                        float y_min_m,
+	                        float y_max_m,
+	                        int canvas_width = 900,
+	                        int canvas_height = 900,
+	                        int padding_px = 50,
+	                        int grid_step_m = 10);
+
+private:
 	vector<STrack*> joint_stracks(vector<STrack*> &tlista, vector<STrack> &tlistb);
 	vector<STrack> joint_stracks(vector<STrack> &tlista, vector<STrack> &tlistb);
 
@@ -42,8 +54,10 @@ class BYTETracker
 	int frame_id;
 	int max_time_lost;
 
-	vector<STrack> tracked_stracks;
-	vector<STrack> lost_stracks;
-	vector<STrack> removed_stracks;
-	byte_kalman::KalmanFilter kalman_filter;
-};
+	vector<STrack> tracked_stracks;
+	vector<STrack> lost_stracks;
+	vector<STrack> removed_stracks;
+	byte_kalman::KalmanFilter kalman_filter;
+	std::unique_ptr<BirdEyeViewRenderer> bev_renderer;
+	int bev_frame_index;
+};

deploy/TensorRT/cpp/include/BirdEyeViewRenderer.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include <opencv2/opencv.hpp>
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "STrack.h"
+
+struct BEVRenderedTrackInfo {
+    int frame_index = -1;
+    int track_id = -1;
+    int class_id = -1;
+    std::vector<float> gp = std::vector<float>(2, 0.0F);
+    cv::Point2i bev_pixel = cv::Point2i(-1, -1);
+    bool inside_bev = false;
+    int track_state = TrackState::Tracked;
+};
+
+class BirdEyeViewRenderer {
+public:
+    BirdEyeViewRenderer(const std::string& output_dir,
+                        float y_min_m,
+                        float y_max_m,
+                        int canvas_width = 900,
+                        int canvas_height = 900,
+                        int padding_px = 50,
+                        int grid_step_m = 10);
+
+    void renderFrame(const std::vector<STrack>& tracked_stracks,
+                     const std::vector<STrack>& lost_stracks,
+                     int frame_index);
+
+    cv::Point2i worldToImage(float x_m, float y_m, bool* inside = nullptr) const;
+
+    const std::vector<BEVRenderedTrackInfo>& latestRenderedTracks() const;
+
+private:
+    struct Range {
+        float min_x;
+        float max_x;
+        float min_y;
+        float max_y;
+    };
+
+    bool ensureOutputDirectory(const std::string& dir) const;
+    std::string frameOutputPath(int frame_index) const;
+    cv::Mat buildCanvas() const;
+    void drawGridAndAxes(cv::Mat& canvas) const;
+    void drawTrackPoint(cv::Mat& canvas, const STrack& track, int frame_index);
+    cv::Scalar getTrackColor(int track_id);
+    std::string resolveClassText(const STrack& track) const;
+
+private:
+    std::string output_dir_;
+    Range world_range_;
+    int canvas_width_;
+    int canvas_height_;
+    int padding_px_;
+    int grid_step_m_;
+
+    std::unordered_map<int, cv::Scalar> track_color_table_;
+    std::vector<BEVRenderedTrackInfo> latest_rendered_tracks_;
+};

deploy/TensorRT/cpp/include/BirdEyeViewTracker.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "BYTETracker.h"
+#include "BirdEyeViewRenderer.h"
+
+class BirdEyeViewTracker : public BYTETracker {
+public:
+    BirdEyeViewTracker(int frame_rate,
+                       int track_buffer,
+                       const std::string& bev_output_dir,
+                       float y_min_m,
+                       float y_max_m,
+                       int canvas_width = 900,
+                       int canvas_height = 900,
+                       int padding_px = 50,
+                       int grid_step_m = 10);
+
+    vector<STrack> update(const vector<Object>& objects) override;
+
+    const BirdEyeViewRenderer& renderer() const;
+    BirdEyeViewRenderer& renderer();
+
+private:
+    BirdEyeViewRenderer bev_renderer_;
+    int bev_frame_index_;
+};

deploy/TensorRT/cpp/include/STrack.h

@@ -10,8 +10,8 @@ enum TrackState { New = 0, Tracked, Lost, Removed };
 
 class STrack
 {
-public:
-	STrack(vector<float> tlwh_, float score);
+public:
+	STrack(vector<float> tlwh_, float score, int class_id = -1, vector<float> gp = vector<float>(2, 0.0F));
 	~STrack();
 
 	vector<float> static tlbr_to_tlwh(vector<float> &tlbr);
@@ -43,8 +43,10 @@ class STrack
 
 	KAL_MEAN mean;
 	KAL_COVA covariance;
-	float score;
-
-private:
+	float score;
+	int class_id;
+	vector<float> gp;
+
+private:
 	byte_kalman::KalmanFilter kalman_filter;
-};
+};

deploy/TensorRT/cpp/src/BYTETracker.cpp

@@ -1,20 +1,22 @@
-#include "BYTETracker.h"
-#include <fstream>
+#include "BYTETracker.h"
+#include "BirdEyeViewRenderer.h"
+#include <fstream>
 
 BYTETracker::BYTETracker(int frame_rate, int track_buffer)
 {
 	track_thresh = 0.5;
 	high_thresh = 0.6;
 	match_thresh = 0.8;
 
-	frame_id = 0;
-	max_time_lost = int(frame_rate / 30.0 * track_buffer);
-	cout << "Init ByteTrack!" << endl;
-}
+	frame_id = 0;
+	bev_frame_index = 0;
+	max_time_lost = int(frame_rate / 30.0 * track_buffer);
+	cout << "Init ByteTrack!" << endl;
+}
 
-BYTETracker::~BYTETracker()
-{
-}
+BYTETracker::~BYTETracker()
+{
+}
 
 vector<STrack> BYTETracker::update(const vector<Object>& objects)
 {
@@ -51,7 +53,7 @@ vector<STrack> BYTETracker::update(const vector<Object>& objects)
 
 			float score = objects[i].prob;
 
-			STrack strack(STrack::tlbr_to_tlwh(tlbr_), score);
+			STrack strack(STrack::tlbr_to_tlwh(tlbr_), score, objects[i].label);
 			if (score >= track_thresh)
 			{
 				detections.push_back(strack);
@@ -227,15 +229,43 @@ vector<STrack> BYTETracker::update(const vector<Object>& objects)
 
 	this->tracked_stracks.clear();
 	this->tracked_stracks.assign(resa.begin(), resa.end());
-	this->lost_stracks.clear();
-	this->lost_stracks.assign(resb.begin(), resb.end());
-
-	for (int i = 0; i < this->tracked_stracks.size(); i++)
-	{
+	this->lost_stracks.clear();
+	this->lost_stracks.assign(resb.begin(), resb.end());
+
+	if (bev_renderer.get() != nullptr)
+	{
+		bev_frame_index++;
+		bev_renderer->renderFrame(this->tracked_stracks, this->lost_stracks, bev_frame_index);
+	}
+
+	for (int i = 0; i < this->tracked_stracks.size(); i++)
+	{
 		if (this->tracked_stracks[i].is_activated)
 		{
 			output_stracks.push_back(this->tracked_stracks[i]);
 		}
 	}
-	return output_stracks;
-}
+	return output_stracks;
+}
+
+const vector<STrack>& BYTETracker::get_tracked_stracks() const
+{
+	return tracked_stracks;
+}
+
+const vector<STrack>& BYTETracker::get_lost_stracks() const
+{
+	return lost_stracks;
+}
+
+void BYTETracker::enable_bev_mapping(const string& output_dir,
+									 float y_min_m,
+									 float y_max_m,
+									 int canvas_width,
+									 int canvas_height,
+									 int padding_px,
+									 int grid_step_m)
+{
+	bev_renderer.reset(new BirdEyeViewRenderer(output_dir, y_min_m, y_max_m, canvas_width, canvas_height, padding_px, grid_step_m));
+	bev_frame_index = 0;
+}