From 58ae1ccd2bad819c4526bd803ba916b62395901a Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Tue, 14 Jul 2026 15:15:17 +0530 Subject: [PATCH 01/18] Use Vishwa2684/mediapipe_ovms for bytetrack demo --- WORKSPACE | 5 +++-- third_party/mediapipe_calculators/BUILD | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 49413cd595..51cb899389 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -191,8 +191,9 @@ http_archive( #### Will be used on feature release git_repository( name = "mediapipe", - remote = "https://github.com/openvinotoolkit/mediapipe", - commit = "12e8d511cfbc5f471c498278a65a02dd250963e8", # top of mediapipe main branch as of 26.11.2025 + remote = "https://github.com/Vishwa2684/mediapipe_ovms", + # commit = "12e8d511cfbc5f471c498278a65a02dd250963e8", # top of mediapipe main branch as of 26.11.2025 + branch = "custom_bytetrack_graph" ) # DEV mediapipe 1 source - adjust local repository path for build diff --git a/third_party/mediapipe_calculators/BUILD b/third_party/mediapipe_calculators/BUILD index a4570a4daa..a22d1b4f2a 100644 --- a/third_party/mediapipe_calculators/BUILD +++ b/third_party/mediapipe_calculators/BUILD @@ -108,6 +108,8 @@ cc_library( "@mediapipe//mediapipe/calculators/video:box_detector_calculator", "@mediapipe//mediapipe/calculators/video:tracked_detection_manager_calculator", "@mediapipe//mediapipe/calculators/video:video_pre_stream_calculator", + #BYTETRACK CALCULATORS (GSoC'2026 - Vishwa2684) + "@mediapipe//mediapipe/graphs/bytetrack/calculators:bytetrack_calculators", ] + select({ "//conditions:default": [ "@mediapipe//mediapipe/calculators/core:packet_cloner_calculator", # TODO windows: stdc++20 required From ed27dc3c64dc45c6607e4bd8672037aa8b3d51f2 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Tue, 14 Jul 2026 15:17:19 +0530 Subject: [PATCH 02/18] Add ffmpeg_output width and height as client params for rtsp sink --- demos/real_time_stream_analysis/python/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/demos/real_time_stream_analysis/python/client.py b/demos/real_time_stream_analysis/python/client.py index 00e76e6561..39934f6e80 100755 --- a/demos/real_time_stream_analysis/python/client.py +++ b/demos/real_time_stream_analysis/python/client.py @@ -23,6 +23,8 @@ parser = argparse.ArgumentParser() parser.add_argument('--grpc_address', required=False, default='localhost:9022', help='Specify url to grpc service') +parser.add_argument('--ffmpeg_output_width', required=False, default=None, type=int, help='Width of the output video') +parser.add_argument('--ffmpeg_output_height', required=False, default=None, type=int, help='Height of the output video') parser.add_argument('--input_stream', required=False, default="rtsp://localhost:8080/channel1", type=str, help='Url of input rtsp stream') parser.add_argument('--output_stream', required=False, default="rtsp://localhost:8080/channel2", type=str, help='Url of output rtsp stream') parser.add_argument('--model_name', required=False, default="holisticTracking", type=str, help='Name of the model') @@ -54,6 +56,6 @@ def postprocess(frame, result): backend = StreamClient.OutputBackends.cv2 exact = True -client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose) +client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose,ffmpeg_output_width=args.ffmpeg_output_width, ffmpeg_output_height=args.ffmpeg_output_height) client.start(ovms_address=args.grpc_address, input_name=args.input_name, model_name=args.model_name, datatype = StreamClient.Datatypes.uint8, batch = False, limit_stream_duration = args.limit_stream_duration, limit_frames = args.limit_frames, streaming_api=True) From 1ad498fb7bae4ac1a1ee638addaa4bf4bd40c1c4 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Tue, 14 Jul 2026 15:17:50 +0530 Subject: [PATCH 03/18] Added bytetrack demo which can be used with RTSP --- demos/mediapipe/bytetrack/README.md | 42 ++++++ .../mediapipe/bytetrack/bytetrack_ovms.pbtxt | 128 ++++++++++++++++++ demos/mediapipe/bytetrack/config.json | 16 +++ 3 files changed, 186 insertions(+) create mode 100644 demos/mediapipe/bytetrack/README.md create mode 100644 demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt create mode 100644 demos/mediapipe/bytetrack/config.json diff --git a/demos/mediapipe/bytetrack/README.md b/demos/mediapipe/bytetrack/README.md new file mode 100644 index 0000000000..ae83c2a79b --- /dev/null +++ b/demos/mediapipe/bytetrack/README.md @@ -0,0 +1,42 @@ +YOLOX Tiny ONNX download link + +https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_tiny.onnx + +After downloading the onnx model, In google colab + +download onnx2tf in colab, move the onnx weights to the notebook and use the following command in colab notebook to convert onnx weights to tflite. + +```bash +!onnx2tf -i yolox_tiny.onnx -o yolox_tiny +``` + +COCO class labels download + +https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_80cl.txt + +------ +Use this command to build container for bytetrack demo +```bash +docker run -d -v $PWD:/demo -p 9000:9000 openvino/model_server:latest --config_path /demo/config.json --port 9000 +``` + +------ + + +------ +To create RTSP input channel +```bash +ffmpeg -f dshow -i video="HP True Vision FHD Camera" -f rtsp -rtsp_transport tcp rtsp://localhost:8554/channel1 +``` + + +To run real_time_stram_analysis for demo use this command +```bash +python client.py --grpc_address localhost:9000 --input_stream 'rtsp://localhost:8554/channel1' --output_stream 'rtsp://localhost:8554/channel2' --model_name ByteTrack --input_name input_video --ffmpeg_output_width 416 --ffmpeg_output_height 416 +``` + +To run RTSP channel2 output + +```bash +ffplay -loglevel verbose -rtsp_transport tcp rtsp://localhost:8554/channel2 +``` \ No newline at end of file diff --git a/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt new file mode 100644 index 0000000000..a896a9c86c --- /dev/null +++ b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt @@ -0,0 +1,128 @@ +input_stream: "IMAGE:input_video" +# input_side_packet: "INPUT_FILE_PATH:input_file_path" #simple_run_graph_main.cc +output_stream: "IMAGE:output" + +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE:input_video" + output_stream: "IMAGE:transformed_input_video" + node_options: { + [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { + output_width: 416 + output_height: 416 + } + } +} + +node { + calculator: "OpenVINOConverterCalculator" + input_stream: "IMAGE:transformed_input_video" + output_stream: "TENSORS:image_tensor" + node_options: { + [type.googleapis.com/mediapipe.OpenVINOConverterCalculatorOptions] { + enable_normalization: true + use_custom_normalization: true + custom_div: 1.0 + custom_sub: 0.0 + } + } +} +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "OpenVINOModelServerSessionCalculator" + output_side_packet: "SESSION:session" + node_options: { + [type.googleapis.com/mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { + servable_name: "yoloxt_float32" # servable name inside OVMS + servable_version: "1" + } + } +} + +node { + calculator: "OpenVINOInferenceCalculator" + input_side_packet: "SESSION:session" + input_stream: "OVTENSORS:image_tensor" + output_stream: "OVTENSORS2:detection_tensors" + node_options: { + [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions]: { + input_order_list :["images"] + output_order_list :["output"] + } + } +} + +### WRITE YOLO SPECIFIC CALCULATORS + +node{ + calculator: "OpenVINOYoloXTensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.OpenVINOYoloXTensorsToDetectionsCalculatorOptions] { + conf_thresh: 0.1 + } + } +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "detections" + output_stream: "filtered_detections" + node_options: { + [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { + min_suppression_threshold: 0.45 + max_num_detections: 100 + overlap_type: INTERSECTION_OVER_UNION + return_empty_detections: true + } + } +} + + +# Maps detection label IDs to the corresponding label text. The label map is +# provided in the label_map_path option. +node { + calculator: "DetectionLabelIdToTextCalculator" + input_stream: "filtered_detections" + output_stream: "output_detections" + node_options: { + [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { + label_map_path: "/demo/coco_80cl.txt" + } + } +} + +node { + calculator: "ByteTrackCalculator" + input_stream: "DETECTIONS:output_detections" + output_stream: "DETECTIONS:tracked_detections" + options: { + [mediapipe.ByteTrackCalculatorOptions.ext] { + track_high_threshold:0.5 + track_low_threshold:0.1 + new_track_threshold:0.35 + matching_threshold: 0.8 + track_buffer: 30 + fuse_score: false + } + } +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionColorByIdCalculator" + input_stream: "DETECTIONS:tracked_detections" + output_stream: "RENDER_DATA:detections_render_data" +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE:transformed_input_video" + input_stream: "detections_render_data" + output_stream: "IMAGE:output" +} \ No newline at end of file diff --git a/demos/mediapipe/bytetrack/config.json b/demos/mediapipe/bytetrack/config.json new file mode 100644 index 0000000000..c3045dd9b1 --- /dev/null +++ b/demos/mediapipe/bytetrack/config.json @@ -0,0 +1,16 @@ +{ + "model_config_list": [ + {"config": { + "name": "yoloxt_float32", + "base_path": "yolox_tiny_float32" + } + } + ], + "mediapipe_config_list": [ + { + "name":"ByteTrack", + "base_path":"./", + "graph_path":"bytetrack_ovms.pbtxt" + } + ] +} \ No newline at end of file From 4ec3459fa2e02e05f511361323c13be9ff45b300 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Thu, 23 Jul 2026 15:19:57 +0530 Subject: [PATCH 04/18] Using input_video instead of transformed_input_video for overlay --- demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt index a896a9c86c..fd8e60f6fa 100644 --- a/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt +++ b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt @@ -1,5 +1,4 @@ input_stream: "IMAGE:input_video" -# input_side_packet: "INPUT_FILE_PATH:input_file_path" #simple_run_graph_main.cc output_stream: "IMAGE:output" node: { @@ -102,11 +101,11 @@ node { output_stream: "DETECTIONS:tracked_detections" options: { [mediapipe.ByteTrackCalculatorOptions.ext] { - track_high_threshold:0.5 - track_low_threshold:0.1 + track_high_threshold:0.7 + track_low_threshold:0.55 new_track_threshold:0.35 matching_threshold: 0.8 - track_buffer: 30 + track_buffer: 60 fuse_score: false } } @@ -122,7 +121,7 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:transformed_input_video" + input_stream: "IMAGE:input_video" input_stream: "detections_render_data" output_stream: "IMAGE:output" } \ No newline at end of file From 89e096eb85efa5ec6cfd8f5784aae1aa0c10c5b7 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Thu, 23 Jul 2026 18:45:11 +0530 Subject: [PATCH 05/18] Update docs --- demos/mediapipe/bytetrack/README.md | 97 ++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/demos/mediapipe/bytetrack/README.md b/demos/mediapipe/bytetrack/README.md index ae83c2a79b..7163f264a0 100644 --- a/demos/mediapipe/bytetrack/README.md +++ b/demos/mediapipe/bytetrack/README.md @@ -1,42 +1,107 @@ -YOLOX Tiny ONNX download link +# ByteTrack Demo Setup + +## 1. Download the YOLOX Tiny ONNX Model + +Download the YOLOX Tiny ONNX model from the official release: https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_tiny.onnx -After downloading the onnx model, In google colab +--- + +## 2. Convert the ONNX Model to TensorFlow Lite -download onnx2tf in colab, move the onnx weights to the notebook and use the following command in colab notebook to convert onnx weights to tflite. +Open a Google Colab notebook and: + +1. Install `onnx2tf`. +2. Upload `yolox_tiny.onnx` to the notebook. +3. Run: ```bash !onnx2tf -i yolox_tiny.onnx -o yolox_tiny ``` -COCO class labels download +This generates the TensorFlow Lite model. + +--- + +## 3. Download COCO Class Labels + +Download the COCO 80-class label file: https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_80cl.txt ------- -Use this command to build container for bytetrack demo +--- + +# Running the Demo + +## 1. Start the OpenVINO Model Server + ```bash -docker run -d -v $PWD:/demo -p 9000:9000 openvino/model_server:latest --config_path /demo/config.json --port 9000 +docker run -d \ + -v $PWD:/demo \ + -p 9000:9000 \ + openvino/model_server:latest \ + --config_path /demo/config.json \ + --port 9000 ``` ------- +--- + +## 2. Create an RTSP Input Stream +Use FFmpeg to publish your webcam as an RTSP stream. ------- -To create RTSP input channel ```bash -ffmpeg -f dshow -i video="HP True Vision FHD Camera" -f rtsp -rtsp_transport tcp rtsp://localhost:8554/channel1 +ffmpeg -f dshow -video_size 1280x720 \ +-i video="HP True Vision FHD Camera" \ +-f rtsp -rtsp_transport tcp \ +rtsp://localhost:8554/channel1 ``` +> **Work in Progress:** The following H.264-based streaming command is still being evaluated and may not work correctly in all setups. -To run real_time_stram_analysis for demo use this command ```bash -python client.py --grpc_address localhost:9000 --input_stream 'rtsp://localhost:8554/channel1' --output_stream 'rtsp://localhost:8554/channel2' --model_name ByteTrack --input_name input_video --ffmpeg_output_width 416 --ffmpeg_output_height 416 +ffmpeg -f dshow \ +-video_size 1280x720 \ +-framerate 30 \ +-i video="HP True Vision FHD Camera" \ +-c:v libx264 \ +-crf 18 \ +-preset veryfast \ +-f rtsp \ +-rtsp_transport tcp \ +rtsp://localhost:8554/channel1 ``` -To run RTSP channel2 output +--- + +## 3. Run the Real-Time Stream Analysis Client ```bash -ffplay -loglevel verbose -rtsp_transport tcp rtsp://localhost:8554/channel2 -``` \ No newline at end of file +python client.py \ + --grpc_address localhost:9000 \ + --input_stream rtsp://localhost:8554/channel1 \ + --output_stream rtsp://localhost:8554/channel2 \ + --model_name ByteTrack \ + --input_name input_video +``` + +--- + +## 4. View the Output Stream + +**Option 1 (recommended):** + +```bash +ffplay -rtsp_transport tcp \ +-vf "scale=704:704,format=yuv420p" \ +rtsp://localhost:8554/channel2 +``` + +**Option 2 (verbose logging):** + +```bash +ffplay -loglevel verbose \ +-rtsp_transport tcp \ +rtsp://localhost:8554/channel2 +``` From f1615dafc4530714a4f962848e63e66708334a04 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Thu, 6 Aug 2026 18:15:47 +0530 Subject: [PATCH 06/18] add logic to check timestamps --- src/kfs_frontend/kfs_graph_executor_impl.cpp | 22 +++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/kfs_frontend/kfs_graph_executor_impl.cpp b/src/kfs_frontend/kfs_graph_executor_impl.cpp index 5c197db6f8..fbb051d5bd 100644 --- a/src/kfs_frontend/kfs_graph_executor_impl.cpp +++ b/src/kfs_frontend/kfs_graph_executor_impl.cpp @@ -1027,15 +1027,16 @@ static inline Status checkTimestamp(const KFSRequest& request, const ::mediapipe return StatusCode::OK; } +// this method static Status deserializeTimestampIfAvailable( const KFSRequest& request, - ::mediapipe::Timestamp& timestamp) { + ::mediapipe::Timestamp& timestamp, + const ::mediapipe::Timestamp& lastPushedTimestamp) { // pass in current stream state auto timestampParamIt = request.parameters().find(TIMESTAMP_PARAMETER_NAME); if (timestampParamIt != request.parameters().end()) { SPDLOG_DEBUG("Found {} timestamp parameter in request for: {}", TIMESTAMP_PARAMETER_NAME, request.model_name()); auto& parameterChoice = timestampParamIt->second; if (parameterChoice.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kInt64Param) { - // Cannot create with error checking since error check = abseil death test timestamp = ::mediapipe::Timestamp::CreateNoErrorChecking(parameterChoice.int64_param()); if (!timestamp.IsRangeValue()) { SPDLOG_DEBUG("Timestamp not in range: {}; for request to: {};", timestamp.DebugString(), request.model_name()); @@ -1047,8 +1048,17 @@ static Status deserializeTimestampIfAvailable( return status; } } else { - auto now = std::chrono::system_clock::now(); - timestamp = ::mediapipe::Timestamp(std::chrono::duration_cast(now.time_since_epoch()).count()); + // steady_clock never jumps backward; system_clock can + auto now = std::chrono::steady_clock::now().time_since_epoch(); + int64_t candidate = std::chrono::duration_cast(now).count(); + timestamp = ::mediapipe::Timestamp::CreateNoErrorChecking(candidate); + } + + // Enforce strict monotonicity regardless of source + if (lastPushedTimestamp != ::mediapipe::Timestamp::Unset() && timestamp <= lastPushedTimestamp) { + SPDLOG_DEBUG("Non-monotonic timestamp detected: new={} <= last={}; for request to: {}", + timestamp.DebugString(), lastPushedTimestamp.DebugString(), request.model_name()); + return Status(StatusCode::MEDIAPIPE_INVALID_TIMESTAMP, "Timestamp did not increase relative to previous packet in stream"); } return StatusCode::OK; } @@ -1133,6 +1143,7 @@ Status onPacketReadySerializeImpl( return status; } +// this method Status createAndPushPacketsImpl( std::shared_ptr request, stream_types_mapping_t& inputTypes, @@ -1141,7 +1152,8 @@ Status createAndPushPacketsImpl( ::mediapipe::Timestamp& currentTimestamp, size_t& numberOfPacketsCreated) { - OVMS_RETURN_ON_FAIL(deserializeTimestampIfAvailable(*request, currentTimestamp)); + const ::mediapipe::Timestamp lastPushedTimestamp = currentTimestamp; // snapshot BEFORE overwrite + OVMS_RETURN_ON_FAIL(deserializeTimestampIfAvailable(*request, currentTimestamp, lastPushedTimestamp)); OVMS_RETURN_ON_FAIL(checkTimestamp(*request, currentTimestamp)); OVMS_RETURN_ON_FAIL(validateRequestCoherencyKFS(*request, request->model_name(), MediapipeGraphDefinition::VERSION)); From 155f27eb2365a16b4387a3eea0d2dd4275e5113b Mon Sep 17 00:00:00 2001 From: Vishwa2684 <115179890+Vishwa2684@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:59:15 +0530 Subject: [PATCH 07/18] formatting in client.py Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- demos/real_time_stream_analysis/python/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/real_time_stream_analysis/python/client.py b/demos/real_time_stream_analysis/python/client.py index 39934f6e80..7da26ab66a 100755 --- a/demos/real_time_stream_analysis/python/client.py +++ b/demos/real_time_stream_analysis/python/client.py @@ -56,6 +56,6 @@ def postprocess(frame, result): backend = StreamClient.OutputBackends.cv2 exact = True -client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose,ffmpeg_output_width=args.ffmpeg_output_width, ffmpeg_output_height=args.ffmpeg_output_height) +client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose, ffmpeg_output_width=args.ffmpeg_output_width, ffmpeg_output_height=args.ffmpeg_output_height) client.start(ovms_address=args.grpc_address, input_name=args.input_name, model_name=args.model_name, datatype = StreamClient.Datatypes.uint8, batch = False, limit_stream_duration = args.limit_stream_duration, limit_frames = args.limit_frames, streaming_api=True) From fdb2dac5722f100f2692eaab9bb2ea147c976cd9 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Wed, 12 Aug 2026 16:49:14 +0530 Subject: [PATCH 08/18] Write a custom calculator to process yolox tiny's ov tensors --- .../mediapipe/bytetrack/bytetrack_ovms.pbtxt | 31 +++- demos/mediapipe/bytetrack/download_models.py | 88 ++++++++++ demos/mediapipe/bytetrack/requirements.txt | Bin 0 -> 316 bytes src/BUILD | 1 + src/yolox/BUILD | 37 ++++ ..._yolox_tensors_to_detections_calculator.cc | 162 ++++++++++++++++++ ...lox_tensors_to_detections_calculator.proto | 17 ++ 7 files changed, 327 insertions(+), 9 deletions(-) create mode 100644 demos/mediapipe/bytetrack/download_models.py create mode 100644 demos/mediapipe/bytetrack/requirements.txt create mode 100644 src/yolox/BUILD create mode 100644 src/yolox/ov_yolox_tensors_to_detections_calculator.cc create mode 100644 src/yolox/ov_yolox_tensors_to_detections_calculator.proto diff --git a/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt index fd8e60f6fa..91e1fb0770 100644 --- a/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt +++ b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt @@ -47,21 +47,34 @@ node { output_stream: "OVTENSORS2:detection_tensors" node_options: { [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions]: { - input_order_list :["images"] - output_order_list :["output"] + input_order_list :["image"] + output_order_list :["boxes","labels"] } } } -### WRITE YOLO SPECIFIC CALCULATORS +### OpenVINOYoloXTensorsToDetectionsCalculator was developed for TFLite specific model, but we can use OVYoloXTensorsToDetectionsCalculator for OpenVINO model. -node{ - calculator: "OpenVINOYoloXTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - output_stream: "DETECTIONS:detections" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOYoloXTensorsToDetectionsCalculatorOptions] { +#node{ +# calculator: "OpenVINOYoloXTensorsToDetectionsCalculator" +# input_stream: "TENSORS:detection_tensors" +# output_stream: "DETECTIONS:detections" +# node_options: { +# [type.googleapis.com/mediapipe.OpenVINOYoloXTensorsToDetectionsCalculatorOptions] { +# conf_thresh: 0.1 +# } +# } +# } + +node { + calculator: "OVYoloXTensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + output_stream: "DETECTIONS:detections" + + node_options: { + [type.googleapis.com/mediapipe.OVYoloXTensorsToDetectionsCalculatorOptions] { conf_thresh: 0.1 + input_size: 416.0 } } } diff --git a/demos/mediapipe/bytetrack/download_models.py b/demos/mediapipe/bytetrack/download_models.py new file mode 100644 index 0000000000..06ff387c36 --- /dev/null +++ b/demos/mediapipe/bytetrack/download_models.py @@ -0,0 +1,88 @@ +import glob +import json +import os +import requests +import openvino as ov +from huggingface_hub import snapshot_download + +os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1" + +COCO_CLASSES_URL = "https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_80cl.txt" + +MODEL_REPO = "OpenVINO/yolox_tiny-fp16-ov" +# --------------------------------------------------------- +# 1. Download model +# --------------------------------------------------------- + +model_dir = snapshot_download(repo_id=MODEL_REPO) + +xml_path = glob.glob(os.path.join(model_dir, "*.xml"))[0] +config_path = os.path.join(model_dir, "config.json") + +print("Found IR :", xml_path) +print("Found config:", config_path) + + +# --------------------------------------------------------- +# 2. Read config.json +# --------------------------------------------------------- + +with open(config_path, "r") as f: + config = json.load(f) + +print("\nModel config:") +print("model_name :", config.get("model_name")) +print("model_type :", config.get("model_type")) +print("input_type :", config.get("input_dtype")) +print("mean_values:", config.get("mean_values")) +print("scale_values:", config.get("scale_values")) + + +# --------------------------------------------------------- +# 3. Parse mean and scale values +# --------------------------------------------------------- +mean_values = [float(x) for x in config["mean_values"].split()] +scale_values = [float(x) for x in config["scale_values"].split()] + +print("\nParsed preprocessing:") +print("mean :", mean_values) +print("scale:", scale_values) + +# --------------------------------------------------------- +# 4. Load OpenVINO model +# --------------------------------------------------------- +core = ov.Core() +model = core.read_model(xml_path) +# --------------------------------------------------------- +# 5. Configure preprocessing +# --------------------------------------------------------- +ppp = ov.preprocess.PrePostProcessor(model) +inp = ppp.input(0) + +# Input coming from user/image: +# f32 NHWC +inp.tensor().set_element_type(ov.Type.f32) .set_layout(ov.Layout("NHWC")) + +# Model expects: +# float32 NCHW +inp.model().set_layout(ov.Layout("NCHW")) + +# Preprocessing: + +inp.preprocess().convert_element_type(ov.Type.f32).convert_layout(ov.Layout("NCHW")).scale(255.0).mean(mean_values).scale(scale_values) +# --------------------------------------------------------- +# 6. Build and save +# --------------------------------------------------------- + +model = ppp.build() + +output_path = "yolox_tiny_float32/1/yolox_tiny_full_preprocess.xml" + +ov.save_model(model, output_path) + +print("\nSaved:", os.path.abspath(output_path)) + +with open("coco_80cl.txt", "wb") as f: + f.write(requests.get(COCO_CLASSES_URL).content) + +print("Downloaded successfully") \ No newline at end of file diff --git a/demos/mediapipe/bytetrack/requirements.txt b/demos/mediapipe/bytetrack/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..be53447c3b774c5a54dfe278369e1d474f638cff GIT binary patch literal 316 zcmZ9H+YW*-5JczM#7~jnT@&#~h!0u;Nu)(U5`JDiEj}32wApQE&TQYWsU~u&mC4Ge z*Nl2lryI9cMfGAy-HLLSH*dSa;HASD3!u^s$h9dp`3ez;uHuC)r0p; zEom(|i54U*S&`zPDbzqR_+98b`VzID%#c*#w)83VQ8}`6va0Z>;_p8`C5G>l#*7-L b3(O1C +#include + +#include + +#include "src/yolox/ov_yolox_tensors_to_detections_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/location_data.pb.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +class OVYoloXTensorsToDetectionsCalculator : public CalculatorBase { + static const std::string INPUT_TAG_NAME; + static const std::string OUTPUT_TAG_NAME; + + float confidenceThreshold_ = 0.1f; + float inputSize_ = 416.0f; + + public: + static absl::Status GetContract(CalculatorContract* cc) { + RET_CHECK(!cc->Inputs().GetTags().empty()); + RET_CHECK(!cc->Outputs().GetTags().empty()); + + cc->Inputs() + .Tag(INPUT_TAG_NAME) + .Set>(); + + cc->Outputs() + .Tag(OUTPUT_TAG_NAME) + .Set>(); + + return absl::OkStatus(); + } + + absl::Status Open(CalculatorContext* cc) final { + const auto& options = + cc->Options< + mediapipe::OVYoloXTensorsToDetectionsCalculatorOptions>(); + + confidenceThreshold_ = + options.has_conf_thresh() ? options.conf_thresh() : 0.1f; + + inputSize_ = + options.has_input_size() ? options.input_size() : 416.0f; + + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) final { + if (cc->Inputs().Tag(INPUT_TAG_NAME).IsEmpty()) { + return absl::OkStatus(); + } + + const auto& tensors = + cc->Inputs() + .Tag(INPUT_TAG_NAME) + .Get>(); + + RET_CHECK_EQ(tensors.size(), 2u); + + const auto& boxesTensor = tensors[0]; + const auto& labelsTensor = tensors[1]; + + RET_CHECK_EQ(boxesTensor.get_element_type(), ov::element::f32); + RET_CHECK_EQ(labelsTensor.get_element_type(), ov::element::i64); + + const auto boxesShape = boxesTensor.get_shape(); + const auto labelsShape = labelsTensor.get_shape(); + + RET_CHECK_EQ(boxesShape.size(), 3u); + RET_CHECK_EQ(boxesShape[0], 1u); + RET_CHECK_EQ(boxesShape[2], 5u); + + RET_CHECK_EQ(labelsShape.size(), 2u); + RET_CHECK_EQ(labelsShape[0], 1u); + RET_CHECK_EQ(labelsShape[1], boxesShape[1]); + + const size_t numBoxes = boxesShape[1]; + + const float* boxes = boxesTensor.data(); + const int64_t* labels = labelsTensor.data(); + + RET_CHECK(boxes != nullptr); + RET_CHECK(labels != nullptr); + + auto detections = + absl::make_unique>(); + + for (size_t i = 0; i < numBoxes; ++i) { + const float x1 = boxes[i * 5 + 0]; + const float y1 = boxes[i * 5 + 1]; + const float x2 = boxes[i * 5 + 2]; + const float y2 = boxes[i * 5 + 3]; + const float confidence = boxes[i * 5 + 4]; + + if (confidence < confidenceThreshold_) { + continue; + } + + if (x2 <= x1 || y2 <= y1) { + continue; + } + + Detection detection; + + auto* locationData = + detection.mutable_location_data(); + + locationData->set_format( + LocationData::RELATIVE_BOUNDING_BOX); + + auto* boundingBox = + locationData->mutable_relative_bounding_box(); + + boundingBox->set_xmin(x1 / inputSize_); + boundingBox->set_ymin(y1 / inputSize_); + boundingBox->set_width((x2 - x1) / inputSize_); + boundingBox->set_height((y2 - y1) / inputSize_); + + detection.add_score(confidence); + detection.add_label_id( + static_cast(labels[i])); + + detections->emplace_back(std::move(detection)); + } + + cc->Outputs() + .Tag(OUTPUT_TAG_NAME) + .Add(detections.release(), cc->InputTimestamp()); + + return absl::OkStatus(); + } +}; + +const std::string OVYoloXTensorsToDetectionsCalculator::INPUT_TAG_NAME{ + "TENSORS"}; + +const std::string OVYoloXTensorsToDetectionsCalculator::OUTPUT_TAG_NAME{ + "DETECTIONS"}; + +REGISTER_CALCULATOR(OVYoloXTensorsToDetectionsCalculator); + +} // namespace mediapipe \ No newline at end of file diff --git a/src/yolox/ov_yolox_tensors_to_detections_calculator.proto b/src/yolox/ov_yolox_tensors_to_detections_calculator.proto new file mode 100644 index 0000000000..6517d2a05b --- /dev/null +++ b/src/yolox/ov_yolox_tensors_to_detections_calculator.proto @@ -0,0 +1,17 @@ +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message OVYoloXTensorsToDetectionsCalculatorOptions { + extend .mediapipe.CalculatorOptions { + optional OVYoloXTensorsToDetectionsCalculatorOptions ext = 211376658; + } + + // Minimum confidence required for a detection. + optional float conf_thresh = 1 [default = 0.10]; + + // Input resolution of the YOLOX model. + optional float input_size = 3 [default = 416.0]; +} \ No newline at end of file From a7a25d91d64242bb1716bab34001ac432e1f201e Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Thu, 13 Aug 2026 22:42:33 +0530 Subject: [PATCH 09/18] Move bytetrack logic from seperate mediapipe repo to OVMS --- WORKSPACE | 5 +- ci/lib_search.py | 1 + demos/mediapipe/bytetrack/README.md | 130 +++--- .../mediapipe/bytetrack/bytetrack_ovms.pbtxt | 6 +- demos/mediapipe/bytetrack/config.json | 4 +- demos/mediapipe/bytetrack/download_models.py | 58 ++- src/BUILD | 1 + src/bytetrack/calculators/BUILD | 124 +++++ src/bytetrack/calculators/basetrack.cc | 10 + src/bytetrack/calculators/basetrack.h | 42 ++ .../calculators/bytetrack_calculator.cc | 424 ++++++++++++++++++ .../calculators/bytetrack_calculator.h | 53 +++ .../calculators/bytetrack_calculator.proto | 17 + src/bytetrack/calculators/kalman_filter.cc | 107 +++++ src/bytetrack/calculators/kalman_filter.h | 34 ++ src/bytetrack/calculators/kalman_matrices.h | 28 ++ src/bytetrack/calculators/matching_utils.h | 335 ++++++++++++++ .../render_data_passthrough_calculator.cc | 31 ++ src/bytetrack/calculators/strack.cc | 150 +++++++ src/bytetrack/calculators/strack.h | 60 +++ src/bytetrack/utils/BUILD | 29 ++ .../utils/detection_color_by_id_calculator.cc | 139 ++++++ .../detection_color_by_id_calculator.proto | 15 + ..._yolox_tensors_to_detections_calculator.cc | 2 +- third_party/mediapipe_calculators/BUILD | 2 - 25 files changed, 1724 insertions(+), 83 deletions(-) create mode 100644 src/bytetrack/calculators/BUILD create mode 100644 src/bytetrack/calculators/basetrack.cc create mode 100644 src/bytetrack/calculators/basetrack.h create mode 100644 src/bytetrack/calculators/bytetrack_calculator.cc create mode 100644 src/bytetrack/calculators/bytetrack_calculator.h create mode 100644 src/bytetrack/calculators/bytetrack_calculator.proto create mode 100644 src/bytetrack/calculators/kalman_filter.cc create mode 100644 src/bytetrack/calculators/kalman_filter.h create mode 100644 src/bytetrack/calculators/kalman_matrices.h create mode 100644 src/bytetrack/calculators/matching_utils.h create mode 100644 src/bytetrack/calculators/render_data_passthrough_calculator.cc create mode 100644 src/bytetrack/calculators/strack.cc create mode 100644 src/bytetrack/calculators/strack.h create mode 100644 src/bytetrack/utils/BUILD create mode 100644 src/bytetrack/utils/detection_color_by_id_calculator.cc create mode 100644 src/bytetrack/utils/detection_color_by_id_calculator.proto diff --git a/WORKSPACE b/WORKSPACE index 5bc3259ee5..c331e27718 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -176,9 +176,8 @@ http_archive( #### Will be used on feature release git_repository( name = "mediapipe", - remote = "https://github.com/Vishwa2684/mediapipe_ovms", - # commit = "12e8d511cfbc5f471c498278a65a02dd250963e8", # top of mediapipe main branch as of 26.11.2025 - branch = "custom_bytetrack_graph" + remote = "https://github.com/openvinotoolkit/mediapipe", + commit = "12e8d511cfbc5f471c498278a65a02dd250963e8", # top of mediapipe main branch as of 26.11.2025 ) # DEV mediapipe 1 source - adjust local repository path for build diff --git a/ci/lib_search.py b/ci/lib_search.py index 29bb3c38e9..86901e416a 100644 --- a/ci/lib_search.py +++ b/ci/lib_search.py @@ -113,6 +113,7 @@ def check_dir(start_dir): 'ovms-c/dist', 'requirements.txt', 'requirements_win.txt', + 'bytetrack_ovms.pbtxt', 'resnet_images.txt', "resnet_labels.txt", 'rest_sdk_v2.10.16.patch', diff --git a/demos/mediapipe/bytetrack/README.md b/demos/mediapipe/bytetrack/README.md index 7163f264a0..c5cd0fe247 100644 --- a/demos/mediapipe/bytetrack/README.md +++ b/demos/mediapipe/bytetrack/README.md @@ -1,107 +1,121 @@ # ByteTrack Demo Setup -## 1. Download the YOLOX Tiny ONNX Model +End-to-end demo: video source (webcam / file / RTSP) → OpenVINO Model Server (YOLOX Tiny+ ByteTrack) → output (screen / file / RTSP). -Download the YOLOX Tiny ONNX model from the official release: +--- -https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_tiny.onnx +## 1. Model Preparation ---- +The detector stage of the pipeline runs on [OpenVINO](https://github.com/openvinotoolkit/openvino)-optimized YOLOX models. The following FP16 variants are currently supported: -## 2. Convert the ONNX Model to TensorFlow Lite +| Model | HuggingFace Repo | +|---|---| +| YOLOX-Tiny (fp16 precision)| `OpenVINO/yolox_tiny-fp16-ov` | +| YOLOX-Tiny (int8 precision)| `OpenVINO/yolox_tiny-int8-ov` | -Open a Google Colab notebook and: +`yolox_tiny-fp16-ov` is the default used in this demo. -1. Install `onnx2tf`. -2. Upload `yolox_tiny.onnx` to the notebook. -3. Run: +### Install requirements ```bash -!onnx2tf -i yolox_tiny.onnx -o yolox_tiny +pip install -r requirements.txt ``` -This generates the TensorFlow Lite model. +### Download a model + +```bash +python download_models.py --model OpenVINO/yolox_tiny-fp16-ov +``` + +> Swap `--model` for any of the four repo IDs listed above to use a different YOLOX size. The script also downloads the COCO class list used for labeling detections. + +This populates the local model directory that `config.json` (used by the OpenVINO Model Server in step 3) points to, and that ByteTrack consumes downstream for tracking. --- -## 3. Download COCO Class Labels +## 2. Start an RTSP relay server -Download the COCO 80-class label file: +If you don't already have one running, start MediaMTX (or an equivalent RTSP server) so the streams below have somewhere to publish to: -https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_80cl.txt +```bash +docker run --rm -d -p 8554:8554 -e RTSP_PROTOCOLS=tcp bluenviron/mediamtx:latest +``` ---- +> Only needed if you plan to use RTSP input and/or output (see Demo 3 below). It isn't required for the local webcam→screen or video-file→mp4 demos. -# Running the Demo +--- -## 1. Start the OpenVINO Model Server +## 3. Start the OpenVINO Model Server ```bash -docker run -d \ - -v $PWD:/demo \ - -p 9000:9000 \ - openvino/model_server:latest \ - --config_path /demo/config.json \ - --port 9000 +docker run -d -v $PWD:/demo -p 9000:9000 openvino/model_server:latest --config_path /demo/config.json --port 9000 ``` --- -## 2. Create an RTSP Input Stream +## 4. Run the Demo + +The `client.py` script (from `real_time_stream_analysis`) supports several combinations of input and output, so the same server can be exercised in different ways depending on what you have available. + +### Demo A — Local webcam → screen -Use FFmpeg to publish your webcam as an RTSP stream. +Reads directly from a local camera and renders the tracked output in a window. ```bash -ffmpeg -f dshow -video_size 1280x720 \ --i video="HP True Vision FHD Camera" \ --f rtsp -rtsp_transport tcp \ -rtsp://localhost:8554/channel1 +python client.py --grpc_address localhost:9000 --input_stream 0 --output_stream screen ``` -> **Work in Progress:** The following H.264-based streaming command is still being evaluated and may not work correctly in all setups. +- `--input_stream 0` — camera device ID `0` (use `1`, `2`, etc. for additional cameras). +- `--output_stream screen` — opens a live preview window instead of writing to a file or stream. + +### Demo B — Video file → video file + +Reads from an encoded video file and writes the annotated result to a new video file. ```bash -ffmpeg -f dshow \ --video_size 1280x720 \ --framerate 30 \ --i video="HP True Vision FHD Camera" \ --c:v libx264 \ --crf 18 \ --preset veryfast \ --f rtsp \ --rtsp_transport tcp \ -rtsp://localhost:8554/channel1 +curl -L "https://raw.githubusercontent.com/FoundationVision/ByteTrack/main/videos/palace.mp4" -o video.mp4 +python client.py --grpc_address localhost:9000 --input_stream video.mp4 --output_stream output.mp4 ``` ---- +- `--input_stream video.mp4` — path to the source video. +- `--output_stream output.mp4` — path where the tracked/annotated video is saved. + +### Demo C — RTSP → RTSP -## 3. Run the Real-Time Stream Analysis Client +Full end-to-end streaming demo: publish a webcam feed to an RTSP endpoint, run detection + tracking on it, and publish the annotated result to a second RTSP endpoint. + +**1. Publish your webcam as an RTSP input stream** ```bash -python client.py \ - --grpc_address localhost:9000 \ - --input_stream rtsp://localhost:8554/channel1 \ - --output_stream rtsp://localhost:8554/channel2 \ - --model_name ByteTrack \ - --input_name input_video +ffmpeg -f dshow -video_size 1280x720 -i video="HP True Vision FHD Camera" -f rtsp -rtsp_transport tcp rtsp://localhost:8554/channel1 ``` ---- +**2. Run the client against the RTSP input/output** -## 4. View the Output Stream +```bash +python client.py --grpc_address localhost:9000 --input_stream rtsp://localhost:8554/channel1 --output_stream rtsp://localhost:8554/channel2 --model_name ByteTrack --input_name input_video +``` + +**3. View the output stream** -**Option 1 (recommended):** +Option 1 (recommended): ```bash -ffplay -rtsp_transport tcp \ --vf "scale=704:704,format=yuv420p" \ -rtsp://localhost:8554/channel2 +ffplay -rtsp_transport tcp -vf "scale=704:704,format=yuv420p" rtsp://localhost:8554/channel2 ``` -**Option 2 (verbose logging):** +Option 2 (verbose logging): ```bash -ffplay -loglevel verbose \ --rtsp_transport tcp \ -rtsp://localhost:8554/channel2 +ffplay -loglevel verbose -rtsp_transport tcp rtsp://localhost:8554/channel2 ``` + +--- + +## Summary of I/O Options + +| Demo | Input | Output | RTSP server required? | +|---|---|---|---| +| A | Local webcam (`0`) | `screen` | No | +| B | Video file (`video.mp4`) | Video file (`output.mp4`) | No | +| C | RTSP stream | RTSP stream | Yes | \ No newline at end of file diff --git a/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt index 91e1fb0770..9969e38350 100644 --- a/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt +++ b/demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt @@ -26,15 +26,13 @@ node { } } } -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. + node { calculator: "OpenVINOModelServerSessionCalculator" output_side_packet: "SESSION:session" node_options: { [type.googleapis.com/mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "yoloxt_float32" # servable name inside OVMS + servable_name: "yolox_tiny-fp16-ov" # servable name inside OVMS servable_version: "1" } } diff --git a/demos/mediapipe/bytetrack/config.json b/demos/mediapipe/bytetrack/config.json index c3045dd9b1..f907d2d92b 100644 --- a/demos/mediapipe/bytetrack/config.json +++ b/demos/mediapipe/bytetrack/config.json @@ -1,8 +1,8 @@ { "model_config_list": [ {"config": { - "name": "yoloxt_float32", - "base_path": "yolox_tiny_float32" + "name": "yolox_tiny-fp16-ov", + "base_path": "yolox_tiny-fp16-ov" } } ], diff --git a/demos/mediapipe/bytetrack/download_models.py b/demos/mediapipe/bytetrack/download_models.py index 06ff387c36..997af5a82c 100644 --- a/demos/mediapipe/bytetrack/download_models.py +++ b/demos/mediapipe/bytetrack/download_models.py @@ -1,15 +1,38 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import glob import json +import argparse import os -import requests import openvino as ov from huggingface_hub import snapshot_download os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1" -COCO_CLASSES_URL = "https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_80cl.txt" +parser = argparse.ArgumentParser() +parser.add_argument( + "--model-repo", + default="OpenVINO/yolox_tiny-fp16-ov", + help="Hugging Face model repository", +) +args = parser.parse_args() -MODEL_REPO = "OpenVINO/yolox_tiny-fp16-ov" +MODEL_REPO = args.model_repo +MODEL_NAME = MODEL_REPO.split("/")[-1] # --------------------------------------------------------- # 1. Download model # --------------------------------------------------------- @@ -36,10 +59,15 @@ print("input_type :", config.get("input_dtype")) print("mean_values:", config.get("mean_values")) print("scale_values:", config.get("scale_values")) +print("classes:", config.get("labels")) +# --------------------------------------------------------- +# 3. Prepare classes list +# --------------------------------------------------------- +classes = config.get("labels").split(" ") # --------------------------------------------------------- -# 3. Parse mean and scale values +# 4. Parse mean and scale values # --------------------------------------------------------- mean_values = [float(x) for x in config["mean_values"].split()] scale_values = [float(x) for x in config["scale_values"].split()] @@ -49,19 +77,19 @@ print("scale:", scale_values) # --------------------------------------------------------- -# 4. Load OpenVINO model +# 5. Load OpenVINO model # --------------------------------------------------------- core = ov.Core() model = core.read_model(xml_path) # --------------------------------------------------------- -# 5. Configure preprocessing +# 6. Configure preprocessing # --------------------------------------------------------- ppp = ov.preprocess.PrePostProcessor(model) inp = ppp.input(0) # Input coming from user/image: # f32 NHWC -inp.tensor().set_element_type(ov.Type.f32) .set_layout(ov.Layout("NHWC")) +inp.tensor().set_element_type(ov.Type.f32).set_layout(ov.Layout("NHWC")) # Model expects: # float32 NCHW @@ -69,20 +97,24 @@ # Preprocessing: -inp.preprocess().convert_element_type(ov.Type.f32).convert_layout(ov.Layout("NCHW")).scale(255.0).mean(mean_values).scale(scale_values) +inp.preprocess().convert_element_type(ov.Type.f32).convert_layout( + ov.Layout("NCHW") +).scale(255.0).mean(mean_values).scale(scale_values) # --------------------------------------------------------- -# 6. Build and save +# 7. Build and save # --------------------------------------------------------- model = ppp.build() -output_path = "yolox_tiny_float32/1/yolox_tiny_full_preprocess.xml" +output_path = f"{MODEL_NAME}/1/{MODEL_NAME}.xml" ov.save_model(model, output_path) print("\nSaved:", os.path.abspath(output_path)) -with open("coco_80cl.txt", "wb") as f: - f.write(requests.get(COCO_CLASSES_URL).content) +with open("coco_80cl.txt", "w") as f: + n = len(classes) + for i, c in enumerate(classes): + f.write(c + ("\n" if i < n - 1 else "")) -print("Downloaded successfully") \ No newline at end of file +print("Downloaded successfully") diff --git a/src/BUILD b/src/BUILD index cc85e8b4b2..6d6261b2fd 100644 --- a/src/BUILD +++ b/src/BUILD @@ -758,6 +758,7 @@ ovms_cc_library( "//src/audio/speech_to_text:s2t_calculator", "//src/audio/text_to_speech:t2s_calculator", "//src/yolox:ov_yolox_tensors_to_detections_calculator", + "//src/bytetrack/calculators:bytetrack_calculators", "//src/audio:audio_utils", "//src/image_gen:imagegen_init", "//src/llm:openai_responses_handler", diff --git a/src/bytetrack/calculators/BUILD b/src/bytetrack/calculators/BUILD new file mode 100644 index 0000000000..6d4e43619c --- /dev/null +++ b/src/bytetrack/calculators/BUILD @@ -0,0 +1,124 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "bytetrack_calculators", + deps = [ + "//src/bytetrack/utils:detection_color_by_id_calculator", + ":kalman_filter", + ":kalman_matrices", + ":strack", + ":bytetrack_calculator", + ":render_data_passthrough", + ], + alwayslink = 1 +) + +mediapipe_proto_library( + name = "bytetrack_calculator_proto", + visibility = ["//visibility:public"], + srcs = ["bytetrack_calculator.proto"], + deps=[ + "@mediapipe//mediapipe/framework:calculator_options_proto", + "@mediapipe//mediapipe/framework:calculator_proto", + ], + alwayslink = 1 +) + +cc_library( + name = "kalman_matrices", + hdrs = ["kalman_matrices.h"], + deps = ["@eigen_archive//:eigen3"], + alwayslink = 1 +) + + +cc_library( + name = "matching_utils", + hdrs = ["matching_utils.h"], + deps = [ + "@mediapipe//mediapipe/util/tracking:box_tracker", + ], + alwayslink = 1 +) + +cc_library( + name = "kalman_filter", + srcs = ["kalman_filter.cc"], + hdrs = ["kalman_filter.h"], + deps = [ + ":kalman_matrices", + "@mediapipe//mediapipe/framework/formats:detection_cc_proto", + "@eigen_archive//:eigen3", + ], + alwayslink = 1 +) + +cc_library( + name = "basetrack", + srcs = ["basetrack.cc"], + hdrs = ["basetrack.h"], + alwayslink = 1 +) + +cc_library( + name = "strack", + srcs = ["strack.cc"], + hdrs = ["strack.h"], + deps = [ + ":kalman_filter", + ":basetrack", + "@mediapipe//mediapipe/framework/formats:detection_cc_proto", + "@eigen_archive//:eigen3", + ], + alwayslink = 1 +) + +cc_library( + name = "bytetrack_calculator", + srcs = ["bytetrack_calculator.cc"], + hdrs = ["bytetrack_calculator.h"], + deps = [ + "@mediapipe//mediapipe/framework:calculator_framework", + "@mediapipe//mediapipe/framework/formats:detection_cc_proto", + ":bytetrack_calculator_cc_proto", + "@mediapipe//mediapipe/framework/port:status", + ":matching_utils", + ":strack", + ":kalman_filter", + ":kalman_matrices", + "@eigen_archive//:eigen3", + ], + alwayslink = 1, +) + +# This calculator is for debugging to check whether render data is correctly passed through the graph. +cc_library( + name = "render_data_passthrough", + srcs = ["render_data_passthrough_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "@mediapipe//mediapipe/framework:calculator_framework", + "@mediapipe//mediapipe/framework/formats:detection_cc_proto", + "@mediapipe//mediapipe/framework/port:status", + "@mediapipe//mediapipe/util:render_data_cc_proto", + ], + alwayslink = 1 +) diff --git a/src/bytetrack/calculators/basetrack.cc b/src/bytetrack/calculators/basetrack.cc new file mode 100644 index 0000000000..1929bc17f2 --- /dev/null +++ b/src/bytetrack/calculators/basetrack.cc @@ -0,0 +1,10 @@ +// basetrack.cpp +#include "src/bytetrack/calculators/basetrack.h" + +namespace mediapipe { +namespace bytetrack { + +int BaseTrack::count_ = 0; + +} // namespace bytetrack +} // namespace mediapipe \ No newline at end of file diff --git a/src/bytetrack/calculators/basetrack.h b/src/bytetrack/calculators/basetrack.h new file mode 100644 index 0000000000..439380c8aa --- /dev/null +++ b/src/bytetrack/calculators/basetrack.h @@ -0,0 +1,42 @@ +#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BASETRACK_H_ +#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BASETRACK_H_ + +namespace mediapipe { +namespace bytetrack { + +class BaseTrack { +public: + enum class TrackState { NEW, + TRACKED, + LOST, + REMOVED }; + + // mirrors Python's next_id() — static counter owned here + static int next_id() { return ++count_; } + static void reset_id() { count_ = 0; } + + // Accessors + int track_id() const { return track_id_; } + int frame_id() const { return frame_id_; } + int start_frame() const { return start_frame_; } + float score() const { return score_; } + TrackState state() const { return state_; } + bool is_activated() const { return is_activated_; } + + void MarkLost() { state_ = TrackState::LOST; } + void MarkRemoved() { state_ = TrackState::REMOVED; } + +protected: + int track_id_ = 0; + int frame_id_ = 0; + int start_frame_ = 0; + float score_ = 0.f; + bool is_activated_ = false; + TrackState state_ = TrackState::NEW; + static int count_; +}; + +} // namespace bytetrack +} // namespace mediapipe + +#endif \ No newline at end of file diff --git a/src/bytetrack/calculators/bytetrack_calculator.cc b/src/bytetrack/calculators/bytetrack_calculator.cc new file mode 100644 index 0000000000..bbb02a8f0c --- /dev/null +++ b/src/bytetrack/calculators/bytetrack_calculator.cc @@ -0,0 +1,424 @@ +#include "bytetrack_calculator.h" +#include "src/bytetrack/calculators/bytetrack_calculator.pb.h" + +#include +#include + +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "src/bytetrack/calculators/strack.h" +#include "src/bytetrack/calculators/kalman_filter.h" +#include "src/bytetrack/calculators/matching_utils.h" + +// node { +// calculator: "ByteTrackCalculator" +// input_stream: "DETECTIONS:detections_with_id" +// output_stream: "DETECTIONS:tracked_detections" +// options: { +// [mediapipe.ByteTrackCalculatorOptions.ext] { +// track_high_threshold:0.5 +// track_low_threshold:0.1 +// new_track_threshold:0.25 +// matching_threshold: 0.75 +// track_buffer: 30 +// fuse_score: true +// } +// } +// } + +namespace mediapipe { +REGISTER_CALCULATOR(ByteTrackCalculator); + +absl::Status ByteTrackCalculator::GetContract(CalculatorContract* cc) { + cc->Inputs().Get("DETECTIONS", 0).Set>(); + cc->Outputs().Get("DETECTIONS", 0).Set>(); + return absl::OkStatus(); +} + +absl::Status ByteTrackCalculator::Open(CalculatorContext* cc) { + options_ = cc->Options(); + match_thresh_ = options_.matching_threshold(); + track_buffer_ = options_.track_buffer(); + track_high_thresh_ = options_.track_high_threshold(); + track_low_thresh_ = options_.track_low_threshold(); + new_track_thresh_ = options_.new_track_threshold(); + fuse_score_ = options_.fuse_score(); + return absl::OkStatus(); +} + +absl::Status ByteTrackCalculator::Process(CalculatorContext* cc) { + if (cc->Inputs().Get("DETECTIONS", 0).IsEmpty()) { + return absl::OkStatus(); // nothing to do + } + int64_t current_ts = cc->InputTimestamp().Microseconds(); + if (last_timestamp_ > 0 && frame_id_ > 1) { + float dt_sec = (current_ts - last_timestamp_) / 1e6f; + float instant_fps = 1.0f / dt_sec; + // Smooth it with a running average to avoid jitter + estimated_fps_ = 0.9f * estimated_fps_ + 0.1f * instant_fps; + max_time_lost_ = static_cast(estimated_fps_ / 30.0f * track_buffer_); + LOG(INFO) << "MAX TIME LOST: " << max_time_lost_; + } + last_timestamp_ = current_ts; + frame_id_++; + LOG(INFO) << "Frame " << frame_id_ + << " tracked_=" << tracked_stracks_.size() + << " lost_=" << lost_stracks_.size(); + + LOG(INFO) << "ByteTrackCalculator::Process called, frame " << frame_id_; + auto high_dets = std::make_unique>(); + auto low_dets = std::make_unique>(); + + if (!cc->Inputs().Tag("DETECTIONS").IsEmpty()) { + const auto& input = cc->Inputs().Tag("DETECTIONS").Get>(); + for (const auto& detection : input) { + float score = detection.score(0); + if (score >= track_high_thresh_) + high_dets->push_back(detection); + else if (score >= track_low_thresh_) + low_dets->push_back(detection); + } + } + + std::vector activated_stracks; + std::vector refind_stracks; + std::vector lost_stracks; + std::vector removed_stracks; + + LOG(INFO) << "High dets: " << high_dets->size() + << " Low dets: " << low_dets->size(); + + // create tracks from high score detections + std::vector detections; + if (high_dets->size() > 0) { + for (auto& d : *high_dets) { + detections.emplace_back(d); + } + } + + // add newly detected tracks + std::vector unconfirmed; + std::vector tracked_stracks; + for (auto& track : tracked_stracks_) { + if (!track.is_activated()) { + unconfirmed.push_back(&track); + } else { + tracked_stracks.push_back(&track); + } + } + + /////////////////////// First association ////////////////////////////////////// + std::vector lost_ptrs; + for (auto& t : lost_stracks_) + lost_ptrs.push_back(&t); + + auto strack_pool = JointStracks(tracked_stracks, lost_ptrs); + bytetrack::STrack::MultiPredict(strack_pool); + auto dists = BuildIoUCostMatrix(strack_pool, detections); + if (fuse_score_) { + dists = FuseScore(dists, detections); + } + // printDists(dists); + // gives 3 vectors + auto [matches, u_track, u_detection] = bytetrack::LinearAssignment(dists, match_thresh_); + + LOG(INFO) << "First step association, " + << " matches size: " << matches.size() + << ", U_track size: " << u_track.size() + << ", U_detection size: " << u_detection.size(); + + for (int k = 0; k < matches.rows(); ++k) { + int itracked = matches(k, 0); + int idet = matches(k, 1); + auto* track = strack_pool[itracked]; + auto det = detections[idet]; + if (track->state() == bytetrack::BaseTrack::TrackState::TRACKED) { + track->Update(detections[idet], frame_id_); + activated_stracks.push_back(*track); + } else { + track->ReActivate(det, frame_id_, false); + refind_stracks.push_back(*track); + } + } + LOG(INFO) << " After 1st assoc: activated=" << activated_stracks.size() + << " refind=" << refind_stracks.size(); + /////////////////////// Second association /////////////////////////// + std::vector detections_second; + if (low_dets->size() > 0) { + for (const auto& d : *low_dets) { + detections_second.emplace_back(d); + } + } + + std::vector r_tracked_stracks; + for (int i : u_track) { + if (strack_pool[i]->state() == bytetrack::BaseTrack::TrackState::TRACKED) { + r_tracked_stracks.push_back(strack_pool[i]); + } + } + + dists = BuildIoUCostMatrix(r_tracked_stracks, detections_second); + if (fuse_score_) { + dists = FuseScore(dists, detections_second); + } + + auto [matches2, u_track2, u_detection_second] = bytetrack::LinearAssignment(dists, 0.5f); + + LOG(INFO) << "Second step association, " + << " matches2 size: " << matches2.size() + << ", U_track2 size: " << u_track2.size() + << ", U_detection_second size: " << u_detection_second.size(); + + for (int k = 0; k < matches2.rows(); ++k) { + int itracked = matches2(k, 0); + int idet = matches2(k, 1); + auto* track = r_tracked_stracks[itracked]; + auto det = detections_second[idet]; + if (track->state() == bytetrack::BaseTrack::TrackState::TRACKED) { + track->Update(det, frame_id_); + activated_stracks.push_back(*track); + } else { + track->ReActivate(det, frame_id_, false); + refind_stracks.push_back(*track); + } + } + + // mark lost tracks + for (int it : u_track2) { + auto* track = r_tracked_stracks[it]; + if (track->state() != bytetrack::BaseTrack::TrackState::LOST) { + track->MarkLost(); + lost_stracks.push_back(*track); + } + } + LOG(INFO) << " After 2nd assoc: lost=" << lost_stracks.size(); + /////////////////////// DEAL W UNCONFIRMED TRACKS /////////////////////////// + std::vector detections_uc; + for (int i : u_detection) { + detections_uc.push_back(detections[i]); + } + + dists = BuildIoUCostMatrix(unconfirmed, detections_uc); + if (fuse_score_) { + dists = FuseScore(dists, detections_uc); + } + + auto [matches3, u_unconfirmed, u_detection_3] = bytetrack::LinearAssignment(dists, 0.7f); + + for (int k = 0; k < matches3.rows(); ++k) { + int itracked = matches3(k, 0); + int idet = matches3(k, 1); + unconfirmed[itracked]->Update(detections_uc[idet], frame_id_); + activated_stracks.push_back(*unconfirmed[itracked]); + } + + for (int it : u_unconfirmed) { + auto* track = unconfirmed[it]; + track->MarkRemoved(); + } + /////////////////////// INITIALIZE NEW TRACKS ///////////////////////////// + for (int inew : u_detection_3) { + auto track = detections_uc[inew]; + if (track.score() < new_track_thresh_) { + continue; + } + track.Activate(&kalman_filter_, frame_id_); + activated_stracks.push_back(track); + } + LOG(INFO) << " After unconfirmed+new: activated=" << activated_stracks.size(); + /////////////////////// UPDATE STATE ///////////////////////////// + for (auto& track : lost_stracks_) { + LOG(INFO) << "Time diff update state " << frame_id_ - track.frame_id() << ","; + if (frame_id_ - track.frame_id() > max_time_lost_) { + track.MarkRemoved(); + removed_stracks.push_back(track); + } + } + + LOG(INFO) << "actiavted_stracks size: " << activated_stracks.size() + << ",refind_stracks size: " << refind_stracks.size() + << ",lost_stracks size: " << lost_stracks.size() + << ",removed_stracks size: " << removed_stracks.size(); + + // Filter tracked_stracks_ to only TRACKED state, then join activated + refind + // (mirrors: self.tracked_stracks = [t for t in self.tracked_stracks if t.state == Tracked]) + { + std::vector only_tracked; + for (auto& t : tracked_stracks_) { + if (t.state() == bytetrack::BaseTrack::TrackState::TRACKED) + only_tracked.push_back(t); + } + tracked_stracks_ = only_tracked; + } + + // joint_stracks(tracked_stracks_, activated_stracks) + { + std::vector cur_ptrs, act_ptrs, ref_ptrs; + for (auto& t : tracked_stracks_) + cur_ptrs.push_back(&t); + for (auto& t : activated_stracks) + act_ptrs.push_back(&t); + for (auto& t : refind_stracks) + ref_ptrs.push_back(&t); + + auto joined = JointStracks(cur_ptrs, act_ptrs); + // joint_stracks(tracked_stracks_, refind_stracks) + joined = JointStracks(joined, ref_ptrs); + + std::vector joined_tracks; + joined_tracks.reserve(joined.size()); + for (auto* t : joined) + joined_tracks.push_back(*t); + tracked_stracks_ = std::move(joined_tracks); + } + + // sub_stracks(lost_stracks_, tracked_stracks_) then extend with lost_stracks (local) + { + std::vector lost_ptrs2, new_tracked_ptrs, removed_ptrs, local_lost_ptrs; + for (auto& t : lost_stracks_) + lost_ptrs2.push_back(&t); + for (auto& t : tracked_stracks_) + new_tracked_ptrs.push_back(&t); // fresh pointers! + for (auto& t : removed_stracks_) + removed_ptrs.push_back(&t); + for (auto& t : lost_stracks) + local_lost_ptrs.push_back(&t); + + auto new_lost = SubStracks(lost_ptrs2, new_tracked_ptrs); + new_lost = JointStracks(new_lost, local_lost_ptrs); // extend + new_lost = SubStracks(new_lost, removed_ptrs); + + // lost_stracks_.clear(); + // for (auto* t : new_lost) lost_stracks_.push_back(*t); + std::vector l_stracks; + l_stracks.reserve(new_lost.size()); + for (auto* t : new_lost) + l_stracks.push_back(*t); + lost_stracks_ = std::move(l_stracks); + } + + // extend removed_stracks_ + for (auto& t : removed_stracks) + removed_stracks_.push_back(t); + + // remove duplicates + { + std::vector tracked_ptrs2, lost_ptrs3; + for (auto& t : tracked_stracks_) + tracked_ptrs2.push_back(&t); + for (auto& t : lost_stracks_) + lost_ptrs3.push_back(&t); + auto [dedup_tracked, dedup_lost] = RemoveDuplicateStracks(tracked_ptrs2, lost_ptrs3); + std::vector new_tracked, new_lost; + new_tracked.reserve(dedup_tracked.size()); + new_lost.reserve(dedup_lost.size()); + for (auto* t : dedup_tracked) + new_tracked.push_back(*t); + for (auto* t : dedup_lost) + new_lost.push_back(*t); + tracked_stracks_ = std::move(new_tracked); + lost_stracks_ = std::move(new_lost); + } + + LOG(INFO) << "After update state"; + LOG(INFO) << " End of frame: tracked=" << tracked_stracks_.size() + << " lost=" << lost_stracks_.size(); + + ////////////////////////////// OUTPUT /////////////////////////////////// + auto output = std::make_unique>(); + for (const auto& t : tracked_stracks_) { + if (!t.is_activated()) + continue; + Detection d; + d.set_detection_id(t.track_id()); + d.add_label(t.label()); + d.add_score(t.score()); + auto* loc = d.mutable_location_data(); + loc->set_format(LocationData::RELATIVE_BOUNDING_BOX); + auto* rb = loc->mutable_relative_bounding_box(); + Eigen::Vector4f box = t.tlwh(); + rb->set_xmin(box(0)); + rb->set_ymin(box(1)); + rb->set_width(box(2)); + rb->set_height(box(3)); + output->push_back(d); + } + LOG(INFO) << "After building detections"; + LOG(INFO) << " Output size: " << output->size(); + // cc->Outputs().Get("DETECTIONS",0).Add( + // output.release(), cc->InputTimestamp()); + cc->Outputs().Get("DETECTIONS", 0).Add(output.release(), cc->Inputs().Get("DETECTIONS", 0).Value().Timestamp()); + return absl::OkStatus(); +} + +std::vector ByteTrackCalculator::JointStracks(std::vector& a, std::vector& b) { + std::unordered_map exists; + std::vector res; + + for (auto* t : a) { + exists[t->track_id()] = true; + res.push_back(t); + } + + for (auto* t : b) { + int tid = t->track_id(); + if (exists.find(tid) == exists.end()) { + exists[tid] = true; + res.push_back(t); + } + } + return res; +} + +std::vector ByteTrackCalculator::SubStracks(std::vector& a, std::vector& b) { + std::unordered_map exists; + std::vector res; + for (auto* t : a) { + exists[t->track_id()] = t; + } + for (auto* t : b) { + int tid = t->track_id(); + if (exists.find(tid) != exists.end()) { + exists.erase(tid); + } + } + for (auto& i : exists) { + res.push_back(i.second); + } + return res; +} + +//// WIP +std::pair, std::vector> +ByteTrackCalculator::RemoveDuplicateStracks(std::vector& a, std::vector& b) { + + auto pdist = BuildIoUCostMatrix(a, b); + + std::vector dupa, dupb; + for (int p = 0; p < (int)a.size(); ++p) { + for (int q = 0; q < (int)b.size(); ++q) { + if (pdist(p, q) < 0.15f) { // high overlap — duplicate + int timep = a[p]->frame_id() - a[p]->start_frame(); + int timeq = b[q]->frame_id() - b[q]->start_frame(); + if (timep > timeq) + dupb.push_back(q); + else + dupa.push_back(p); + } + } + } + + std::vector resa, resb; + for (int i = 0; i < (int)a.size(); ++i) + if (std::find(dupa.begin(), dupa.end(), i) == dupa.end()) + resa.push_back(a[i]); + for (int i = 0; i < (int)b.size(); ++i) + if (std::find(dupb.begin(), dupb.end(), i) == dupb.end()) + resb.push_back(b[i]); + + return {resa, resb}; +} + +} // namespace mediapipe diff --git a/src/bytetrack/calculators/bytetrack_calculator.h b/src/bytetrack/calculators/bytetrack_calculator.h new file mode 100644 index 0000000000..e253f2c846 --- /dev/null +++ b/src/bytetrack/calculators/bytetrack_calculator.h @@ -0,0 +1,53 @@ +#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ +#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ + +#include "src/bytetrack/calculators/bytetrack_calculator.pb.h" + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "src/bytetrack/calculators/strack.h" +#include "src/bytetrack/calculators/kalman_filter.h" + +namespace mediapipe { +class ByteTrackCalculator : public CalculatorBase { +public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Open(CalculatorContext* cc) override; + absl::Status Process(CalculatorContext* cc) override; + +private: + ::mediapipe::ByteTrackCalculatorOptions options_; + std::vector tracked_stracks_; + std::vector lost_stracks_; + std::vector removed_stracks_; + + int track_buffer_; + float det_thresh_; + float match_thresh_; + float track_high_thresh_; + float track_low_thresh_; + float new_track_thresh_; + bool fuse_score_; + int frame_id_ = 0; + int max_time_lost_ = 30; + int64_t last_timestamp_ = -1; + float estimated_fps_ = 30.0f; + + bytetrack::KalmanFilter kalman_filter_; + + static std::vector JointStracks( + std::vector& a, + std::vector& b); + + static std::vector SubStracks( + std::vector& a, + std::vector& b); + + static std::pair, std::vector> + RemoveDuplicateStracks( + std::vector& a, + std::vector& b); +}; +} // namespace mediapipe + +#endif \ No newline at end of file diff --git a/src/bytetrack/calculators/bytetrack_calculator.proto b/src/bytetrack/calculators/bytetrack_calculator.proto new file mode 100644 index 0000000000..7d91b63caf --- /dev/null +++ b/src/bytetrack/calculators/bytetrack_calculator.proto @@ -0,0 +1,17 @@ +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message ByteTrackCalculatorOptions { + extend mediapipe.CalculatorOptions { + optional ByteTrackCalculatorOptions ext = 247258241; + } + optional float matching_threshold = 2 [default = 0.8]; + optional int32 track_buffer = 3 [default = 30]; + optional float track_high_threshold = 4 [default = 0.25]; + optional float track_low_threshold = 5 [default = 0.1]; + optional float new_track_threshold = 6 [default = 0.25]; + optional bool fuse_score = 7 [default = true]; +} diff --git a/src/bytetrack/calculators/kalman_filter.cc b/src/bytetrack/calculators/kalman_filter.cc new file mode 100644 index 0000000000..dfa9259f33 --- /dev/null +++ b/src/bytetrack/calculators/kalman_filter.cc @@ -0,0 +1,107 @@ +#include "src/bytetrack/calculators/kalman_matrices.h" + +#include + +#include + +#include "mediapipe/framework/formats/detection.pb.h" +#include "src/bytetrack/calculators/kalman_filter.h" + +namespace mediapipe { +namespace bytetrack { + +KalmanFilter::KalmanFilter() : + motion_mat_(MakeMotionMatrix()), + update_mat_(MakeUpdateMatrix()), + kStdWeightPos_(kStdWeightPos), + kStdWeightVel_(kStdWeightVel) {} + +std::pair KalmanFilter::Initiate(const Eigen::Vector4f detection) { + Mean mean; + mean << detection(0), detection(1), detection(2), detection(3), 0.f, 0.f, 0.f, 0.f; + + Eigen::Matrix std_vals; + std_vals << 2.f * kStdWeightPos_ * detection(3), + 2.f * kStdWeightPos_ * detection(3), + 1e-2f, + 2.f * kStdWeightPos_ * detection(3), + 10.f * kStdWeightVel_ * detection(3), + 10.f * kStdWeightVel_ * detection(3), + 1e-5f, + 10.f * kStdWeightVel_ * detection(3); + + Cov cov = std_vals.array().square().matrix().asDiagonal(); + return {mean, cov}; +} + +std::pair KalmanFilter::Predict(const Mean& mean, const Cov& cov) const { + const float h = mean(3); + Eigen::Matrix std_pv; + std_pv << kStdWeightPos_ * h, kStdWeightPos_ * h, 1e-2f, kStdWeightPos_ * h, + kStdWeightVel_ * h, kStdWeightVel_ * h, 1e-5f, kStdWeightVel_ * h; + const Cov motion_cov = std_pv.array().square().matrix().asDiagonal(); + const Mean est_mean = mean * motion_mat_.transpose(); + const Cov est_cov = motion_mat_ * cov * motion_mat_.transpose() + motion_cov; + + return {est_mean, est_cov}; +} + +std::pair KalmanFilter::Update(const Mean& mean, const Cov& cov, const Eigen::Vector4f xyah) const { + const float x_c = xyah(0); + const float y_c = xyah(1); + const float ar = xyah(2); + const float h = xyah(3); + + Eigen::Matrix measurement; + measurement << x_c, y_c, ar, h; + // Innovation covariance in measurement space + Eigen::Matrix noise_std; + noise_std << kStdWeightPos_ * mean(3), + kStdWeightPos_ * mean(3), + 1e-1f, + kStdWeightPos_ * mean(3); + const Eigen::Matrix innov_cov = noise_std.array().square().matrix().asDiagonal(); + + const Eigen::Matrix proj_mean = mean * update_mat_.transpose(); + const Eigen::Matrix proj_cov = update_mat_ * cov * update_mat_.transpose() + innov_cov; + + // Kalman gain via Cholesky solve: K = (P H^T) (H P H^T + R)^{-1} + const Eigen::Matrix PHt = cov * update_mat_.transpose(); + const Eigen::Matrix K = proj_cov.llt().solve(PHt.transpose()).transpose(); + + const Mean updated_mean = mean + (measurement - proj_mean) * K.transpose(); + const Cov updated_cov = cov - K * proj_cov * K.transpose(); + + return {updated_mean, updated_cov}; +} + +std::pair KalmanFilter::MultiPredict(const MeanMatrix& means, const CovMatrix& covs) const { + const int N = means.rows(); + + // Build Nx8 std matrix — each row is the std devs for one track + MeanMatrix std_mat(N, 8); + for (int i = 0; i < N; ++i) { + const float h = means(i, 3); + std_mat.row(i) << kStdWeightPos_ * h, kStdWeightPos_ * h, 1e-2f, kStdWeightPos_ * h, + kStdWeightVel_ * h, kStdWeightVel_ * h, 1e-5f, kStdWeightVel_ * h; + } + + // Predicted means: (N,8) @ F^T + MeanMatrix pred_means = means * motion_mat_.transpose(); + + // Predicted covariances per track + CovMatrix pred_covs(N); + for (int i = 0; i < N; ++i) { + const Cov motion_cov = std_mat.row(i) + .array() + .square() + .matrix() + .asDiagonal(); + pred_covs[i] = motion_mat_ * covs[i] * motion_mat_.transpose() + motion_cov; + } + + return {pred_means, pred_covs}; +} + +} // namespace bytetrack +} // namespace mediapipe diff --git a/src/bytetrack/calculators/kalman_filter.h b/src/bytetrack/calculators/kalman_filter.h new file mode 100644 index 0000000000..4d4fc4ee56 --- /dev/null +++ b/src/bytetrack/calculators/kalman_filter.h @@ -0,0 +1,34 @@ +#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ +#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ +#include + +#include + +#include "mediapipe/framework/formats/detection.pb.h" + +namespace mediapipe { +namespace bytetrack { + +using Mean = Eigen::Matrix; +using Cov = Eigen::Matrix; +using MeanMatrix = Eigen::Matrix; +using CovMatrix = std::vector; +class KalmanFilter { +public: + KalmanFilter(); + std::pair Initiate(const Eigen::Vector4f detection); + std::pair Predict(const Mean& mean, const Cov& cov) const; + std::pair Update(const Mean& mean, const Cov& cov, const Eigen::Vector4f xyah) const; + std::pair MultiPredict(const MeanMatrix& means, const CovMatrix& covs) const; + +private: + Eigen::Matrix motion_mat_; + Eigen::Matrix update_mat_; + float kStdWeightPos_; + float kStdWeightVel_; +}; + +} // namespace bytetrack +} // namespace mediapipe + +#endif \ No newline at end of file diff --git a/src/bytetrack/calculators/kalman_matrices.h b/src/bytetrack/calculators/kalman_matrices.h new file mode 100644 index 0000000000..163e597182 --- /dev/null +++ b/src/bytetrack/calculators/kalman_matrices.h @@ -0,0 +1,28 @@ +// kalman_matrices.h +#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ +#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ + +#include + +namespace mediapipe { +namespace bytetrack { + +static constexpr float kStdWeightPos = 1.0f / 20.0f; +static constexpr float kStdWeightVel = 1.0f / 160.0f; + +inline Eigen::Matrix MakeMotionMatrix() { + Eigen::Matrix F = Eigen::Matrix::Identity(); + F.block<4, 4>(0, 4) = Eigen::Matrix4f::Identity(); + return F; +} + +inline Eigen::Matrix MakeUpdateMatrix() { + Eigen::Matrix H = Eigen::Matrix::Zero(); + H.block<4, 4>(0, 0) = Eigen::Matrix4f::Identity(); + return H; +} + +} // namespace bytetrack +} // namespace mediapipe + +#endif \ No newline at end of file diff --git a/src/bytetrack/calculators/matching_utils.h b/src/bytetrack/calculators/matching_utils.h new file mode 100644 index 0000000000..da76f26fbb --- /dev/null +++ b/src/bytetrack/calculators/matching_utils.h @@ -0,0 +1,335 @@ +#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ +#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ + +#include +#include +#include + +#include + +#include "src/bytetrack/calculators/strack.h" + +namespace mediapipe { +namespace bytetrack { + +struct AssignmentResult { + Eigen::MatrixXi matches; // shape (K, 2) — col 0 = track idx, col 1 = box idx + Eigen::VectorXi unmatched_tracks; // shape (P,) + Eigen::VectorXi unmatched_boxes; // shape (Q,) +}; + +// IoU between two [top, left, bottom, right] boxes +inline float ComputeIoU(const Eigen::Vector4f& a, + const Eigen::Vector4f& b) { + float inter_x1 = std::max(a[0], b[0]); // left + float inter_y1 = std::max(a[1], b[1]); // top + float inter_x2 = std::min(a[2], b[2]); // right + float inter_y2 = std::min(a[3], b[3]); // bottom + + float inter_w = std::max(0.f, inter_x2 - inter_x1); + float inter_h = std::max(0.f, inter_y2 - inter_y1); + float inter_area = inter_w * inter_h; + if (inter_area == 0.f) + return 0.f; + + float area_a = (a[2] - a[0]) * (a[3] - a[1]); // w * h + float area_b = (b[2] - b[0]) * (b[3] - b[1]); + return inter_area / (area_a + area_b - inter_area); +} + +inline Eigen::MatrixXf BuildIoUCostMatrix( + const std::vector& tracks, + const std::vector& detections) { + int N = tracks.size(); + int M = detections.size(); + + Eigen::MatrixXf cost(N, M); + + for (int i = 0; i < N; ++i) { + auto tb = tracks[i]->tlbr(); + for (int j = 0; j < M; ++j) { + auto bb = detections[j].tlbr(); + cost(i, j) = 1.f - ComputeIoU(tb, bb); + } + } + + return cost; +} + +inline Eigen::MatrixXf FuseScore( + const Eigen::MatrixXf cost_matrix, + const std::vector& detections) { + if (cost_matrix.size() == 0) { + return cost_matrix; + } + Eigen::MatrixXf iou_sim = 1.0f - cost_matrix.array(); + + Eigen::RowVectorXf det_scores(detections.size()); + for (int i = 0; i < (int)detections.size(); i++) { + det_scores(i) = detections[i].score(); + } + + Eigen::MatrixXf det_scores_mat = det_scores.replicate(cost_matrix.rows(), 1); + Eigen::MatrixXf fuse_sim = iou_sim.array() * det_scores_mat.array(); + Eigen::MatrixXf fuse_cost = 1.0f - fuse_sim.array(); + + return fuse_cost; +} + +inline Eigen::MatrixXf BuildIoUCostMatrix( + const std::vector& a, + const std::vector& b) { + int N = a.size(); + int M = b.size(); + + Eigen::MatrixXf cost(N, M); + + for (int i = 0; i < N; ++i) { + auto ta = a[i]->tlbr(); + for (int j = 0; j < M; ++j) { + auto tb = b[j]->tlbr(); + cost(i, j) = 1.f - ComputeIoU(ta, tb); + } + } + + return cost; +} + +// inline AssignmentResult LinearAssignment(const Eigen::MatrixXf& cost, float thresh){ +// int N = (int)cost.rows(); +// int M = (int)cost.cols(); + +// // Collect candidates below threshold +// std::vector> entries; +// entries.reserve(N * M); +// for (int i = 0; i < N; ++i) +// for (int j = 0; j < M; ++j) +// if (cost(i,j) <= thresh) +// entries.emplace_back(cost(i,j), i, j); + +// std::sort(entries.begin(), entries.end()); + +// // Greedy assignment +// std::vector track_used(N, false); +// std::vector box_used(M, false); +// std::vector> matched; +// matched.reserve(std::min(N, M)); + +// for (auto& [c, i, j] : entries) { +// if (!track_used[i] && !box_used[j]) { +// matched.emplace_back(i, j); +// track_used[i] = true; +// box_used[j] = true; +// } +// } + +// // Pack into Eigen outputs +// int K = (int)matched.size(); +// Eigen::MatrixXi matches(K, 2); // (K,2) — mirrors np.empty((0,2)) when K=0 +// for (int k = 0; k < K; ++k) { +// matches(k, 0) = matched[k].first; +// matches(k, 1) = matched[k].second; +// } + +// // Count unmatched first, then fill — avoids push_back on Eigen vectors +// int n_ut = (int)std::count(track_used.begin(), track_used.end(), false); +// int n_ub = (int)std::count(box_used.begin(), box_used.end(), false); + +// Eigen::VectorXi unmatched_tracks(n_ut); +// Eigen::VectorXi unmatched_boxes(n_ub); + +// for (int i = 0, k = 0; i < N; ++i) +// if (!track_used[i]) unmatched_tracks(k++) = i; +// for (int j = 0, k = 0; j < M; ++j) +// if (!box_used[j]) unmatched_boxes(k++) = j; + +// return {matches, unmatched_tracks, unmatched_boxes}; +// } + +inline AssignmentResult LinearAssignment(const Eigen::MatrixXf& cost, float thresh) { + int N = (int)cost.rows(); + int M = (int)cost.cols(); + + // Empty matrix early exit — mirrors Python: if cost_matrix.size == 0 + if (N == 0 || M == 0) { + Eigen::MatrixXi matches(0, 2); + Eigen::VectorXi u_tracks(N), u_boxes(M); + for (int i = 0; i < N; ++i) + u_tracks(i) = i; + for (int j = 0; j < M; ++j) + u_boxes(j) = j; + return {matches, u_tracks, u_boxes}; + } + + // Pad to square S x S — mirrors lap.lapjv extend_cost=True + int S = std::max(N, M); + const float INF = 1e9f; + + Eigen::MatrixXf cost_sq = Eigen::MatrixXf::Constant(S, S, INF); + for (int i = 0; i < N; ++i) + for (int j = 0; j < M; ++j) + cost_sq(i, j) = cost(i, j); + + // Dual variables and assignment vectors + std::vector u(S, 0.f), v(S, 0.f); + std::vector row2col(S, -1), col2row(S, -1); + + // Phase 1: Column reduction — init v[j] to column minimum + for (int j = 0; j < S; ++j) { + int best_i = 0; + float best_v = cost_sq(0, j); + for (int i = 1; i < S; ++i) { + if (cost_sq(i, j) < best_v) { + best_v = cost_sq(i, j); + best_i = i; + } + } + v[j] = best_v; + if (row2col[best_i] == -1) { + row2col[best_i] = j; + col2row[j] = best_i; + } + } + + // Phase 2: Augmenting row reduction (2 passes) + for (int pass = 0; pass < 2; ++pass) { + for (int i = 0; i < S; ++i) { + if (row2col[i] != -1) + continue; + int j1 = -1, j2 = -1; + float u1 = INF, u2 = INF; + for (int j = 0; j < S; ++j) { + float h = cost_sq(i, j) - v[j]; + if (h < u2) { + if (h < u1) { + u2 = u1; + j2 = j1; + u1 = h; + j1 = j; + } else { + u2 = h; + j2 = j; + } + } + } + u[i] = u1; + if (col2row[j1] == -1) { + row2col[i] = j1; + col2row[j1] = i; + } else { + v[j1] -= (u2 - u1); + } + } + } + + // Phase 3: Augmentation via shortest path (Dijkstra with potentials) + std::vector dist(S); + std::vector pred(S, -1); + std::vector visited(S, false); + + for (int i_start = 0; i_start < S; ++i_start) { + if (row2col[i_start] != -1) + continue; + + std::fill(dist.begin(), dist.end(), INF); + std::fill(pred.begin(), pred.end(), -1); + std::fill(visited.begin(), visited.end(), false); + + for (int j = 0; j < S; ++j) + dist[j] = cost_sq(i_start, j) - u[i_start] - v[j]; + + int j_end = -1; + float d_min = INF; + + for (int iter = 0; iter < S; ++iter) { + // Pick unvisited col with smallest dist + int j_min = -1; + d_min = INF; + for (int j = 0; j < S; ++j) + if (!visited[j] && dist[j] < d_min) { + d_min = dist[j]; + j_min = j; + } + + if (j_min == -1) + break; + visited[j_min] = true; + + if (col2row[j_min] == -1) { + j_end = j_min; + break; + } + + // Relax edges through the row that owns j_min + int i_next = col2row[j_min]; + u[i_next] = cost_sq(i_next, j_min) - v[j_min] - d_min; // update dual + for (int j = 0; j < S; ++j) { + if (visited[j]) + continue; + float nd = d_min + cost_sq(i_next, j) - u[i_next] - v[j]; + if (nd < dist[j]) { + dist[j] = nd; + pred[j] = j_min; + } + } + } + + // Update col duals along the path + for (int j = 0; j < S; ++j) + if (visited[j]) + v[j] += dist[j] - d_min; + u[i_start] += d_min; + + // Augment: flip assignments along path back to i_start + int j_cur = j_end; + while (j_cur != -1) { + int i_cur = (pred[j_cur] == -1) ? i_start : col2row[pred[j_cur]]; + col2row[j_cur] = i_cur; + row2col[i_cur] = j_cur; + j_cur = pred[j_cur]; + } + } + + // Extract matches — mirrors: for ix, mx in enumerate(x): if mx >= 0 + // Apply cost_limit=thresh filter here + std::vector track_used(N, false); + std::vector box_used(M, false); + std::vector> matched; + + for (int i = 0; i < N; ++i) { + int j = row2col[i]; + if (j < M && cost(i, j) <= thresh) { + matched.emplace_back(i, j); + track_used[i] = true; + box_used[j] = true; + } + } + + // Pack into AssignmentResult + int K = (int)matched.size(); + Eigen::MatrixXi matches(K, 2); + for (int k = 0; k < K; ++k) { + matches(k, 0) = matched[k].first; + matches(k, 1) = matched[k].second; + } + + int n_ut = (int)std::count(track_used.begin(), track_used.end(), false); + int n_ub = (int)std::count(box_used.begin(), box_used.end(), false); + + Eigen::VectorXi unmatched_tracks(n_ut); + Eigen::VectorXi unmatched_boxes(n_ub); + + for (int i = 0, k = 0; i < N; ++i) + if (!track_used[i]) + unmatched_tracks(k++) = i; + for (int j = 0, k = 0; j < M; ++j) + if (!box_used[j]) + unmatched_boxes(k++) = j; + + return {matches, unmatched_tracks, unmatched_boxes}; +} + +} // namespace bytetrack +} // namespace mediapipe + +#endif \ No newline at end of file diff --git a/src/bytetrack/calculators/render_data_passthrough_calculator.cc b/src/bytetrack/calculators/render_data_passthrough_calculator.cc new file mode 100644 index 0000000000..db21e318df --- /dev/null +++ b/src/bytetrack/calculators/render_data_passthrough_calculator.cc @@ -0,0 +1,31 @@ +#include "mediapipe/framework/calculator_framework.h" + +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe { + +class PassThroughRenderDataCalculator : public CalculatorBase { +public: + static absl::Status GetContract(CalculatorContract* cc) { + cc->Inputs().Tag("RENDER_DATA").Set(); + cc->Outputs().Tag("RENDER_DATA").Set(); + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) override { + const auto& render_data = + cc->Inputs().Tag("RENDER_DATA").Get(); + + LOG(INFO) << "RenderData: num objects = " + << render_data.render_annotations_size(); + + // Forward unchanged + cc->Outputs().Tag("RENDER_DATA").AddPacket(MakePacket(render_data).At(cc->InputTimestamp())); + + return absl::OkStatus(); + } +}; + +REGISTER_CALCULATOR(PassThroughRenderDataCalculator); + +} // namespace mediapipe \ No newline at end of file diff --git a/src/bytetrack/calculators/strack.cc b/src/bytetrack/calculators/strack.cc new file mode 100644 index 0000000000..f62524ed92 --- /dev/null +++ b/src/bytetrack/calculators/strack.cc @@ -0,0 +1,150 @@ +#include "src/bytetrack/calculators/basetrack.h" + +#include + +#include + +#include "mediapipe/framework/formats/detection.pb.h" +#include "src/bytetrack/calculators/kalman_filter.h" +#include "src/bytetrack/calculators/kalman_matrices.h" +#include "src/bytetrack/calculators/strack.h" + +namespace mediapipe { +namespace bytetrack { + +KalmanFilter STrack::shared_kalman; + +STrack::STrack(const Detection& det) { + const auto& loc = det.location_data(); + score_ = det.score_size() > 0 ? det.score(0) : 0.0f; + label_ = det.label_size() > 0 ? det.label(0) : ""; + if (loc.format() == LocationData::RELATIVE_BOUNDING_BOX) { + const auto& rb = loc.relative_bounding_box(); + tlwh_ << rb.xmin(), rb.ymin(), rb.width(), rb.height(); + } else if (loc.format() == LocationData::BOUNDING_BOX) { + const auto& b = loc.bounding_box(); + tlwh_ << b.xmin(), b.ymin(), b.width(), b.height(); + } else { + tlwh_.setZero(); + } +} + +void STrack::Predict() { + Mean mean_state = mean(); + if (state() != TrackState::TRACKED) { + mean_state(7) = 0.f; + } + auto [new_mean, new_cov] = kf_->Predict(mean_state, cov()); + mean_ = new_mean; + cov_ = new_cov; +} + +void STrack::Activate(KalmanFilter* kalman_filter, int frame_id) { + kf_ = kalman_filter; + track_id_ = next_id(); + auto [new_mean, new_cov] = kf_->Initiate(TlwhToXyah(tlwh_)); + mean_ = new_mean; + cov_ = new_cov; + + tracklet_len_ = 0; + state_ = TrackState::TRACKED; + if (frame_id == 1) + is_activated_ = true; + frame_id_ = frame_id; + start_frame_ = frame_id; +} + +void STrack::ReActivate(const STrack& new_track, int frame_id, bool new_id) { + Eigen::Vector4f new_tlwh = new_track.tlwh_; + auto [new_mean, new_cov] = kf_->Update(mean_, cov_, TlwhToXyah(new_tlwh)); + mean_ = new_mean; + cov_ = new_cov; + tracklet_len_ = 0; + state_ = TrackState::TRACKED; + is_activated_ = true; + + frame_id_ = frame_id; + if (new_id) + track_id_ = next_id(); + score_ = new_track.score(); +} + +void STrack::Update(const STrack& new_track, int frame_id) { + frame_id_ = frame_id; + tracklet_len_ += 1; + + Eigen::Vector4f new_tlwh = new_track.tlwh_; + auto [new_mean, new_cov] = kf_->Update(mean_, cov_, TlwhToXyah(new_tlwh)); + mean_ = new_mean; + cov_ = new_cov; + + state_ = TrackState::TRACKED; + is_activated_ = true; + score_ = new_track.score(); +} + +void STrack::MultiPredict(std::vector& tracks) { + long int n = tracks.size(); + if (n > 0) { + MeanMatrix multi_mean(n, 8); + for (int i = 0; i < n; i++) { + multi_mean.row(i) = tracks[i]->mean_; + } + CovMatrix multi_cov(n); + for (int i = 0; i < n; i++) { + multi_cov[i] = tracks[i]->cov_; + } + for (int i = 0; i < n; i++) { + if (tracks[i]->state_ != TrackState::TRACKED) { + multi_mean(i, 7) = 0.f; + } + } + auto [updated_means, updated_covs] = shared_kalman.MultiPredict(multi_mean, multi_cov); + for (int i = 0; i < n; i++) { + tracks[i]->mean_ = updated_means.row(i); + tracks[i]->cov_ = updated_covs[i]; + } + } +} + +Eigen::Vector4f STrack::TlwhToXyah(const Eigen::Vector4f& tlwh) { + Eigen::Vector4f xyah; + float x = tlwh(0); + float y = tlwh(1); + float w = tlwh(2); + float h = tlwh(3); + + xyah(0) = x + w / 2.0f; + xyah(1) = y + h / 2.0f; + xyah(2) = w / h; + xyah(3) = h; + + return xyah; +} + +Eigen::Vector4f STrack::tlwh() const { + // Before activation — return the raw detection box + if (kf_ == nullptr) + return tlwh_; + + // After activation — reconstruct from Kalman mean + // mean_ = [cx, cy, ar, h, vx, vy, var, vh] + Eigen::Vector4f ret; + ret(0) = mean_(0); + ret(1) = mean_(1); + ret(2) = mean_(2) * mean_(3); + ret(3) = mean_(3); + ret(0) -= ret(2) / 2.f; + ret(1) -= ret(3) / 2.f; + return ret; +} + +Eigen::Vector4f STrack::tlbr() const { + Eigen::Vector4f ret = tlwh(); + ret(2) += ret(0); + ret(3) += ret(1); + return ret; +} + +} // namespace bytetrack +} // namespace mediapipe \ No newline at end of file diff --git a/src/bytetrack/calculators/strack.h b/src/bytetrack/calculators/strack.h new file mode 100644 index 0000000000..402f43cb69 --- /dev/null +++ b/src/bytetrack/calculators/strack.h @@ -0,0 +1,60 @@ +#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_STRACK_H_ +#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_STRACK_H_ + +#include +#include + +#include + +#include "mediapipe/framework/formats/detection.pb.h" +#include "src/bytetrack/calculators/kalman_filter.h" +#include "src/bytetrack/calculators/basetrack.h" + +namespace mediapipe { +namespace bytetrack { + +using Mean = Eigen::Matrix; +using Cov = Eigen::Matrix; +using MeanMatrix = Eigen::Matrix; +using CovMatrix = std::vector; + +class STrack : public BaseTrack { +public: + static KalmanFilter shared_kalman; + + explicit STrack(const Detection& det); + + void Activate(KalmanFilter* kalman_filter, int frame_id); + void ReActivate(const STrack& new_track, int frame_id, bool new_id = false); + void Update(const STrack& new_track, int frame_id); + + void Predict(); + static void MultiPredict(std::vector& tracks); + + Eigen::Vector4f tlwh() const; + Eigen::Vector4f tlbr() const; + // static Eigen::Vector4f TlbrToTlwh(const Eigen::Vector4f& tlbr); + // static Eigen::Vector4f TlwhToTlbr(const Eigen::Vector4f& tlwh); + static Eigen::Vector4f TlwhToXyah(const Eigen::Vector4f& tlwh); + + // KalmanState ToProto() const; + int tracklet_len() const { return tracklet_len_; } + std::string label() const { return label_; } + const Mean& mean() const { return mean_; } + const Cov& cov() const { return cov_; } + +private: + // STrack-only — Kalman state and detection origin + // score_, is_activated_, tracklet_len_ are inherited from BaseTrack — do NOT redeclare + Eigen::Vector4f tlwh_; + std::string label_; + KalmanFilter* kf_ = nullptr; + Mean mean_ = Mean::Zero(); + Cov cov_ = Cov::Zero(); + int tracklet_len_ = 0; +}; + +} // namespace bytetrack +} // namespace mediapipe + +#endif \ No newline at end of file diff --git a/src/bytetrack/utils/BUILD b/src/bytetrack/utils/BUILD new file mode 100644 index 0000000000..564ff9db2f --- /dev/null +++ b/src/bytetrack/utils/BUILD @@ -0,0 +1,29 @@ +load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "detection_color_by_id_calculator", + srcs = ["detection_color_by_id_calculator.cc"], + deps=[ + ":detection_color_by_id_calculator_cc_proto", + "@mediapipe//mediapipe/util:render_data_cc_proto", + "@mediapipe//mediapipe/framework:calculator_framework", + "@mediapipe//mediapipe/framework/formats:detection_cc_proto", + "@mediapipe//mediapipe/framework/formats:location_data_cc_proto", + "@mediapipe//mediapipe/framework/port:ret_check", + "@mediapipe//mediapipe/util:color_cc_proto", + ], + alwayslink = 1, +) + +mediapipe_proto_library( + name = "detection_color_by_id_calculator_proto", + srcs = ["detection_color_by_id_calculator.proto"], + deps = [ + "@mediapipe//mediapipe/framework:calculator_options_proto", + "@mediapipe//mediapipe/framework:calculator_proto", + ], +) \ No newline at end of file diff --git a/src/bytetrack/utils/detection_color_by_id_calculator.cc b/src/bytetrack/utils/detection_color_by_id_calculator.cc new file mode 100644 index 0000000000..d60e902719 --- /dev/null +++ b/src/bytetrack/utils/detection_color_by_id_calculator.cc @@ -0,0 +1,139 @@ +//***************************************************************************** +// Copyright (c) 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "src/bytetrack/utils/detection_color_by_id_calculator.pb.h" + +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/util/render_data.pb.h" +#include "mediapipe/util/color.pb.h" + +namespace mediapipe { + +class DetectionColorByIdCalculator : public CalculatorBase { +public: + static absl::Status GetContract(CalculatorContract* cc) { + cc->Inputs().Tag("DETECTIONS").Set>(); + cc->Outputs().Tag("RENDER_DATA").Set(); + return absl::OkStatus(); + } + absl::Status Open(CalculatorContext* cc) override { + const auto& options = + cc->Options(); + thickness_ = options.has_thickness() ? options.thickness() : 4.0f; + saturation_ = options.has_saturation() ? options.saturation() : 0.85f; + value_ = options.has_value() ? options.value() : 0.95f; + return absl::OkStatus(); + } + absl::Status Process(CalculatorContext* cc) override { + const auto& detections = + cc->Inputs().Tag("DETECTIONS").Get>(); + + auto render_data = std::make_unique(); + + for (const auto& det : detections) { + int id = det.detection_id(); + mediapipe::Color color = IdToColor(id); + + const auto& bbox = det.location_data().relative_bounding_box(); + + // ── 1. Bounding box ────────────────────────────────────────────────── + { + auto* a = render_data->add_render_annotations(); + *a->mutable_color() = color; + a->set_thickness(thickness_); + + auto* rect = a->mutable_rectangle(); + rect->set_left(bbox.xmin()); + rect->set_top(bbox.ymin()); + rect->set_right(bbox.xmin() + bbox.width()); + rect->set_bottom(bbox.ymin() + bbox.height()); + rect->set_normalized(true); + } + + // ── 2. Label ─────────────────────────────────────────────────────── + { + auto* a = render_data->add_render_annotations(); + *a->mutable_color() = color; // same color as box + a->set_thickness(thickness_ - 1.0f); + + auto* text = a->mutable_text(); + std::string label = "ID:" + std::to_string(id); + if (!det.label().empty()) + label += " " + det.label(0); + if (det.score_size() > 0) { + char buf[8]; + std::snprintf(buf, sizeof(buf), " %.2f", det.score(0)); + label += buf; + } + + text->set_display_text(label); + text->set_normalized(true); + text->set_left(bbox.xmin() + 0.005f); + text->set_baseline(bbox.ymin() + 0.04f); + text->set_font_height(0.035f); + } + } + + cc->Outputs().Tag("RENDER_DATA").Add(render_data.release(), cc->InputTimestamp()); + return absl::OkStatus(); + } + +private: + mediapipe::Color IdToColor(int id) { + // Golden angle ensures max visual distance between consecutive IDs + const float kGoldenAngle = 137.508f; + float hue = std::fmod(id * kGoldenAngle, 360.0f); + float chroma = value_ * saturation_; + float x = chroma * (1.0f - std::fabs(std::fmod(hue / 60.0f, 2.0f) - 1.0f)); + float m = value_ - chroma; + + float r = 0, g = 0, b = 0; + if (hue < 60) { + r = chroma; + g = x; + } else if (hue < 120) { + r = x; + g = chroma; + } else if (hue < 180) { + g = chroma; + b = x; + } else if (hue < 240) { + g = x; + b = chroma; + } else if (hue < 300) { + r = x; + b = chroma; + } else { + r = chroma; + b = x; + } + + mediapipe::Color color; + color.set_r(static_cast((r + m) * 255)); + color.set_g(static_cast((g + m) * 255)); + color.set_b(static_cast((b + m) * 255)); + return color; + } + float thickness_ = 4.0f; + float saturation_ = 0.85f; + float value_ = 0.95f; +}; + +REGISTER_CALCULATOR(DetectionColorByIdCalculator); + +} // namespace mediapipe \ No newline at end of file diff --git a/src/bytetrack/utils/detection_color_by_id_calculator.proto b/src/bytetrack/utils/detection_color_by_id_calculator.proto new file mode 100644 index 0000000000..9461e84be8 --- /dev/null +++ b/src/bytetrack/utils/detection_color_by_id_calculator.proto @@ -0,0 +1,15 @@ +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message DetectionColorByIdCalculatorOptions{ + // hue is calculated using track_id* 137.508f % 360, In order to get unique color for bbox based on id + extend CalculatorOptions { + optional DetectionColorByIdCalculatorOptions ext = 259397841; + } + optional float saturation = 1 [default = 0.85]; + optional float value = 2 [default = 0.95]; + optional float thickness = 3 [default = 5.0]; +} \ No newline at end of file diff --git a/src/yolox/ov_yolox_tensors_to_detections_calculator.cc b/src/yolox/ov_yolox_tensors_to_detections_calculator.cc index 999b501002..d627ceb9c8 100644 --- a/src/yolox/ov_yolox_tensors_to_detections_calculator.cc +++ b/src/yolox/ov_yolox_tensors_to_detections_calculator.cc @@ -35,7 +35,7 @@ class OVYoloXTensorsToDetectionsCalculator : public CalculatorBase { float confidenceThreshold_ = 0.1f; float inputSize_ = 416.0f; - public: +public: static absl::Status GetContract(CalculatorContract* cc) { RET_CHECK(!cc->Inputs().GetTags().empty()); RET_CHECK(!cc->Outputs().GetTags().empty()); diff --git a/third_party/mediapipe_calculators/BUILD b/third_party/mediapipe_calculators/BUILD index a22d1b4f2a..a4570a4daa 100644 --- a/third_party/mediapipe_calculators/BUILD +++ b/third_party/mediapipe_calculators/BUILD @@ -108,8 +108,6 @@ cc_library( "@mediapipe//mediapipe/calculators/video:box_detector_calculator", "@mediapipe//mediapipe/calculators/video:tracked_detection_manager_calculator", "@mediapipe//mediapipe/calculators/video:video_pre_stream_calculator", - #BYTETRACK CALCULATORS (GSoC'2026 - Vishwa2684) - "@mediapipe//mediapipe/graphs/bytetrack/calculators:bytetrack_calculators", ] + select({ "//conditions:default": [ "@mediapipe//mediapipe/calculators/core:packet_cloner_calculator", # TODO windows: stdc++20 required From 7524ee6ecd38159ccc5468829859ca83f1b88baf Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Thu, 13 Aug 2026 23:35:23 +0530 Subject: [PATCH 10/18] Add license headers to files --- src/bytetrack/calculators/BUILD | 4 +++- src/bytetrack/calculators/basetrack.cc | 15 +++++++++++++++ src/bytetrack/calculators/basetrack.h | 16 ++++++++++++++++ .../calculators/bytetrack_calculator.cc | 16 ++++++++++++++++ src/bytetrack/calculators/bytetrack_calculator.h | 16 ++++++++++++++++ .../calculators/bytetrack_calculator.proto | 16 ++++++++++++++++ src/bytetrack/calculators/kalman_filter.cc | 16 ++++++++++++++++ src/bytetrack/calculators/kalman_filter.h | 16 ++++++++++++++++ src/bytetrack/calculators/kalman_matrices.h | 15 +++++++++++++++ src/bytetrack/calculators/matching_utils.h | 16 ++++++++++++++++ .../render_data_passthrough_calculator.cc | 16 ++++++++++++++++ src/bytetrack/calculators/strack.cc | 16 ++++++++++++++++ src/bytetrack/calculators/strack.h | 16 ++++++++++++++++ src/bytetrack/utils/BUILD | 16 ++++++++++++++++ .../utils/detection_color_by_id_calculator.proto | 16 ++++++++++++++++ src/yolox/BUILD | 16 ++++++++++++++++ ..._yolox_tensors_to_detections_calculator.proto | 16 ++++++++++++++++ 17 files changed, 257 insertions(+), 1 deletion(-) diff --git a/src/bytetrack/calculators/BUILD b/src/bytetrack/calculators/BUILD index 6d4e43619c..b3e1641f1e 100644 --- a/src/bytetrack/calculators/BUILD +++ b/src/bytetrack/calculators/BUILD @@ -1,4 +1,5 @@ -# Copyright 2019 The MediaPipe Authors. +# +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") diff --git a/src/bytetrack/calculators/basetrack.cc b/src/bytetrack/calculators/basetrack.cc index 1929bc17f2..c096c201e1 100644 --- a/src/bytetrack/calculators/basetrack.cc +++ b/src/bytetrack/calculators/basetrack.cc @@ -1,3 +1,18 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** // basetrack.cpp #include "src/bytetrack/calculators/basetrack.h" diff --git a/src/bytetrack/calculators/basetrack.h b/src/bytetrack/calculators/basetrack.h index 439380c8aa..9ad0fbb5a1 100644 --- a/src/bytetrack/calculators/basetrack.h +++ b/src/bytetrack/calculators/basetrack.h @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BASETRACK_H_ #define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BASETRACK_H_ diff --git a/src/bytetrack/calculators/bytetrack_calculator.cc b/src/bytetrack/calculators/bytetrack_calculator.cc index bbb02a8f0c..1640bac1f3 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.cc +++ b/src/bytetrack/calculators/bytetrack_calculator.cc @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #include "bytetrack_calculator.h" #include "src/bytetrack/calculators/bytetrack_calculator.pb.h" diff --git a/src/bytetrack/calculators/bytetrack_calculator.h b/src/bytetrack/calculators/bytetrack_calculator.h index e253f2c846..20d96623df 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.h +++ b/src/bytetrack/calculators/bytetrack_calculator.h @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ #define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ diff --git a/src/bytetrack/calculators/bytetrack_calculator.proto b/src/bytetrack/calculators/bytetrack_calculator.proto index 7d91b63caf..d3e0af5603 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.proto +++ b/src/bytetrack/calculators/bytetrack_calculator.proto @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + syntax = "proto2"; package mediapipe; diff --git a/src/bytetrack/calculators/kalman_filter.cc b/src/bytetrack/calculators/kalman_filter.cc index dfa9259f33..bf83fac362 100644 --- a/src/bytetrack/calculators/kalman_filter.cc +++ b/src/bytetrack/calculators/kalman_filter.cc @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #include "src/bytetrack/calculators/kalman_matrices.h" #include diff --git a/src/bytetrack/calculators/kalman_filter.h b/src/bytetrack/calculators/kalman_filter.h index 4d4fc4ee56..93337db12a 100644 --- a/src/bytetrack/calculators/kalman_filter.h +++ b/src/bytetrack/calculators/kalman_filter.h @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ #define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ #include diff --git a/src/bytetrack/calculators/kalman_matrices.h b/src/bytetrack/calculators/kalman_matrices.h index 163e597182..886b94938e 100644 --- a/src/bytetrack/calculators/kalman_matrices.h +++ b/src/bytetrack/calculators/kalman_matrices.h @@ -1,3 +1,18 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** // kalman_matrices.h #ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ #define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ diff --git a/src/bytetrack/calculators/matching_utils.h b/src/bytetrack/calculators/matching_utils.h index da76f26fbb..f10a3ba732 100644 --- a/src/bytetrack/calculators/matching_utils.h +++ b/src/bytetrack/calculators/matching_utils.h @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ #define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ diff --git a/src/bytetrack/calculators/render_data_passthrough_calculator.cc b/src/bytetrack/calculators/render_data_passthrough_calculator.cc index db21e318df..a69aa87587 100644 --- a/src/bytetrack/calculators/render_data_passthrough_calculator.cc +++ b/src/bytetrack/calculators/render_data_passthrough_calculator.cc @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/util/render_data.pb.h" diff --git a/src/bytetrack/calculators/strack.cc b/src/bytetrack/calculators/strack.cc index f62524ed92..1169f8ac56 100644 --- a/src/bytetrack/calculators/strack.cc +++ b/src/bytetrack/calculators/strack.cc @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #include "src/bytetrack/calculators/basetrack.h" #include diff --git a/src/bytetrack/calculators/strack.h b/src/bytetrack/calculators/strack.h index 402f43cb69..f16cce9f0d 100644 --- a/src/bytetrack/calculators/strack.h +++ b/src/bytetrack/calculators/strack.h @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + #ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_STRACK_H_ #define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_STRACK_H_ diff --git a/src/bytetrack/utils/BUILD b/src/bytetrack/utils/BUILD index 564ff9db2f..b3d3a88928 100644 --- a/src/bytetrack/utils/BUILD +++ b/src/bytetrack/utils/BUILD @@ -1,3 +1,19 @@ +# +# Copyright (c) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") licenses(["notice"]) diff --git a/src/bytetrack/utils/detection_color_by_id_calculator.proto b/src/bytetrack/utils/detection_color_by_id_calculator.proto index 9461e84be8..d7bcdd05b5 100644 --- a/src/bytetrack/utils/detection_color_by_id_calculator.proto +++ b/src/bytetrack/utils/detection_color_by_id_calculator.proto @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + syntax = "proto2"; package mediapipe; diff --git a/src/yolox/BUILD b/src/yolox/BUILD index 9da4fc432f..027fac018e 100644 --- a/src/yolox/BUILD +++ b/src/yolox/BUILD @@ -1,3 +1,19 @@ +# +# Copyright (c) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") mediapipe_proto_library( diff --git a/src/yolox/ov_yolox_tensors_to_detections_calculator.proto b/src/yolox/ov_yolox_tensors_to_detections_calculator.proto index 6517d2a05b..81cacceeb4 100644 --- a/src/yolox/ov_yolox_tensors_to_detections_calculator.proto +++ b/src/yolox/ov_yolox_tensors_to_detections_calculator.proto @@ -1,3 +1,19 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + syntax = "proto2"; package mediapipe; From d2d0703c7239d63edbba97013e640efd923fd570 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Wed, 19 Aug 2026 15:20:01 +0530 Subject: [PATCH 11/18] Updated spelling-whitelist.txt --- spelling-whitelist.txt | 3 +++ src/bytetrack/calculators/basetrack.cc | 2 +- src/bytetrack/calculators/basetrack.h | 6 +++--- src/bytetrack/calculators/bytetrack_calculator.cc | 6 ++++-- src/bytetrack/calculators/bytetrack_calculator.h | 9 ++++++--- src/bytetrack/calculators/kalman_filter.h | 8 +++++--- src/bytetrack/calculators/kalman_matrices.h | 6 +++--- src/bytetrack/calculators/matching_utils.h | 7 ++++--- .../calculators/render_data_passthrough_calculator.cc | 2 +- src/bytetrack/calculators/strack.cc | 5 +++-- src/bytetrack/calculators/strack.h | 6 +++--- src/bytetrack/utils/detection_color_by_id_calculator.cc | 2 +- src/yolox/ov_yolox_tensors_to_detections_calculator.cc | 2 +- 13 files changed, 38 insertions(+), 26 deletions(-) diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index b458909bb3..8bde75bd95 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -38,3 +38,6 @@ windows_parse_tests.bat:141: SEH ==> SHE windows_parse_tests.bat:144: SEH ==> SHE src/test/llm/output_parsers/gemma4_output_parser_test.cpp src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this +STrack +strack +nd \ No newline at end of file diff --git a/src/bytetrack/calculators/basetrack.cc b/src/bytetrack/calculators/basetrack.cc index c096c201e1..63aa88806a 100644 --- a/src/bytetrack/calculators/basetrack.cc +++ b/src/bytetrack/calculators/basetrack.cc @@ -22,4 +22,4 @@ namespace bytetrack { int BaseTrack::count_ = 0; } // namespace bytetrack -} // namespace mediapipe \ No newline at end of file +} // namespace mediapipe diff --git a/src/bytetrack/calculators/basetrack.h b/src/bytetrack/calculators/basetrack.h index 9ad0fbb5a1..ed20a28ad2 100644 --- a/src/bytetrack/calculators/basetrack.h +++ b/src/bytetrack/calculators/basetrack.h @@ -14,8 +14,8 @@ // limitations under the License. //***************************************************************************** -#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BASETRACK_H_ -#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BASETRACK_H_ +#ifndef SRC_BYTETRACK_CALCULATORS_BASETRACK_H_ +#define SRC_BYTETRACK_CALCULATORS_BASETRACK_H_ namespace mediapipe { namespace bytetrack { @@ -55,4 +55,4 @@ class BaseTrack { } // namespace bytetrack } // namespace mediapipe -#endif \ No newline at end of file +#endif // SRC_BYTETRACK_CALCULATORS_BASETRACK_H_ diff --git a/src/bytetrack/calculators/bytetrack_calculator.cc b/src/bytetrack/calculators/bytetrack_calculator.cc index 1640bac1f3..9a303ed8e4 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.cc +++ b/src/bytetrack/calculators/bytetrack_calculator.cc @@ -14,11 +14,14 @@ // limitations under the License. //***************************************************************************** -#include "bytetrack_calculator.h" +#include "src/bytetrack/calculators/bytetrack_calculator.h" #include "src/bytetrack/calculators/bytetrack_calculator.pb.h" #include +#include #include +#include +#include #include @@ -409,7 +412,6 @@ std::vector ByteTrackCalculator::SubStracks(std::vector, std::vector> ByteTrackCalculator::RemoveDuplicateStracks(std::vector& a, std::vector& b) { - auto pdist = BuildIoUCostMatrix(a, b); std::vector dupa, dupb; diff --git a/src/bytetrack/calculators/bytetrack_calculator.h b/src/bytetrack/calculators/bytetrack_calculator.h index 20d96623df..da1148f1fc 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.h +++ b/src/bytetrack/calculators/bytetrack_calculator.h @@ -14,8 +14,11 @@ // limitations under the License. //***************************************************************************** -#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ -#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ +#ifndef SRC_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ +#define SRC_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ + +#include +#include #include "src/bytetrack/calculators/bytetrack_calculator.pb.h" @@ -66,4 +69,4 @@ class ByteTrackCalculator : public CalculatorBase { }; } // namespace mediapipe -#endif \ No newline at end of file +#endif // SRC_BYTETRACK_CALCULATORS_BYTETRACK_CALCULATOR_H_ diff --git a/src/bytetrack/calculators/kalman_filter.h b/src/bytetrack/calculators/kalman_filter.h index 93337db12a..5fc6ca9575 100644 --- a/src/bytetrack/calculators/kalman_filter.h +++ b/src/bytetrack/calculators/kalman_filter.h @@ -14,8 +14,10 @@ // limitations under the License. //***************************************************************************** -#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ -#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ +#ifndef SRC_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ +#define SRC_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ + +#include #include #include @@ -47,4 +49,4 @@ class KalmanFilter { } // namespace bytetrack } // namespace mediapipe -#endif \ No newline at end of file +#endif // SRC_BYTETRACK_CALCULATORS_KALMAN_FILTER_H_ diff --git a/src/bytetrack/calculators/kalman_matrices.h b/src/bytetrack/calculators/kalman_matrices.h index 886b94938e..c53c97721a 100644 --- a/src/bytetrack/calculators/kalman_matrices.h +++ b/src/bytetrack/calculators/kalman_matrices.h @@ -14,8 +14,8 @@ // limitations under the License. //***************************************************************************** // kalman_matrices.h -#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ -#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ +#ifndef SRC_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ +#define SRC_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ #include @@ -40,4 +40,4 @@ inline Eigen::Matrix MakeUpdateMatrix() { } // namespace bytetrack } // namespace mediapipe -#endif \ No newline at end of file +#endif // SRC_BYTETRACK_CALCULATORS_KALMAN_MATRICES_H_ diff --git a/src/bytetrack/calculators/matching_utils.h b/src/bytetrack/calculators/matching_utils.h index f10a3ba732..0c66544fd5 100644 --- a/src/bytetrack/calculators/matching_utils.h +++ b/src/bytetrack/calculators/matching_utils.h @@ -14,11 +14,12 @@ // limitations under the License. //***************************************************************************** -#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ -#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ +#ifndef SRC_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ +#define SRC_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ #include #include +#include #include #include @@ -348,4 +349,4 @@ inline AssignmentResult LinearAssignment(const Eigen::MatrixXf& cost, float thre } // namespace bytetrack } // namespace mediapipe -#endif \ No newline at end of file +#endif // SRC_BYTETRACK_CALCULATORS_MATCHING_UTILS_H_ diff --git a/src/bytetrack/calculators/render_data_passthrough_calculator.cc b/src/bytetrack/calculators/render_data_passthrough_calculator.cc index a69aa87587..cb9c43d40c 100644 --- a/src/bytetrack/calculators/render_data_passthrough_calculator.cc +++ b/src/bytetrack/calculators/render_data_passthrough_calculator.cc @@ -44,4 +44,4 @@ class PassThroughRenderDataCalculator : public CalculatorBase { REGISTER_CALCULATOR(PassThroughRenderDataCalculator); -} // namespace mediapipe \ No newline at end of file +} // namespace mediapipe diff --git a/src/bytetrack/calculators/strack.cc b/src/bytetrack/calculators/strack.cc index 1169f8ac56..40000cf9f2 100644 --- a/src/bytetrack/calculators/strack.cc +++ b/src/bytetrack/calculators/strack.cc @@ -16,6 +16,7 @@ #include "src/bytetrack/calculators/basetrack.h" +#include #include #include @@ -100,7 +101,7 @@ void STrack::Update(const STrack& new_track, int frame_id) { } void STrack::MultiPredict(std::vector& tracks) { - long int n = tracks.size(); + std::size_t n = tracks.size(); if (n > 0) { MeanMatrix multi_mean(n, 8); for (int i = 0; i < n; i++) { @@ -163,4 +164,4 @@ Eigen::Vector4f STrack::tlbr() const { } } // namespace bytetrack -} // namespace mediapipe \ No newline at end of file +} // namespace mediapipe diff --git a/src/bytetrack/calculators/strack.h b/src/bytetrack/calculators/strack.h index f16cce9f0d..345a0de89a 100644 --- a/src/bytetrack/calculators/strack.h +++ b/src/bytetrack/calculators/strack.h @@ -14,8 +14,8 @@ // limitations under the License. //***************************************************************************** -#ifndef MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_STRACK_H_ -#define MEDIAPIPE_GRAPHS_BYTETRACK_CALCULATORS_STRACK_H_ +#ifndef SRC_BYTETRACK_CALCULATORS_STRACK_H_ +#define SRC_BYTETRACK_CALCULATORS_STRACK_H_ #include #include @@ -73,4 +73,4 @@ class STrack : public BaseTrack { } // namespace bytetrack } // namespace mediapipe -#endif \ No newline at end of file +#endif // SRC_BYTETRACK_CALCULATORS_STRACK_H_ diff --git a/src/bytetrack/utils/detection_color_by_id_calculator.cc b/src/bytetrack/utils/detection_color_by_id_calculator.cc index d60e902719..4aac069178 100644 --- a/src/bytetrack/utils/detection_color_by_id_calculator.cc +++ b/src/bytetrack/utils/detection_color_by_id_calculator.cc @@ -136,4 +136,4 @@ class DetectionColorByIdCalculator : public CalculatorBase { REGISTER_CALCULATOR(DetectionColorByIdCalculator); -} // namespace mediapipe \ No newline at end of file +} // namespace mediapipe diff --git a/src/yolox/ov_yolox_tensors_to_detections_calculator.cc b/src/yolox/ov_yolox_tensors_to_detections_calculator.cc index d627ceb9c8..6ad749a454 100644 --- a/src/yolox/ov_yolox_tensors_to_detections_calculator.cc +++ b/src/yolox/ov_yolox_tensors_to_detections_calculator.cc @@ -159,4 +159,4 @@ const std::string OVYoloXTensorsToDetectionsCalculator::OUTPUT_TAG_NAME{ REGISTER_CALCULATOR(OVYoloXTensorsToDetectionsCalculator); -} // namespace mediapipe \ No newline at end of file +} // namespace mediapipe From 03691402a0cd2a1263e08f1e1530e691678b0342 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Thu, 20 Aug 2026 22:00:29 +0530 Subject: [PATCH 12/18] Update README --- demos/mediapipe/bytetrack/README.md | 117 ++++++------------- demos/mediapipe/bytetrack/requirements.txt | Bin 316 -> 300 bytes src/kfs_frontend/kfs_graph_executor_impl.cpp | 22 +--- 3 files changed, 38 insertions(+), 101 deletions(-) diff --git a/demos/mediapipe/bytetrack/README.md b/demos/mediapipe/bytetrack/README.md index c5cd0fe247..dfaf8ead1e 100644 --- a/demos/mediapipe/bytetrack/README.md +++ b/demos/mediapipe/bytetrack/README.md @@ -1,121 +1,70 @@ # ByteTrack Demo Setup -End-to-end demo: video source (webcam / file / RTSP) → OpenVINO Model Server (YOLOX Tiny+ ByteTrack) → output (screen / file / RTSP). +End-to-end demo: video source (webcam / file) → OpenVINO Model Server (YOLOX Tiny + ByteTrack) → output (screen / file). --- -## 1. Model Preparation +## Steps -The detector stage of the pipeline runs on [OpenVINO](https://github.com/openvinotoolkit/openvino)-optimized YOLOX models. The following FP16 variants are currently supported: +### 1. Clone the repository -| Model | HuggingFace Repo | -|---|---| -| YOLOX-Tiny (fp16 precision)| `OpenVINO/yolox_tiny-fp16-ov` | -| YOLOX-Tiny (int8 precision)| `OpenVINO/yolox_tiny-int8-ov` | - -`yolox_tiny-fp16-ov` is the default used in this demo. - -### Install requirements - -```bash -pip install -r requirements.txt -``` - -### Download a model - -```bash -python download_models.py --model OpenVINO/yolox_tiny-fp16-ov -``` - -> Swap `--model` for any of the four repo IDs listed above to use a different YOLOX size. The script also downloads the COCO class list used for labeling detections. - -This populates the local model directory that `config.json` (used by the OpenVINO Model Server in step 3) points to, and that ByteTrack consumes downstream for tracking. - ---- - -## 2. Start an RTSP relay server - -If you don't already have one running, start MediaMTX (or an equivalent RTSP server) so the streams below have somewhere to publish to: - -```bash -docker run --rm -d -p 8554:8554 -e RTSP_PROTOCOLS=tcp bluenviron/mediamtx:latest -``` - -> Only needed if you plan to use RTSP input and/or output (see Demo 3 below). It isn't required for the local webcam→screen or video-file→mp4 demos. - ---- - -## 3. Start the OpenVINO Model Server +Clone the repository, switch to the `gsoc_bytetrack` branch, and move into the demo directory: ```bash -docker run -d -v $PWD:/demo -p 9000:9000 openvino/model_server:latest --config_path /demo/config.json --port 9000 +git clone https://github.com/Vishwa2684/model_server +cd model_server +git checkout gsoc_bytetrack +cd demos/mediapipe/bytetrack ``` ---- - -## 4. Run the Demo - -The `client.py` script (from `real_time_stream_analysis`) supports several combinations of input and output, so the same server can be exercised in different ways depending on what you have available. +### 2. Install requirements -### Demo A — Local webcam → screen - -Reads directly from a local camera and renders the tracked output in a window. +Install all the Python dependencies needed by the client and the model download script. Run this from inside the `demos/mediapipe/bytetrack` directory: ```bash -python client.py --grpc_address localhost:9000 --input_stream 0 --output_stream screen +pip install -r requirements.txt ``` -- `--input_stream 0` — camera device ID `0` (use `1`, `2`, etc. for additional cameras). -- `--output_stream screen` — opens a live preview window instead of writing to a file or stream. - -### Demo B — Video file → video file +### 3. Download a model -Reads from an encoded video file and writes the annotated result to a new video file. +Download the detector model that the OpenVINO Model Server will use. This same command also fetches the COCO class list used for labeling detections: ```bash -curl -L "https://raw.githubusercontent.com/FoundationVision/ByteTrack/main/videos/palace.mp4" -o video.mp4 -python client.py --grpc_address localhost:9000 --input_stream video.mp4 --output_stream output.mp4 +python download_models.py --model OpenVINO/yolox_tiny-fp16-ov ``` -- `--input_stream video.mp4` — path to the source video. -- `--output_stream output.mp4` — path where the tracked/annotated video is saved. +> Swap `--model` for any of the repo IDs listed below to use a different YOLOX size. -### Demo C — RTSP → RTSP +| Model | HuggingFace Repo | +|---|---| +| YOLOX-Tiny (fp16 precision)| `OpenVINO/yolox_tiny-fp16-ov` | +| YOLOX-Tiny (int8 precision)| `OpenVINO/yolox_tiny-int8-ov` | -Full end-to-end streaming demo: publish a webcam feed to an RTSP endpoint, run detection + tracking on it, and publish the annotated result to a second RTSP endpoint. +`yolox_tiny-fp16-ov` is the default used in this demo. -**1. Publish your webcam as an RTSP input stream** +This step populates the local model directory that `config.json` (used by the OpenVINO Model Server in step 4) points to, and that ByteTrack consumes downstream for tracking. -```bash -ffmpeg -f dshow -video_size 1280x720 -i video="HP True Vision FHD Camera" -f rtsp -rtsp_transport tcp rtsp://localhost:8554/channel1 -``` +### 4. Start the OpenVINO Model Server -**2. Run the client against the RTSP input/output** +Bring up the OpenVINO Model Server as a Docker container. This mounts your current directory into the container so it can read `config.json`, and exposes port 9000 for the client to connect to: ```bash -python client.py --grpc_address localhost:9000 --input_stream rtsp://localhost:8554/channel1 --output_stream rtsp://localhost:8554/channel2 --model_name ByteTrack --input_name input_video +docker run -d -v $PWD:/demo -p 9000:9000 openvino/model_server:latest --config_path /demo/config.json --port 9000 ``` -**3. View the output stream** +Leave this container running in the background — the client in the next step connects to it over gRPC. -Option 1 (recommended): +### 5. Run the demo — local webcam → screen -```bash -ffplay -rtsp_transport tcp -vf "scale=704:704,format=yuv420p" rtsp://localhost:8554/channel2 -``` - -Option 2 (verbose logging): +With the model server running, run the client script. This reads directly from your local webcam, runs it through detection + ByteTrack tracking, and renders the annotated output live in a window on your screen: ```bash -ffplay -loglevel verbose -rtsp_transport tcp rtsp://localhost:8554/channel2 +cd ../../real_time_stream_analysis/python +python client.py --grpc_address localhost:9000 --input_stream 0 --output_stream screen --model_name ByteTrack --input_name input_video ``` ---- - -## Summary of I/O Options +- `--grpc_address localhost:9000` — address of the OpenVINO Model Server started in step 4. +- `--input_stream 0` — camera device ID `0` (use `1`, `2`, etc. if you have multiple cameras and want a different one). +- `--output_stream screen` — opens a live preview window instead of writing to a file or stream. -| Demo | Input | Output | RTSP server required? | -|---|---|---|---| -| A | Local webcam (`0`) | `screen` | No | -| B | Video file (`video.mp4`) | Video file (`output.mp4`) | No | -| C | RTSP stream | RTSP stream | Yes | \ No newline at end of file +A window should open showing your webcam feed with tracked bounding boxes drawn on it in real time. To use different input and output streams for real time. Read the documentation on [real time stream analysis](../../real_time_stream_analysis/python/README.md) \ No newline at end of file diff --git a/demos/mediapipe/bytetrack/requirements.txt b/demos/mediapipe/bytetrack/requirements.txt index be53447c3b774c5a54dfe278369e1d474f638cff..66e7fd02ece5d0dd9afe72e565b6fddbcedf2a11 100644 GIT binary patch delta 31 kcmdnPw1#Pd!o&|O{3Z-~43Z1B8CEnWQI(J{E0Is0|3O&3A_LR diff --git a/src/kfs_frontend/kfs_graph_executor_impl.cpp b/src/kfs_frontend/kfs_graph_executor_impl.cpp index fbb051d5bd..5c197db6f8 100644 --- a/src/kfs_frontend/kfs_graph_executor_impl.cpp +++ b/src/kfs_frontend/kfs_graph_executor_impl.cpp @@ -1027,16 +1027,15 @@ static inline Status checkTimestamp(const KFSRequest& request, const ::mediapipe return StatusCode::OK; } -// this method static Status deserializeTimestampIfAvailable( const KFSRequest& request, - ::mediapipe::Timestamp& timestamp, - const ::mediapipe::Timestamp& lastPushedTimestamp) { // pass in current stream state + ::mediapipe::Timestamp& timestamp) { auto timestampParamIt = request.parameters().find(TIMESTAMP_PARAMETER_NAME); if (timestampParamIt != request.parameters().end()) { SPDLOG_DEBUG("Found {} timestamp parameter in request for: {}", TIMESTAMP_PARAMETER_NAME, request.model_name()); auto& parameterChoice = timestampParamIt->second; if (parameterChoice.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kInt64Param) { + // Cannot create with error checking since error check = abseil death test timestamp = ::mediapipe::Timestamp::CreateNoErrorChecking(parameterChoice.int64_param()); if (!timestamp.IsRangeValue()) { SPDLOG_DEBUG("Timestamp not in range: {}; for request to: {};", timestamp.DebugString(), request.model_name()); @@ -1048,17 +1047,8 @@ static Status deserializeTimestampIfAvailable( return status; } } else { - // steady_clock never jumps backward; system_clock can - auto now = std::chrono::steady_clock::now().time_since_epoch(); - int64_t candidate = std::chrono::duration_cast(now).count(); - timestamp = ::mediapipe::Timestamp::CreateNoErrorChecking(candidate); - } - - // Enforce strict monotonicity regardless of source - if (lastPushedTimestamp != ::mediapipe::Timestamp::Unset() && timestamp <= lastPushedTimestamp) { - SPDLOG_DEBUG("Non-monotonic timestamp detected: new={} <= last={}; for request to: {}", - timestamp.DebugString(), lastPushedTimestamp.DebugString(), request.model_name()); - return Status(StatusCode::MEDIAPIPE_INVALID_TIMESTAMP, "Timestamp did not increase relative to previous packet in stream"); + auto now = std::chrono::system_clock::now(); + timestamp = ::mediapipe::Timestamp(std::chrono::duration_cast(now.time_since_epoch()).count()); } return StatusCode::OK; } @@ -1143,7 +1133,6 @@ Status onPacketReadySerializeImpl( return status; } -// this method Status createAndPushPacketsImpl( std::shared_ptr request, stream_types_mapping_t& inputTypes, @@ -1152,8 +1141,7 @@ Status createAndPushPacketsImpl( ::mediapipe::Timestamp& currentTimestamp, size_t& numberOfPacketsCreated) { - const ::mediapipe::Timestamp lastPushedTimestamp = currentTimestamp; // snapshot BEFORE overwrite - OVMS_RETURN_ON_FAIL(deserializeTimestampIfAvailable(*request, currentTimestamp, lastPushedTimestamp)); + OVMS_RETURN_ON_FAIL(deserializeTimestampIfAvailable(*request, currentTimestamp)); OVMS_RETURN_ON_FAIL(checkTimestamp(*request, currentTimestamp)); OVMS_RETURN_ON_FAIL(validateRequestCoherencyKFS(*request, request->model_name(), MediapipeGraphDefinition::VERSION)); From 06102406343cb72512d0f4ded3d1b286b653b2a6 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Mon, 24 Aug 2026 18:50:44 +0530 Subject: [PATCH 13/18] Add ByteTrack calculators to WhitelistRegistered.MediapipeCalculatorsList --- src/test/mediapipeflow_test.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index c9fa5c22f2..6daa122d0e 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -3811,6 +3811,7 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "BeginLoopUint64tCalculator", "BoxDetectorCalculator", "BoxTrackerCalculator", + "ByteTrackCalculator", "CallbackCalculator", "CallbackPacketCalculator", "CallbackWithHeaderCalculator", @@ -3849,6 +3850,9 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "DetectionClassificationCombinerCalculator", "DetectionClassificationResultCalculator", "DetectionClassificationSerializationCalculator", + "DetectionColorByIdCalculator", + "PassThroughRenderDataCalculator", + "OVYoloXTensorsToDetectionsCalculator", "DetectionExtractionCalculator", "DetectionLabelIdToTextCalculator", "DetectionLetterboxRemovalCalculator", From d8f82c5b5f5f8e3831f0a648fdb1b891045f5b2e Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Wed, 26 Aug 2026 19:17:56 +0530 Subject: [PATCH 14/18] add os.makedirs to create directory --- demos/mediapipe/bytetrack/download_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/mediapipe/bytetrack/download_models.py b/demos/mediapipe/bytetrack/download_models.py index 997af5a82c..4e68753818 100644 --- a/demos/mediapipe/bytetrack/download_models.py +++ b/demos/mediapipe/bytetrack/download_models.py @@ -107,7 +107,7 @@ model = ppp.build() output_path = f"{MODEL_NAME}/1/{MODEL_NAME}.xml" - +os.makedirs(os.path.dirname(output_path), exist_ok=True) ov.save_model(model, output_path) print("\nSaved:", os.path.abspath(output_path)) From edde13f2a8575a2c3b1efe7e05b113b8b83ddf13 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Wed, 26 Aug 2026 19:22:10 +0530 Subject: [PATCH 15/18] Minor change in README --- demos/mediapipe/bytetrack/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/mediapipe/bytetrack/README.md b/demos/mediapipe/bytetrack/README.md index dfaf8ead1e..bcf038874b 100644 --- a/demos/mediapipe/bytetrack/README.md +++ b/demos/mediapipe/bytetrack/README.md @@ -33,7 +33,7 @@ Download the detector model that the OpenVINO Model Server will use. This same c python download_models.py --model OpenVINO/yolox_tiny-fp16-ov ``` -> Swap `--model` for any of the repo IDs listed below to use a different YOLOX size. +> Swap `--model-repo` for any of the repo IDs listed below to use a different YOLOX size. | Model | HuggingFace Repo | |---|---| From c3eb6affcd2af0cdb9c407c1666303193a296100 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Wed, 26 Aug 2026 23:50:20 +0530 Subject: [PATCH 16/18] Remove removed_stracks_ in ByteTrackCalculator --- src/bytetrack/calculators/BUILD | 3 ++- src/bytetrack/calculators/bytetrack_calculator.cc | 13 ++++++------- src/bytetrack/calculators/bytetrack_calculator.h | 1 - src/bytetrack/calculators/matching_utils.h | 2 +- src/bytetrack/calculators/strack.h | 2 +- .../utils/detection_color_by_id_calculator.cc | 6 +++++- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/bytetrack/calculators/BUILD b/src/bytetrack/calculators/BUILD index b3e1641f1e..daec9152b8 100644 --- a/src/bytetrack/calculators/BUILD +++ b/src/bytetrack/calculators/BUILD @@ -56,7 +56,8 @@ cc_library( name = "matching_utils", hdrs = ["matching_utils.h"], deps = [ - "@mediapipe//mediapipe/util/tracking:box_tracker", + ":strack", + "@eigen_archive//:eigen3", ], alwayslink = 1 ) diff --git a/src/bytetrack/calculators/bytetrack_calculator.cc b/src/bytetrack/calculators/bytetrack_calculator.cc index 9a303ed8e4..f21ef067f4 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.cc +++ b/src/bytetrack/calculators/bytetrack_calculator.cc @@ -86,7 +86,7 @@ absl::Status ByteTrackCalculator::Process(CalculatorContext* cc) { << " tracked_=" << tracked_stracks_.size() << " lost_=" << lost_stracks_.size(); - LOG(INFO) << "ByteTrackCalculator::Process called, frame " << frame_id_; + // LOG(INFO) << "ByteTrackCalculator::Process called, frame " << frame_id_; auto high_dets = std::make_unique>(); auto low_dets = std::make_unique>(); @@ -300,7 +300,7 @@ absl::Status ByteTrackCalculator::Process(CalculatorContext* cc) { lost_ptrs2.push_back(&t); for (auto& t : tracked_stracks_) new_tracked_ptrs.push_back(&t); // fresh pointers! - for (auto& t : removed_stracks_) + for (auto& t : removed_stracks) removed_ptrs.push_back(&t); for (auto& t : lost_stracks) local_lost_ptrs.push_back(&t); @@ -313,14 +313,13 @@ absl::Status ByteTrackCalculator::Process(CalculatorContext* cc) { // for (auto* t : new_lost) lost_stracks_.push_back(*t); std::vector l_stracks; l_stracks.reserve(new_lost.size()); - for (auto* t : new_lost) - l_stracks.push_back(*t); + for (auto* t : new_lost) { + if (t->state() != bytetrack::BaseTrack::TrackState::REMOVED) + l_stracks.push_back(*t); + } lost_stracks_ = std::move(l_stracks); } - // extend removed_stracks_ - for (auto& t : removed_stracks) - removed_stracks_.push_back(t); // remove duplicates { diff --git a/src/bytetrack/calculators/bytetrack_calculator.h b/src/bytetrack/calculators/bytetrack_calculator.h index da1148f1fc..dd56e3ab9b 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.h +++ b/src/bytetrack/calculators/bytetrack_calculator.h @@ -38,7 +38,6 @@ class ByteTrackCalculator : public CalculatorBase { ::mediapipe::ByteTrackCalculatorOptions options_; std::vector tracked_stracks_; std::vector lost_stracks_; - std::vector removed_stracks_; int track_buffer_; float det_thresh_; diff --git a/src/bytetrack/calculators/matching_utils.h b/src/bytetrack/calculators/matching_utils.h index 0c66544fd5..65ffc3fd35 100644 --- a/src/bytetrack/calculators/matching_utils.h +++ b/src/bytetrack/calculators/matching_utils.h @@ -74,7 +74,7 @@ inline Eigen::MatrixXf BuildIoUCostMatrix( } inline Eigen::MatrixXf FuseScore( - const Eigen::MatrixXf cost_matrix, + const Eigen::MatrixXf& cost_matrix, const std::vector& detections) { if (cost_matrix.size() == 0) { return cost_matrix; diff --git a/src/bytetrack/calculators/strack.h b/src/bytetrack/calculators/strack.h index 345a0de89a..45cefb6848 100644 --- a/src/bytetrack/calculators/strack.h +++ b/src/bytetrack/calculators/strack.h @@ -55,7 +55,7 @@ class STrack : public BaseTrack { // KalmanState ToProto() const; int tracklet_len() const { return tracklet_len_; } - std::string label() const { return label_; } + const std::string& label() const { return label_; } const Mean& mean() const { return mean_; } const Cov& cov() const { return cov_; } diff --git a/src/bytetrack/utils/detection_color_by_id_calculator.cc b/src/bytetrack/utils/detection_color_by_id_calculator.cc index 4aac069178..9b8d85b4d3 100644 --- a/src/bytetrack/utils/detection_color_by_id_calculator.cc +++ b/src/bytetrack/utils/detection_color_by_id_calculator.cc @@ -16,6 +16,10 @@ #include "src/bytetrack/utils/detection_color_by_id_calculator.pb.h" #include +#include +#include +#include +#include #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/detection.pb.h" @@ -34,7 +38,7 @@ class DetectionColorByIdCalculator : public CalculatorBase { absl::Status Open(CalculatorContext* cc) override { const auto& options = cc->Options(); - thickness_ = options.has_thickness() ? options.thickness() : 4.0f; + thickness_ = options.has_thickness() ? options.thickness() : 5.0f; saturation_ = options.has_saturation() ? options.saturation() : 0.85f; value_ = options.has_value() ? options.value() : 0.95f; return absl::OkStatus(); From 76e1bbf7423b57ad2e22c675c7cf233b4ffaa128 Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Thu, 27 Aug 2026 22:32:31 +0530 Subject: [PATCH 17/18] use atomic variable for counter to prevent data race --- src/bytetrack/calculators/basetrack.cc | 4 +++- src/bytetrack/calculators/basetrack.h | 8 +++++--- src/bytetrack/calculators/bytetrack_calculator.cc | 7 +++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/bytetrack/calculators/basetrack.cc b/src/bytetrack/calculators/basetrack.cc index 63aa88806a..11aa702fba 100644 --- a/src/bytetrack/calculators/basetrack.cc +++ b/src/bytetrack/calculators/basetrack.cc @@ -16,10 +16,12 @@ // basetrack.cpp #include "src/bytetrack/calculators/basetrack.h" +#include + namespace mediapipe { namespace bytetrack { -int BaseTrack::count_ = 0; +std::atomic BaseTrack::count_{0}; } // namespace bytetrack } // namespace mediapipe diff --git a/src/bytetrack/calculators/basetrack.h b/src/bytetrack/calculators/basetrack.h index ed20a28ad2..d1f42dc012 100644 --- a/src/bytetrack/calculators/basetrack.h +++ b/src/bytetrack/calculators/basetrack.h @@ -17,6 +17,8 @@ #ifndef SRC_BYTETRACK_CALCULATORS_BASETRACK_H_ #define SRC_BYTETRACK_CALCULATORS_BASETRACK_H_ +#include + namespace mediapipe { namespace bytetrack { @@ -28,8 +30,8 @@ class BaseTrack { REMOVED }; // mirrors Python's next_id() — static counter owned here - static int next_id() { return ++count_; } - static void reset_id() { count_ = 0; } + static int next_id() { return count_.fetch_add(1, std::memory_order_relaxed) + 1; } + static void reset_id() { count_.store(0, std::memory_order_relaxed); } // Accessors int track_id() const { return track_id_; } @@ -49,7 +51,7 @@ class BaseTrack { float score_ = 0.f; bool is_activated_ = false; TrackState state_ = TrackState::NEW; - static int count_; + static std::atomic count_; }; } // namespace bytetrack diff --git a/src/bytetrack/calculators/bytetrack_calculator.cc b/src/bytetrack/calculators/bytetrack_calculator.cc index f21ef067f4..9feb32eb13 100644 --- a/src/bytetrack/calculators/bytetrack_calculator.cc +++ b/src/bytetrack/calculators/bytetrack_calculator.cc @@ -314,13 +314,12 @@ absl::Status ByteTrackCalculator::Process(CalculatorContext* cc) { std::vector l_stracks; l_stracks.reserve(new_lost.size()); for (auto* t : new_lost) { - if (t->state() != bytetrack::BaseTrack::TrackState::REMOVED) - l_stracks.push_back(*t); - } + if (t->state() != bytetrack::BaseTrack::TrackState::REMOVED) + l_stracks.push_back(*t); + } lost_stracks_ = std::move(l_stracks); } - // remove duplicates { std::vector tracked_ptrs2, lost_ptrs3; From d413dd6873db78e6dabdc87fb68e8d047ff159ef Mon Sep 17 00:00:00 2001 From: Vishwa2684 Date: Sat, 29 Aug 2026 22:10:09 +0530 Subject: [PATCH 18/18] Refactor LinearAssignment to improve condition checks for column assignments --- src/bytetrack/calculators/matching_utils.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/bytetrack/calculators/matching_utils.h b/src/bytetrack/calculators/matching_utils.h index 65ffc3fd35..5283608931 100644 --- a/src/bytetrack/calculators/matching_utils.h +++ b/src/bytetrack/calculators/matching_utils.h @@ -230,11 +230,13 @@ inline AssignmentResult LinearAssignment(const Eigen::MatrixXf& cost, float thre } } u[i] = u1; - if (col2row[j1] == -1) { - row2col[i] = j1; - col2row[j1] = i; - } else { - v[j1] -= (u2 - u1); + if (j1 != -1) { + if (col2row[j1] == -1) { + row2col[i] = j1; + col2row[j1] = i; + } else { + v[j1] -= (u2 - u1); + } } } }