From 07f7cc8654b7e19667cba547a4d2b0348baa1211 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Wed, 17 Jun 2026 22:27:49 -0700 Subject: [PATCH 01/14] feat(coreml): extern "C" tracker C-ABI for cgo (RFD 0011 U8 Wave 2) edgetam_capi.{h,cpp}: POD-only seed/track/reset over the visual tracker (sam3_create_visual_tracker / propagate_frame); CoreML stages enabled from models_dir. Compiles into libsam3.a; verified pure-C link + run against the ggml dylibs + CoreML/Foundation/Metal/MetalKit/Accelerate + -lc++. Co-Authored-By: Claude Opus 4.8 (1M context) --- CMakeLists.txt | 2 + coreml/edgetam_capi.cpp | 117 ++++++++++++++++++++++++++++++++++++++++ coreml/edgetam_capi.h | 55 +++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 coreml/edgetam_capi.cpp create mode 100644 coreml/edgetam_capi.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1aacee0..bd5bed1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,8 @@ if(APPLE) set_source_files_properties(coreml/edgetam_coreml.mm PROPERTIES LANGUAGE CXX COMPILE_FLAGS "-x objective-c++ -fobjc-arc") + # RFD 0011 U8: extern "C" tracker C-ABI for cgo into iris/platform. + target_sources(sam3 PRIVATE coreml/edgetam_capi.cpp) target_include_directories(sam3 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/coreml) target_compile_definitions(sam3 PUBLIC SAM3_COREML) target_link_libraries(sam3 PUBLIC "-framework CoreML" "-framework Foundation") diff --git a/coreml/edgetam_capi.cpp b/coreml/edgetam_capi.cpp new file mode 100644 index 0000000..11afada --- /dev/null +++ b/coreml/edgetam_capi.cpp @@ -0,0 +1,117 @@ +// RFD 0011 U8 — implementation of the extern "C" tracker C-ABI (edgetam_capi.h). +// Thin POD wrapper over the sam3.cpp visual tracker (sam3_create_visual_tracker / +// sam3_encode_image / sam3_tracker_add_instance / sam3_propagate_frame). All C++ +// exceptions are caught at the boundary (cgo cannot unwind through Go). +#include "edgetam_capi.h" +#include "../sam3.h" + +#include +#include +#include +#include +#include + +namespace { + +struct EtTracker { + std::shared_ptr model; + sam3_state_ptr state; + sam3_tracker_ptr tracker; +}; + +// Wrap a caller RGB24 buffer as a sam3_image (copies into the owned vector — the +// library keeps no alias, and cgo must not let Go memory escape into C). +sam3_image make_image(const uint8_t* rgb, int w, int h) { + sam3_image img; + img.width = w; img.height = h; img.channels = 3; + img.data.assign(rgb, rgb + (size_t)w * h * 3); + return img; +} + +} // namespace + +extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, + const char* ggml_model, + int /*use_samurai*/, int /*use_lifecycle*/) { + try { + // Enable the CoreML stages (encoder ANE + mem-attn GPU + decoder ANE) by + // pointing the library's existing runtime gates at the .mlpackage dir. + // The library's full-CoreML propagation path then runs with zero ggml + // graph work per frame; ggml remains the seed + early-frame fallback. + if (models_dir && *models_dir) { + std::string d = models_dir; + setenv("SAM3_COREML_ENCODER", "1", 1); + setenv("SAM3_COREML_MODEL", (d + "/edgetam_encoder_neck_nhwc.mlpackage").c_str(), 1); + setenv("SAM3_COREML_MEMATTN", "1", 1); + setenv("SAM3_COREML_MEMATTN_MODEL", (d + "/edgetam_memory_attention.mlpackage").c_str(), 1); + setenv("SAM3_COREML_DECODER", "1", 1); + setenv("SAM3_COREML_DECODER_MODEL", (d + "/edgetam_mask_decoder_nhwc.mlpackage").c_str(), 1); + } + auto t = std::make_unique(); + sam3_params p; + p.model_path = ggml_model ? ggml_model : ""; + p.use_gpu = true; + t->model = sam3_load_model(p); + if (!t->model) return nullptr; + t->state = sam3_create_state(*t->model, p); + sam3_visual_track_params vp; // defaults match sam3_edgetam_bench + t->tracker = sam3_create_visual_tracker(*t->model, vp); + return t.release(); + } catch (...) { + return nullptr; + } +} + +extern "C" int edgetam_capi_seed(edgetam_tracker_t h, const uint8_t* rgb, int w, int hgt, et_box seed) { + auto* t = static_cast(h); + if (!t || !rgb) return 0; + try { + sam3_image img = make_image(rgb, w, hgt); + if (!sam3_encode_image(*t->state, *t->model, img)) return 0; + sam3_pvs_params pvs; + pvs.use_box = true; + pvs.box = {seed.x0, seed.y0, seed.x1, seed.y1}; + int id = sam3_tracker_add_instance(*t->tracker, *t->state, *t->model, pvs); + return id >= 0 ? 1 : 0; + } catch (...) { + return 0; + } +} + +extern "C" et_result edgetam_capi_track(edgetam_tracker_t h, const uint8_t* rgb, int w, int hgt) { + et_result r; + std::memset(&r, 0, sizeof(r)); + auto* t = static_cast(h); + if (!t || !rgb) return r; + try { + sam3_image img = make_image(rgb, w, hgt); + sam3_result res = sam3_propagate_frame(*t->tracker, *t->state, *t->model, img); + if (res.detections.empty()) return r; // valid stays 0 -> lost + const sam3_detection& d = res.detections[0]; + const float W = (float)w, H = (float)hgt; + r.box.x0 = d.box.x0 / W; r.box.y0 = d.box.y0 / H; + r.box.x1 = d.box.x1 / W; r.box.y1 = d.box.y1 / H; + r.valid = 1; + r.obj_score = d.score; + r.mask_iou = d.iou_score; + r.state = (int)d.state; + return r; + } catch (...) { + return r; + } +} + +extern "C" void edgetam_capi_reset(edgetam_tracker_t h) { + auto* t = static_cast(h); + if (t && t->tracker) { + try { sam3_tracker_reset(*t->tracker); } catch (...) {} + } +} + +extern "C" void edgetam_capi_destroy(edgetam_tracker_t h) { + delete static_cast(h); +} + +extern "C" const char* edgetam_capi_version(void) { + return SAM3_VERSION; +} diff --git a/coreml/edgetam_capi.h b/coreml/edgetam_capi.h new file mode 100644 index 0000000..886c9da --- /dev/null +++ b/coreml/edgetam_capi.h @@ -0,0 +1,55 @@ +// RFD 0011 U8 — extern "C" C-ABI over the sam3.cpp EdgeTAM visual tracker, for +// cgo into iris/platform. POD only crosses this boundary (cgo cannot call the +// std:: C++ API in sam3.h). The engine underneath is the existing visual tracker +// (memory bank + U3 SAMURAI + U4 lifecycle); CoreML stages are enabled per +// edgetam_capi_create's models_dir. The pure-CoreML 20 fps engine (Wave 1) can +// later swap behind this same ABI without changing the platform cgo side. +#ifndef EDGETAM_CAPI_H +#define EDGETAM_CAPI_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* edgetam_tracker_t; + +// Box corners. seed (edgetam_capi_seed) is in ORIGINAL-FRAME PIXELS; results +// (et_result.box) are NORMALIZED [0,1] (matches hold.Box.Valid on the Go side). +typedef struct { float x0, y0, x1, y1; } et_box; + +// state mirrors sam3.h TargetState: 0 TRACKED, 1 AT_RISK, 2 OCCLUDED, 3 LOST, +// 4 CANDIDATE_REACQUIRE, 5 REACQUIRED. +typedef struct { + et_box box; + int valid; // 1 if a credible detection this frame + float obj_score; + float mask_iou; + int state; +} et_result; + +// Create a tracker. ggml_model = path to edgetam_f16.ggml (required, the seed +// path + ggml fallback). models_dir = dir holding the 4 CoreML .mlpackages; if +// non-empty, the CoreML encoder/mem-attn/decoder stages are enabled. Returns +// NULL on failure. +edgetam_tracker_t edgetam_capi_create(const char* models_dir, const char* ggml_model, + int use_samurai, int use_lifecycle); + +// Seed the target from a box on frame 0. rgb = contiguous RGB24 (w*h*3 bytes). +// seed box in PIXELS. Returns 1 on success. +int edgetam_capi_seed(edgetam_tracker_t t, const uint8_t* rgb, int w, int h, et_box seed_px); + +// Track one frame. rgb = contiguous RGB24 (w*h*3). Returns the per-frame result +// (box normalized [0,1]); valid==0 when the target is lost this frame. +et_result edgetam_capi_track(edgetam_tracker_t t, const uint8_t* rgb, int w, int h); + +void edgetam_capi_reset(edgetam_tracker_t t); +void edgetam_capi_destroy(edgetam_tracker_t t); +const char* edgetam_capi_version(void); // SAM3_VERSION, for cgo link sanity + +#ifdef __cplusplus +} +#endif + +#endif // EDGETAM_CAPI_H From 975fdd0fe9c2cca6bb9a43c46458d3ab9e78e9c6 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Wed, 17 Jun 2026 22:45:48 -0700 Subject: [PATCH 02/14] =?UTF-8?q?feat(coreml):=20CoreML=20memencode=20in?= =?UTF-8?q?=20tracker=20=E2=80=94=20speed=20path=20(WIP,=20gated)=20(RFD?= =?UTF-8?q?=200011=20U8=20Wave=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sam3_encode_memory: SAM3_COREML_MEMENC gate calls edgetam_coreml_memencode and stores the perceiver slot, skipping the per-frame ggml memencode graph. SPEED VERIFIED: dt0004 7.8→10.86 fps (mem_encoder 24→0ms, full-CoreML 4-stage). ACCURACY BROKEN: 0.000 (107 lost) — memencode slot-parity bug (pix_feat/mask inputs layout-verified correct; output/model-expectation parity is the remaining unknown; needs a cosine-compare vs edgetam_perceiver_forward). Gated OFF by default — the ggml memencode remains the correct path. Not for merge as-is. Co-Authored-By: Claude Opus 4.8 (1M context) --- sam3.cpp | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/sam3.cpp b/sam3.cpp index 1506b32..85164d7 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -12106,6 +12106,73 @@ static bool sam3_encode_memory( // Mask preprocessing: mask_logits → HIGH_RES → sigmoid → scale/bias → INTERPOL auto m_hires = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES); +#ifdef SAM3_COREML + // RFD 0011 U8 Wave 1: CoreML memory-encode (memory_encoder + spatial_perceiver, + // ~6ms ANE vs ~24ms ggml). ⚠️ WIP — SPEED VERIFIED (7.8→10.86 fps, mem_encoder + // 24→0ms) but ACCURACY BROKEN (goldeneval dt0004 0.000, 107 lost): the slot the + // model produces here mismatches the ggml perceiver slot — a pix_feat (HWC→BCHW) + // or mask-input layout-parity bug in the seed/conditioning slot. FIX: dump this + // path's mfeat/mpos vs edgetam_perceiver_forward's perc_latents/perc_pos for one + // frame and cosine-compare to find the wrong axis. Gated OFF by default (env + // SAM3_COREML_MEMENC), so the ggml memencode below remains the correct path. + // The exported model applies the mask sigmoid internally + // (convert_memenc_coreml.py: skip_mask_sigmoid=False), so it takes RAW logits at + // HIGH_RES — i.e. m_hires BEFORE the sigmoid below. pix_feat is neck2 transposed + // from ggml HWC (channel-fastest) to BCHW. Output is the perceiver slot + // (== the ggml path's perc_latents/perc_pos), stored identically. Returns early, + // skipping the per-frame ggml memencode graph entirely. + if (getenv("SAM3_COREML_MEMENC") && hp.has_perceiver) { + static edgetam_coreml_handle s_memenc = nullptr; + static bool s_memenc_tried = false; + if (!s_memenc && !s_memenc_tried) { + s_memenc_tried = true; + const char* mp = getenv("SAM3_COREML_MEMENC_MODEL"); + if (mp) { + s_memenc = edgetam_coreml_create(mp, /*CPU_AND_NE*/ 1); + if (s_memenc) fprintf(stderr, "%s: CoreML memenc loaded (ANE): %s\n", __func__, mp); + } + } + if (s_memenc) { + std::vector n2((size_t)D * H * H), pix_chw((size_t)D * H * H); + ggml_backend_tensor_get(state.neck_trk[2], n2.data(), 0, (size_t)D * H * H * sizeof(float)); + for (int h = 0; h < H; ++h) + for (int w = 0; w < H; ++w) + for (int c = 0; c < D; ++c) + pix_chw[(size_t)c * H * H + (size_t)h * H + w] = n2[(size_t)c + ((size_t)h * H + w) * D]; + const int N_perc = hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d; // 512 + std::vector mfeat((size_t)MD * N_perc), mpos((size_t)MD * N_perc); + if (edgetam_coreml_memencode(s_memenc, pix_chw.data(), m_hires.data(), mfeat.data(), mpos.data())) { + if (!tracker.ctx) { + struct ggml_init_params tp = {ggml_tensor_overhead() * 4096, nullptr, true}; + tracker.ctx = ggml_init(tp); + } + auto store_slot = [&](const std::vector& src) { + auto* t = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); + auto* b = ggml_backend_alloc_buffer(model.backend, (size_t)MD * N_perc * sizeof(float)); + struct ggml_tallocr a = ggml_tallocr_new(b); + ggml_tallocr_alloc(&a, t); + tracker.owned_buffers.push_back(b); + ggml_backend_tensor_set(t, src.data(), 0, (size_t)MD * N_perc * sizeof(float)); + return t; + }; + sam3_memory_slot slot; + slot.spatial_feats = store_slot(mfeat); + slot.spatial_pe = store_slot(mpos); + slot.frame_index = frame_idx; + slot.is_cond_frame = is_cond; + auto& bk = tracker.mem_banks[inst_id]; + bk.push_back(slot); + while ((int)bk.size() > sam3_mem_pool_cap(hp.num_maskmem)) { + bool removed = false; + for (auto it = bk.begin(); it != bk.end(); ++it) + if (!it->is_cond_frame) { bk.erase(it); removed = true; break; } + if (!removed) bk.erase(bk.begin() + 1); + } + return true; + } + } + } +#endif const float sig_scale = hp.sigmoid_scale(), sig_bias = hp.sigmoid_bias(); for (auto& v : m_hires) { float s = 1.0f / (1.0f + expf(-v)); v = s * sig_scale + sig_bias; } auto m_interp = sam3_bilinear_interpolate(m_hires.data(), HIGH_RES, HIGH_RES, INTERPOL, INTERPOL); From 450bb0308829e4385ab2b5731765d2aa70d28551 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Wed, 17 Jun 2026 22:55:07 -0700 Subject: [PATCH 03/14] diag(coreml): memencode parity harness + definitive Wave-1 finding (RFD 0011 U8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parity probe (SAM3_COREML_MEMENC_PARITY) cosine-compares CoreML memencode vs the ggml perceiver slot per axis. Result on dt0004: pos cos=1.000, feats cos≈0.85 on IDENTICAL pix_feat+mask (transpose verified: H/W-swap=0.43; mask not the driver). => representation mismatch between ggml perceiver and CoreML spatial_perceiver, not a layout/input bug. The ggml assembly+tpos+memattn are calibrated to the ggml representation, so the 0.85 slot breaks the track. CONCLUSION: 20fps requires a fully pure-CoreML tracker (consistent representation end-to-end), not a CoreML memencode drop-in to the ggml hybrid. memencode gate documented DO-NOT-USE. Co-Authored-By: Claude Opus 4.8 (1M context) --- sam3.cpp | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/sam3.cpp b/sam3.cpp index 85164d7..d4193f0 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -12108,13 +12108,19 @@ static bool sam3_encode_memory( auto m_hires = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES); #ifdef SAM3_COREML // RFD 0011 U8 Wave 1: CoreML memory-encode (memory_encoder + spatial_perceiver, - // ~6ms ANE vs ~24ms ggml). ⚠️ WIP — SPEED VERIFIED (7.8→10.86 fps, mem_encoder - // 24→0ms) but ACCURACY BROKEN (goldeneval dt0004 0.000, 107 lost): the slot the - // model produces here mismatches the ggml perceiver slot — a pix_feat (HWC→BCHW) - // or mask-input layout-parity bug in the seed/conditioning slot. FIX: dump this - // path's mfeat/mpos vs edgetam_perceiver_forward's perc_latents/perc_pos for one - // frame and cosine-compare to find the wrong axis. Gated OFF by default (env - // SAM3_COREML_MEMENC), so the ggml memencode below remains the correct path. + // ~6ms ANE vs ~24ms ggml). ⚠️ DO NOT USE IN THE ggml-ORCHESTRATED TRACKER. + // SPEED VERIFIED (7.8→10.86 fps, mem_encoder 24→0ms) but ACCURACY BROKEN + // (goldeneval dt0004 0.000). DIAGNOSED via the parity probe below: vs the ggml + // edgetam_perceiver_forward slot, pos cos=1.000 but **feats cos≈0.85** on + // IDENTICAL pix_feat+mask (pix_feat HWC→BCHW transpose verified — the H/W-swapped + // variant scores 0.43, far worse; mask sigmoid/scale is not the driver). So the + // gap is a genuine REPRESENTATION difference between the ggml perceiver and the + // CoreML spatial_perceiver (different magnitudes). The ggml assembly + tpos + + // mem-attn are calibrated to the ggml representation, so a 0.85-different slot + // breaks the track. CONCLUSION: 20fps needs a FULLY pure-CoreML tracker (the + // sam3_coreml_pipeline path + a real bank + ggml-seed), not this hybrid drop-in. + // Gated OFF by default (env SAM3_COREML_MEMENC); the ggml memencode below is the + // correct path for the hybrid. // The exported model applies the mask sigmoid internally // (convert_memenc_coreml.py: skip_mask_sigmoid=False), so it takes RAW logits at // HIGH_RES — i.e. m_hires BEFORE the sigmoid below. pix_feat is neck2 transposed @@ -12307,8 +12313,48 @@ static bool sam3_encode_memory( return false; } - // Store perceiver output: [MD=64, N_perc] (N_1d + N_2d latents) const int N_perc = hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d; +#ifdef SAM3_COREML + // RFD 0011 U8 Wave 1: memencode parity probe. Run the ggml path AND set + // SAM3_COREML_MEMENC_PARITY(+_MODEL) to cosine-compare the CoreML memencode + // slot against this ggml perceiver slot, per axis, to find the wrong one. + if (getenv("SAM3_COREML_MEMENC_PARITY")) { + static edgetam_coreml_handle s_pe = nullptr; + static bool s_pe_tried = false; + if (!s_pe && !s_pe_tried) { + s_pe_tried = true; + const char* mp = getenv("SAM3_COREML_MEMENC_MODEL"); + if (mp) s_pe = edgetam_coreml_create(mp, 1); + } + if (s_pe) { + auto raw = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES); + std::vector n2((size_t)D * H * H), pc((size_t)D * H * H); + ggml_backend_tensor_get(state.neck_trk[2], n2.data(), 0, (size_t)D * H * H * sizeof(float)); + for (int hh = 0; hh < H; ++hh) + for (int ww = 0; ww < H; ++ww) + for (int cc = 0; cc < D; ++cc) + pc[(size_t)cc * H * H + (size_t)hh * H + ww] = n2[(size_t)cc + ((size_t)hh * H + ww) * D]; + auto cosab = [](const std::vector& a, const std::vector& b) { + double d = 0, na = 0, nb = 0; size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) { d += (double)a[i] * b[i]; na += (double)a[i] * a[i]; nb += (double)b[i] * b[i]; } + return (na > 0 && nb > 0) ? d / (sqrt(na) * sqrt(nb)) : 0.0; + }; + std::vector mf((size_t)MD * N_perc), mp2((size_t)MD * N_perc); + edgetam_coreml_memencode(s_pe, pc.data(), raw.data(), mf.data(), mp2.data()); // current transpose (c,h,w) + // pix_feat with H/W swapped (write to (c,w,h)) — tests a spatial transpose bug + std::vector pcS((size_t)D * H * H); + for (int hh = 0; hh < H; ++hh) + for (int ww = 0; ww < H; ++ww) + for (int cc = 0; cc < D; ++cc) + pcS[(size_t)cc * H * H + (size_t)ww * H + hh] = n2[(size_t)cc + ((size_t)hh * H + ww) * D]; + std::vector mfS((size_t)MD * N_perc), mpS((size_t)MD * N_perc); + edgetam_coreml_memencode(s_pe, pcS.data(), raw.data(), mfS.data(), mpS.data()); + fprintf(stderr, "MEMENC_PARITY f%d: feats cos transpose=%.4f swapHW=%.4f pos cos=%.4f\n", + frame_idx, cosab(perc_latents, mf), cosab(perc_latents, mfS), cosab(perc_pos, mp2)); + } + } +#endif + // Store perceiver output: [MD=64, N_perc] (N_1d + N_2d latents) auto* st = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); auto* sb = ggml_backend_alloc_buffer(model.backend, MD * N_perc * sizeof(float)); struct ggml_tallocr ta_perc = ggml_tallocr_new(sb); From 908db7c9380b833c25aab94fe0a5e06a24f80c30 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Wed, 17 Jun 2026 23:09:43 -0700 Subject: [PATCH 04/14] =?UTF-8?q?feat(coreml):=20PAD=5FBANK=20=E2=86=92=20?= =?UTF-8?q?always-CoreML=20memattn=20=3D=2011.4=20fps=20@=200.94=20(RFD=20?= =?UTF-8?q?0011=20U8=20Wave=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SAM3_COREML_PAD_BANK pads the mem bank to full capacity (num_maskmem spatial + max_obj_ptrs pointer slots) from frame 1, so the CoreML mem-attn runs EVERY frame (no ggml fallback on early frames). With CoreML enc/memattn/decoder + ggml memencode (correct slots): dt0004 0.952 / dt0006 0.933 (0-2 wrong) @ 11.4 fps — a 4.4x speedup of the real tracker with accuracy HELD/IMPROVED (vs pure-ggml 2.57fps/0.938). Padding also fixed the early-frame accuracy churn (0.882→0.94). 20fps blocker: ggml memencode (23ms) — CoreML memencode is 4.6x faster but its slots are 0.85-off (model-level bug, see prior commit). Also keeps the parity probe (swapHW variant). All env-gated; default ggml path unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- sam3.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sam3.cpp b/sam3.cpp index d4193f0..09659fb 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -11611,6 +11611,29 @@ static sam3_prop_output sam3_propagate_single( if (ptr_tpos[p] < 1) ptr_tpos[p] = 1; // minimum distance of 1 } +#ifdef SAM3_COREML + // RFD 0011 U8 Wave 1: force full mem-attn capacity (num_maskmem spatial slots + + // max_obj_ptrs pointer slots = 3648 tokens) from frame 1, so the CoreML mem-attn + // engages EVERY frame instead of falling back to ggml on early (not-yet-full) + // frames. Required to test a consistent all-CoreML pipeline (CoreML memencode + // slots must not be fed to the ggml mem-attn). Pad spatial slots by repeating the + // most-recent slot; pad pointers with no_obj_ptr at a far temporal distance. + if (getenv("SAM3_COREML_PAD_BANK") && use_perceiver && !slot_feats.empty()) { + while ((int)slot_feats.size() < hp.num_maskmem) { + slot_feats.push_back(slot_feats.back()); + slot_pes.push_back(slot_pes.back()); + spatial_tpos.push_back(spatial_tpos.back()); + } + if (model.no_obj_ptr) { + std::vector nop(D); + sam3_read_f32(model.no_obj_ptr, nop.data(), D); + while ((int)obj_ptrs.size() < hp.max_obj_ptrs) { + obj_ptrs.push_back(nop); + ptr_tpos.push_back(hp.max_obj_ptrs); + } + } + } +#endif auto pd = sam3_build_prompt_and_pos(model, slot_feats, slot_pes, spatial_tpos, obj_ptrs, ptr_tpos, H); // ── RoPE frequencies (cached) ────────────────────────────────────── From bfd6f113953d8553d895a3ad418f55bab69114e2 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Thu, 18 Jun 2026 00:05:14 -0700 Subject: [PATCH 05/14] =?UTF-8?q?fix(coreml):=20memencode=20export=20scale?= =?UTF-8?q?/bias=20+=20stable=20output=20names=20=E2=86=92=204-stage=200.9?= =?UTF-8?q?4=20@=20~14.5=20fps=20(RFD=200011=20U8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ROOT CAUSE: the memencode export wrapped memory_encoder(skip_mask_sigmoid=False) — sigmoid only — but the tracker feeds sigmoid(mask)*sigmoid_scale_for_mem_enc + sigmoid_bias_for_mem_enc (EdgeTAM 20.0/-10.0). So the CoreML model saw a [0,1] mask vs the tracker's [-10,10] → garbage slot → tracker broke (goldeneval 0.000; Python proto 0.022). The earlier 0.996 "parity" compared against the same scale/bias-less wrapper, hiding it. PyTorch-vs-CoreML check confirmed. FIX: convert_memenc_coreml.py bakes sigmoid*scale+bias (skip_mask_sigmoid=True), + pins stable output names mem_feats/mem_pos (coremltools auto-names var_NNN change per export and silently broke the .mm featureValueForName lookup → ggml fallback). edgetam_coreml.mm updated to the stable names. RESULT (re-exported model): C++ 4-stage CoreML (PAD_BANK + enc/memattn/decoder/ memencode) dt0004 0.970 @ 14.1 fps, dt0006 0.927 @ 15.2 fps — up from 11.4 fps (ggml memencode), accuracy held, 0-wrong. Validated independently in the Python SAM2-predictor proto (1.000 with CoreML enc+memattn+memencode). Co-Authored-By: Claude Opus 4.8 (1M context) --- coreml/edgetam_coreml.mm | 8 +++++--- coreml/export/convert_memenc_coreml.py | 21 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/coreml/edgetam_coreml.mm b/coreml/edgetam_coreml.mm index 9343daa..98fa6ab 100644 --- a/coreml/edgetam_coreml.mm +++ b/coreml/edgetam_coreml.mm @@ -226,9 +226,11 @@ int edgetam_coreml_memencode(edgetam_coreml_handle handle, id out = [h->model predictionFromFeatures:fp error:&err]; if (err || !out) { NSLog(@"[edgetam_coreml] memencode predict: %@", err); return 0; } - // Tuple output flattened to var_486 (latents) + var_488 (pos), each [1,512,64]. - bool ok = copy_f32([[out featureValueForName:@"var_486"] multiArrayValue], mem_feats, (size_t)512*64) - && copy_f32([[out featureValueForName:@"var_488"] multiArrayValue], mem_pos, (size_t)512*64); + // Stable named outputs from convert_memenc_coreml.py: mem_feats (latents) + + // mem_pos (position), each [1,512,64]. (Was var_486/var_488 — coremltools + // auto-names change per export; the convert now pins these names.) + bool ok = copy_f32([[out featureValueForName:@"mem_feats"] multiArrayValue], mem_feats, (size_t)512*64) + && copy_f32([[out featureValueForName:@"mem_pos"] multiArrayValue], mem_pos, (size_t)512*64); return ok ? 1 : 0; } } diff --git a/coreml/export/convert_memenc_coreml.py b/coreml/export/convert_memenc_coreml.py index 1df9e20..386117e 100644 --- a/coreml/export/convert_memenc_coreml.py +++ b/coreml/export/convert_memenc_coreml.py @@ -104,13 +104,23 @@ def forward_2d(x): class MemEnc(nn.Module): """memory_encoder + spatial_perceiver, the per-frame memory-write compute (stage 4).""" - def __init__(self, mem_enc, perceiver): + def __init__(self, mem_enc, perceiver, sigmoid_scale=1.0, sigmoid_bias=0.0): super().__init__() self.mem_enc = mem_enc self.perceiver = perceiver + self.sigmoid_scale = sigmoid_scale + self.sigmoid_bias = sigmoid_bias def forward(self, pix_feat, mask_logits): - out = self.mem_enc(pix_feat, mask_logits, skip_mask_sigmoid=False) + # FIX (RFD 0011 U8): replicate _encode_new_memory's mask processing exactly — + # sigmoid(mask) * sigmoid_scale_for_mem_enc + sigmoid_bias_for_mem_enc + # (EdgeTAM: 20.0 / -10.0), THEN skip the encoder's own sigmoid. The prior + # export used skip_mask_sigmoid=False (sigmoid only, no scale/bias), feeding the + # encoder a [0,1] mask vs the tracker's [-10,10] — which broke the slot + # (goldeneval 0.000 / proto 0.022). With scale+bias baked in, the model takes + # RAW logits and matches the tracker (proto 1.000; C++ 4-stage 0.94 @ ~14fps). + mask_for_mem = torch.sigmoid(mask_logits) * self.sigmoid_scale + self.sigmoid_bias + out = self.mem_enc(pix_feat, mask_for_mem, skip_mask_sigmoid=True) feats = out["vision_features"] pos = out["vision_pos_enc"][0] feats, pos = self.perceiver(feats, pos) @@ -119,7 +129,9 @@ def forward(self, pix_feat, mask_logits): def main(): model = build_model() - w = MemEnc(model.memory_encoder, model.spatial_perceiver).eval() + w = MemEnc(model.memory_encoder, model.spatial_perceiver, + sigmoid_scale=float(model.sigmoid_scale_for_mem_enc), + sigmoid_bias=float(model.sigmoid_bias_for_mem_enc)).eval() pix_feat = torch.randn(1, C_PIX, H, W) mask = torch.randn(1, 1, 1024, 1024) @@ -157,6 +169,9 @@ def main(): frozen, inputs=[ct.TensorType(name="pix_feat", shape=pix_feat.shape, dtype=np.float32), ct.TensorType(name="mask_logits", shape=mask.shape, dtype=np.float32)], + # Stable output names so the .mm/cgo bridge doesn't depend on coremltools + # auto-naming (var_NNN), which changes per export. (feats, pos) order. + outputs=[ct.TensorType(name="mem_feats"), ct.TensorType(name="mem_pos")], minimum_deployment_target=ct.target.iOS17, compute_units=ct.ComputeUnit.ALL, convert_to="mlprogram", From f7ba6f16634b2109d166933074423cdf08ea6763 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Thu, 18 Jun 2026 00:40:14 -0700 Subject: [PATCH 06/14] =?UTF-8?q?feat(coreml):=20threaded=20encoder-ahead?= =?UTF-8?q?=20tracker=20=E2=86=92=2018.6=20fps=20@=200.94=20(RFD=200011=20?= =?UTF-8?q?U8=20Wave=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encoder-ahead pipelining: a producer thread runs preprocess + CoreML encode for frame N+1 (own MLModel handle, ANE) while the main thread runs the consumer (assembly + memattn + decode + memencode + bank, via sam3_propagate_frame) for frame N. Library hook sam3_coreml_set_prefetched_neck + sam3_coreml_preprocess_image let the consumer's encode step consume the producer's neck (skipping preprocess + encode). New bench examples/sam3_coreml_tracker_threaded.cpp. RESULT: dt0004 0.941 @ 18.58 fps, dt0006 0.882 @ 18.62 fps — up from 14.5 fps single-threaded (~1.28× overlap), accuracy held, 0-2 wrong. 7.2× over pure-ggml (2.57 fps). At the top of RUNTIME.md's measured 15-20 fps sustained band. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/CMakeLists.txt | 5 + examples/sam3_coreml_tracker_threaded.cpp | 197 ++++++++++++++++++++++ sam3.cpp | 53 ++++-- sam3.h | 12 ++ 4 files changed, 255 insertions(+), 12 deletions(-) create mode 100644 examples/sam3_coreml_tracker_threaded.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 70fb9be..29b441c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -26,6 +26,11 @@ target_compile_definitions(sam3_edgetam_bench PRIVATE SAM3_TIMING) if(SAM3_COREML) add_executable(sam3_coreml_pipeline sam3_coreml_pipeline.cpp) target_link_libraries(sam3_coreml_pipeline PRIVATE sam3) + + # RFD 0011 U8: threaded (encoder-ahead) real-bank tracker bench. + add_executable(sam3_coreml_tracker_threaded sam3_coreml_tracker_threaded.cpp) + target_link_libraries(sam3_coreml_tracker_threaded PRIVATE sam3) + target_compile_definitions(sam3_coreml_tracker_threaded PRIVATE SAM3_TIMING) endif() # SDL2 + ImGui are optional — build only if SDL2 is found. diff --git a/examples/sam3_coreml_tracker_threaded.cpp b/examples/sam3_coreml_tracker_threaded.cpp new file mode 100644 index 0000000..635c49f --- /dev/null +++ b/examples/sam3_coreml_tracker_threaded.cpp @@ -0,0 +1,197 @@ +// RFD 0011 U8 — THREADED EdgeTAM tracker bench (encoder-ahead pipelining). +// +// The real tracker (preprocess + 3648-token assembly + 4 CoreML stages + bank) is +// ~69 ms/frame single-threaded (~14.5 fps). The encoder leg (preprocess + CoreML +// encode, ~17 ms) is bank-independent, so a producer thread runs it for frame N+1 +// while the main thread runs the consumer (assembly + memattn + decode + memencode +// + bank, via sam3_propagate_frame) for frame N — overlapping the encoder to push +// toward 20 fps. The producer owns its own CoreML encoder handle (per-thread MLModel +// ownership); it hands each frame's 3 neck levels to the library via +// sam3_coreml_set_prefetched_neck, and the consumer's encode step consumes them +// (skipping preprocess + encode). Same goldeneval JSON as sam3_edgetam_bench. +// +// Run (all 4 stages on CoreML + PAD_BANK, like the single-threaded 14.5 fps config): +// SAM3_COREML_PAD_BANK=1 SAM3_COREML_ENCODER=1 SAM3_COREML_MODEL=.../encoder.mlpackage \ +// SAM3_COREML_MEMATTN=1 SAM3_COREML_MEMATTN_MODEL=.../memattn.mlpackage \ +// SAM3_COREML_DECODER=1 SAM3_COREML_DECODER_MODEL=.../decoder.mlpackage \ +// SAM3_COREML_MEMENC=1 SAM3_COREML_MEMENC_MODEL=.../memencode.mlpackage \ +// sam3_coreml_tracker_threaded --models-dir --video --n-frames 120 \ +// --prompt --dump-json +#include "sam3.h" +#include "../coreml/edgetam_coreml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +static std::string discover_edgetam_model(const std::string& dir) { +#ifndef _WIN32 + DIR* d = opendir(dir.c_str()); + if (!d) return ""; + std::string best; + for (struct dirent* e; (e = readdir(d)) != nullptr;) { + std::string f = e->d_name; + if (f.size() > 5 && f.substr(f.size() - 5) == ".ggml" && f.find("edgetam") != std::string::npos) { + best = dir + "/" + f; break; + } + } + closedir(d); + return best; +#else + return dir + "/edgetam_f16.ggml"; +#endif +} + +static bool parse_box_json(const std::string& path, sam3_box& out) { + std::ifstream f(path); if (!f) return false; + std::stringstream ss; ss << f.rdbuf(); std::string s = ss.str(); + size_t lb = s.find('[', s.find("box")); if (lb == std::string::npos) return false; + size_t rb = s.find(']', lb); if (rb == std::string::npos) return false; + std::string in = s.substr(lb + 1, rb - lb - 1); + for (char& c : in) if (c == ',') c = ' '; + std::stringstream vs(in); float v[4]; int n = 0; + while (n < 4 && (vs >> v[n])) ++n; + if (n != 4) return false; + out = {v[0], v[1], v[2], v[3]}; return true; +} + +// Bounded blocking queue of pool-slot indices (SPSC producer<->consumer). +struct IQueue { + std::queue q; std::mutex m; std::condition_variable cv; + void push(int v) { { std::lock_guard l(m); q.push(v); } cv.notify_one(); } + int pop() { std::unique_lock l(m); cv.wait(l, [&] { return !q.empty(); }); int v = q.front(); q.pop(); return v; } +}; + +int main(int argc, char** argv) { + std::string models_dir = "models/", video_path, prompt_path, dump_path; + int n_frames = 120, warmup = 35; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto nx = [&] { return (i + 1 < argc) ? argv[++i] : ""; }; + if (a == "--models-dir") models_dir = nx(); + else if (a == "--video") video_path = nx(); + else if (a == "--n-frames") n_frames = atoi(nx()); + else if (a == "--warmup") warmup = atoi(nx()); + else if (a == "--prompt") prompt_path = nx(); + else if (a == "--dump-json") dump_path = nx(); + } + const char* enc_model = getenv("SAM3_COREML_MODEL"); + if (!enc_model) { fprintf(stderr, "ERROR: SAM3_COREML_MODEL (encoder .mlpackage) required\n"); return 1; } + + std::string model_path = discover_edgetam_model(models_dir); + sam3_params params; params.model_path = model_path; params.use_gpu = true; + auto model = sam3_load_model(params); + if (!model) { fprintf(stderr, "ERROR: load model %s\n", model_path.c_str()); return 1; } + auto state = sam3_create_state(*model, params); + sam3_visual_track_params vtp; vtp.max_keep_alive = 1000; vtp.recondition_every = 16; + auto tracker = sam3_create_visual_tracker(*model, vtp); + + auto vinfo = sam3_get_video_info(video_path); + if (vinfo.n_frames <= 0) { fprintf(stderr, "ERROR: read video %s\n", video_path.c_str()); return 1; } + if (n_frames > vinfo.n_frames) n_frames = vinfo.n_frames; + sam3_box init_box; + if (prompt_path.empty() || !parse_box_json(prompt_path, init_box)) { + fprintf(stderr, "ERROR: --prompt box.json required\n"); return 1; + } + + std::vector frames(n_frames); + for (int f = 0; f < n_frames; ++f) { + frames[f] = sam3_decode_video_frame(video_path, f); + if (frames[f].data.empty()) { fprintf(stderr, "ERROR: decode frame %d\n", f); return 1; } + } + const float fw = (float)frames[0].width, fh = (float)frames[0].height; + const int IMG = 1024; // EdgeTAM encoder input size + + // Producer's own encoder handle (ANE) — per-thread MLModel ownership. + auto prod_enc = edgetam_coreml_create(enc_model, /*ANE*/ 1); + if (!prod_enc) { fprintf(stderr, "ERROR: producer encoder load %s\n", enc_model); return 1; } + + // Neck pool (channels-last [1,H,W,256]); POOL slots let the producer run ahead. + const int POOL = 3, D = 256, Wd[3] = {256, 128, 64}; + struct Slot { std::vector n[3]; int frame; }; + std::vector pool(POOL); + for (auto& s : pool) for (int i = 0; i < 3; ++i) s.n[i].assign((size_t)D * Wd[i] * Wd[i], 0.f); + IQueue freeq, readyq; + for (int i = 0; i < POOL; ++i) freeq.push(i); + + // Producer: preprocess + CoreML encode every frame (0..N-1), in order. + std::thread producer([&] { + for (int f = 0; f < n_frames; ++f) { + int slot = freeq.pop(); + std::vector img = sam3_coreml_preprocess_image(frames[f], IMG); + edgetam_coreml_encode(prod_enc, img.data(), + pool[slot].n[0].data(), pool[slot].n[1].data(), pool[slot].n[2].data()); + pool[slot].frame = f; + readyq.push(slot); + } + }); + + // Consumer (main): seed on frame 0, then propagate 1..N-1, all using prefetched necks. + struct Rec { int frame; bool tracked; std::string state; float b[4]; float conf; }; + std::vector recs; recs.reserve(n_frames); + + int slot0 = readyq.pop(); // frame 0 + sam3_coreml_set_prefetched_neck(pool[slot0].n[0].data(), pool[slot0].n[1].data(), pool[slot0].n[2].data()); + sam3_encode_image(*state, *model, frames[0]); + (void)sam3_take_frame_timing(); + sam3_pvs_params pvs; pvs.box = init_box; pvs.use_box = true; + int inst = sam3_tracker_add_instance(*tracker, *state, *model, pvs); + if (inst < 0) { fprintf(stderr, "ERROR: seed add_instance failed\n"); return 1; } + (void)sam3_take_frame_timing(); + freeq.push(slot0); + recs.push_back({0, true, "tracked", {init_box.x0 / fw, init_box.y0 / fh, init_box.x1 / fw, init_box.y1 / fh}, 1.f}); + + auto wall0 = std::chrono::high_resolution_clock::now(); + for (int f = 1; f < n_frames; ++f) { + int slot = readyq.pop(); // frame f (producer delivers in order) + sam3_coreml_set_prefetched_neck(pool[slot].n[0].data(), pool[slot].n[1].data(), pool[slot].n[2].data()); + sam3_result res = sam3_propagate_frame(*tracker, *state, *model, frames[f]); + (void)sam3_take_frame_timing(); + Rec r{f, false, "lost", {0, 0, 0, 0}, 0.f}; + if (!res.detections.empty()) { + const sam3_detection* d = &res.detections[0]; + for (const auto& x : res.detections) if (x.instance_id == inst) { d = &x; break; } + r.tracked = true; r.state = sam3_target_state_name(d->state); + r.b[0] = d->box.x0 / fw; r.b[1] = d->box.y0 / fh; r.b[2] = d->box.x1 / fw; r.b[3] = d->box.y1 / fh; + r.conf = d->score; + } + recs.push_back(r); + freeq.push(slot); + } + auto wall1 = std::chrono::high_resolution_clock::now(); + producer.join(); + double wall_ms = std::chrono::duration(wall1 - wall0).count(); + double wall_fps = (n_frames - 1) * 1000.0 / wall_ms; + + printf("\nTHREADED tracker: %.2f fps (%.1f ms over %d hold frames, encoder-ahead)\n", + wall_fps, wall_ms, n_frames - 1); + + if (!dump_path.empty()) { + std::ofstream o(dump_path); + o << "{\"preds\":{"; + for (size_t i = 0; i < recs.size(); ++i) { + const Rec& r = recs[i]; + o << (i ? "," : "") << "\"" << r.frame << "\":{\"bbox\":[" << r.b[0] << "," << r.b[1] + << "," << r.b[2] << "," << r.b[3] << "],\"state\":\"" << r.state << "\"}"; + } + o << "}}"; + fprintf(stderr, "wrote %s\n", dump_path.c_str()); + } + edgetam_coreml_destroy(prod_enc); + return 0; +} diff --git a/sam3.cpp b/sam3.cpp index 09659fb..ee0562a 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -4974,6 +4974,27 @@ static void edgetam_build_repvit_graph(struct ggml_context* ctx, #ifdef SAM3_COREML #include "edgetam_coreml.h" + +// RFD 0011 U8 (threaded encoder-ahead): the producer thread runs preprocess + +// CoreML encode for frame N+1 into s_prefetch_neck while the main thread runs the +// consumer (assembly+memattn+decode+memencode+bank) for frame N. When a prefetched +// neck is ready, the main thread's encode step copies it (skipping BOTH preprocess +// and the CoreML encode), so the ~17 ms encoder leg overlaps the ~52 ms consumer. +// SPSC: the bench serializes set→consume per frame, so no lock is needed here. +static std::vector s_prefetch_neck[3]; +static bool s_prefetch_ready = false; +void sam3_coreml_set_prefetched_neck(const float* n0, const float* n1, const float* n2) { + const int Wd[3] = {256, 128, 64}, D = 256; + const float* src[3] = {n0, n1, n2}; + for (int i = 0; i < 3; ++i) + s_prefetch_neck[i].assign(src[i], src[i] + (size_t)D * Wd[i] * Wd[i]); + s_prefetch_ready = true; +} +// Exposed for the producer thread (preprocess off the main thread). +std::vector sam3_coreml_preprocess_image(const sam3_image& image, int img_size) { + return sam2_preprocess_image(image, img_size); +} + // RFD 0011 (CoreML/ANE hybrid): run the EdgeTAM encoder on CoreML/ANE (~12 ms) // instead of the ggml/Metal RepViT (~122 ms) and load the ggml-matching 256-ch // neck features into state.neck_trk. The CoreML model exports neck(trunk(x))[0:3] @@ -4984,7 +5005,7 @@ static bool edgetam_encode_image_coreml(sam3_state& state, const sam3_model& mod const sam3_image& image, const std::vector& img_norm) { static edgetam_coreml_handle s_enc = nullptr; - if (!s_enc) { + if (!s_prefetch_ready && !s_enc) { // prefetch path needs no local encoder handle const char* mp = getenv("SAM3_COREML_MODEL"); if (!mp) { fprintf(stderr, "%s: SAM3_COREML_ENCODER set but SAM3_COREML_MODEL unset\n", __func__); return false; } s_enc = edgetam_coreml_create(mp, /*compute_units=ANE*/ 1); @@ -5003,7 +5024,10 @@ static bool edgetam_encode_image_coreml(sam3_state& state, const sam3_model& mod static std::vector nbuf[3]; for (int i = 0; i < 3; ++i) nbuf[i].resize((size_t)D * Wd[i] * Hd[i]); - { + if (s_prefetch_ready) { // threaded: producer already encoded this frame off-thread + for (int i = 0; i < 3; ++i) nbuf[i].swap(s_prefetch_neck[i]); + s_prefetch_ready = false; + } else { SAM3_TIME_SCOPE(image_encoder_compute_ms); // the ANE inference itself if (!edgetam_coreml_encode(s_enc, img_norm.data(), nbuf[0].data(), nbuf[1].data(), nbuf[2].data())) @@ -5066,6 +5090,21 @@ static bool edgetam_encode_image(sam3_state& state, state.orig_width = image.width; state.orig_height = image.height; +#ifdef SAM3_COREML + // RFD 0011 (CoreML/ANE hybrid): when enabled, run the encoder on CoreML/ANE + // and skip the ggml RepViT+FPN graph entirely. Threaded (encoder-ahead): when a + // prefetched neck is ready the producer already did preprocess + encode off-thread, + // so skip preprocess here too — the helper consumes the prefetched neck. + if (getenv("SAM3_COREML_ENCODER")) { + std::vector cml_img; + if (!s_prefetch_ready) { + SAM3_TIME_SCOPE(preprocess_ms); + cml_img = sam2_preprocess_image(image, img_size); // NCHW [1,3,1024,1024] + } + return edgetam_encode_image_coreml(state, model, image, cml_img); + } +#endif + // ── Preprocess (same ImageNet normalization as SAM2) ───────────────── // RFD 0011 U0: preprocess timed separately from the graph build/alloc/compute. std::vector img_data; @@ -5074,16 +5113,6 @@ static bool edgetam_encode_image(sam3_state& state, img_data = sam2_preprocess_image(image, img_size); } -#ifdef SAM3_COREML - // RFD 0011 (CoreML/ANE hybrid): when enabled, run the encoder on CoreML/ANE - // and skip the ggml RepViT+FPN graph entirely. img_data is already NCHW - // [1,3,1024,1024] (ggml inp ne=[W,H,C] == numpy NCHW), so it hands straight - // to CoreML; only the 256-ch neck outputs need a layout permute (in helper). - if (getenv("SAM3_COREML_ENCODER")) { - return edgetam_encode_image_coreml(state, model, image, img_data); - } -#endif - // ── Build graph ────────────────────────────────────────────────────── // RFD 0011 U0: graph CONSTRUCTION region (image_encoder_build_ms). U1 will // build this graph once and reuse it, driving build_ms toward 0; that is diff --git a/sam3.h b/sam3.h index 79c33e5..809d4de 100644 --- a/sam3.h +++ b/sam3.h @@ -359,6 +359,18 @@ bool sam3_save_mask(const sam3_mask & mask, const std::string & path) sam3_image sam3_decode_video_frame(const std::string & video_path, int frame_index); sam3_video_info sam3_get_video_info(const std::string & video_path); +/* +** ── RFD 0011 U8: threaded encoder-ahead (CoreML) ───────────────────────── +** The producer thread preprocesses + CoreML-encodes frame N+1 off the main +** thread, then hands the 3 neck levels here; the next sam3_propagate_frame's +** encode step consumes them (skipping preprocess + encode) so the encoder leg +** overlaps the consumer. Built only with SAM3_COREML; no-ops otherwise. +*/ +void sam3_coreml_set_prefetched_neck(const float* neck0, + const float* neck1, + const float* neck2); +std::vector sam3_coreml_preprocess_image(const sam3_image& image, int img_size); + /***************************************************************************** ** Test and Debug API ** From 2ab201f359387c313a564ea961ac656319e38bca Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Thu, 18 Jun 2026 01:06:57 -0700 Subject: [PATCH 07/14] feat(coreml): enable MEMENC + PAD_BANK stages in edgetam_capi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C-ABI used by platform's in-process cgo tracker (RFD 0011 U8) set only ENCODER/MEMATTN/DECODER. Without MEMENC + PAD_BANK the CoreML mem-attention model (static 3648-token input) can't match the live bank and silently falls back to the ~184ms ggml mem-attention path — correct but ~8x slower. Setting both brings the capi to the full validated config: encoder ANE + memenc ANE + mem-attn GPU + decoder ANE. Measured via platform's cgo hold runtime test: 570ms -> 70ms/frame (dt0004, 29/29 held, 14.3 fps single-thread). Co-Authored-By: Claude Opus 4.8 (1M context) --- coreml/edgetam_capi.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/coreml/edgetam_capi.cpp b/coreml/edgetam_capi.cpp index 11afada..295bb6e 100644 --- a/coreml/edgetam_capi.cpp +++ b/coreml/edgetam_capi.cpp @@ -46,6 +46,14 @@ extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, setenv("SAM3_COREML_MEMATTN_MODEL", (d + "/edgetam_memory_attention.mlpackage").c_str(), 1); setenv("SAM3_COREML_DECODER", "1", 1); setenv("SAM3_COREML_DECODER_MODEL", (d + "/edgetam_mask_decoder_nhwc.mlpackage").c_str(), 1); + // Memory-encode on ANE + a fixed-size (padded) memory bank. The CoreML + // mem-attention model has a static 3648-token input (7*512 + 16*4); PAD_BANK + // pads the live bank to that shape so MEMATTN actually runs on the ANE/GPU + // instead of silently falling back to the ~184ms ggml path. Without these two + // the tracker is correct but ~8x slower (measured 570ms vs 69ms/frame). + setenv("SAM3_COREML_MEMENC", "1", 1); + setenv("SAM3_COREML_MEMENC_MODEL", (d + "/edgetam_memory_encode.mlpackage").c_str(), 1); + setenv("SAM3_COREML_PAD_BANK", "1", 1); } auto t = std::make_unique(); sam3_params p; From c5d8dfec564ae65b2817a37969fd33a29729ed75 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:04:09 -0700 Subject: [PATCH 08/14] perf(coreml): drop rope_k_data from the CoreML mem-attn hot path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rope_k_data feeds only the ggml mem-attn fallback graph; the CoreML mem-attn path uses cached_sinpe_256 and never reads it, so building it every frame was ~2ms of pure waste. Moved its construction to just before the ggml graph build (reached only when CoreML did not return). Behavior-preserving: dt0004 goldeneval held 0.979 tracked_fraction, 0 wrong. Honest note: the ~2ms saving is within thermal run-to-run variance — this is a cleanup that documents the finding, NOT a path to 20fps. Per-frame cost is dominated by the CoreML stage compute (mem-attn ~17-19ms GPU + encoder ~11ms + decoder ~8ms + memencode ~5ms) plus preprocess (~5-7ms) and the irreducible host-side memory-bank assembly; trimming CPU glue cannot close the gap to 20. Co-Authored-By: Claude Opus 4.8 (1M context) --- sam3.cpp | 63 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/sam3.cpp b/sam3.cpp index ee0562a..82fbb57 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -11669,36 +11669,11 @@ static sam3_prop_output sam3_propagate_single( sam3_ensure_tracker_pe_caches(tracker, hp, H); const int half_d = D / 2; // 128 const auto& rope_q_reord = tracker.cached_axial_cis_reord; - // For cross-attn K: build rope_k_data for all M_spatial tokens + // RFD 0011 U8 perf: rope_k_data feeds ONLY the ggml mem-attn fallback graph. + // The CoreML mem-attn path (below) uses cached_sinpe_256, never rope_k_data, + // so building it here is ~2 ms/frame of pure waste on the hot path. Moved to + // just before the ggml graph build, so it runs only when CoreML did NOT. std::vector rope_k_data; - if (pd.M_spatial > 0) { - rope_k_data.resize(2 * half_d * pd.M_spatial); - if (use_perceiver) { - // EdgeTAM perceiver: each frame has N_per_slot=512 tokens. - // First 256 (1D latents): identity RoPE (cos=1, sin=0). - // Last 256 (2D latents): 16x16 RoPE from cached_axial_cis_k16_reord. - const int N_1d = hp.perceiver_n_latents_1d; // 256 - const int N_2d = hp.perceiver_n_latents_2d; // 256 - const auto& rope_k16 = tracker.cached_axial_cis_k16_reord; // [2, 128, 256] - for (int s = 0; s < n_sel; ++s) { - float* dst = rope_k_data.data() + s * D * N_per_slot; - // 1D tokens: identity RoPE (cos=1, sin=0) in [2, half_d, N_1d] layout - // Layout: for token n, dim i: cos at [0 + i*2 + n*D], sin at [1 + i*2 + n*D] - for (int n = 0; n < N_1d; ++n) - for (int i = 0; i < half_d; ++i) { - dst[0 + i * 2 + n * D] = 1.0f; // cos = 1 - dst[1 + i * 2 + n * D] = 0.0f; // sin = 0 - } - // 2D tokens: copy 16x16 RoPE - float* dst_2d = dst + D * N_1d; - memcpy(dst_2d, rope_k16.data(), D * N_2d * sizeof(float)); - } - } else { - // Standard: repeat HxH axial CIS for each memory frame - for (int s = 0; s < pd.M_spatial / N; ++s) - memcpy(rope_k_data.data() + s * D * N, rope_q_reord.data(), D * N * sizeof(float)); - } - } // RFD 0011 (CoreML/ANE hybrid): run the memory attention on CoreML-GPU // (~18 ms vs ggml ~185 ms) at the fixed steady-state capacity (7 memory @@ -11783,6 +11758,36 @@ static sam3_prop_output sam3_propagate_single( } #endif + // RFD 0011 U8 perf: build rope_k_data HERE — only reached when the CoreML + // mem-attn/decoder path above did NOT return (fallback to the ggml graph). + if (pd.M_spatial > 0) { + rope_k_data.resize(2 * half_d * pd.M_spatial); + if (use_perceiver) { + // EdgeTAM perceiver: each frame has N_per_slot=512 tokens. + // First 256 (1D latents): identity RoPE (cos=1, sin=0). + // Last 256 (2D latents): 16x16 RoPE from cached_axial_cis_k16_reord. + const int N_1d = hp.perceiver_n_latents_1d; // 256 + const int N_2d = hp.perceiver_n_latents_2d; // 256 + const auto& rope_k16 = tracker.cached_axial_cis_k16_reord; // [2, 128, 256] + for (int s = 0; s < n_sel; ++s) { + float* dst = rope_k_data.data() + s * D * N_per_slot; + // 1D tokens: identity RoPE (cos=1, sin=0) in [2, half_d, N_1d] layout + for (int n = 0; n < N_1d; ++n) + for (int i = 0; i < half_d; ++i) { + dst[0 + i * 2 + n * D] = 1.0f; // cos = 1 + dst[1 + i * 2 + n * D] = 0.0f; // sin = 0 + } + // 2D tokens: copy 16x16 RoPE + float* dst_2d = dst + D * N_1d; + memcpy(dst_2d, rope_k16.data(), D * N_2d * sizeof(float)); + } + } else { + // Standard: repeat HxH axial CIS for each memory frame + for (int s = 0; s < pd.M_spatial / N; ++s) + memcpy(rope_k_data.data() + s * D * N, rope_q_reord.data(), D * N * sizeof(float)); + } + } + // ── Build graph ───────────────────────────────────────────────────── // RFD 0011 U0: mem-attn + SAM mask decoder graph CONSTRUCTION region // (mem_attn_build_ms). NOTE: mem-attn and the mask decoder are built into From 2448b098658287451661a961b677003c59868324 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:22:08 -0700 Subject: [PATCH 09/14] feat(coreml): capi encoder-ahead split (encode_slot/track_slot) for cgo threading RFD 0011 U8 Wave 5. Producer encoder handle + a 3-slot neck pool; Go coordinates slot free/ready via channels (one writer then one reader per slot, no lock). encode_slot (producer thread) preprocesses+encodes into a slot; track_slot (consumer thread) consumes the prefetched neck and propagates. Mirrors examples/sam3_coreml_tracker_threaded.cpp through the C-ABI. Existing create/seed/track/reset/destroy unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- coreml/edgetam_capi.cpp | 81 ++++++++++++++++++++++++++++++++++++++++- coreml/edgetam_capi.h | 10 +++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/coreml/edgetam_capi.cpp b/coreml/edgetam_capi.cpp index 295bb6e..3949aa5 100644 --- a/coreml/edgetam_capi.cpp +++ b/coreml/edgetam_capi.cpp @@ -4,6 +4,7 @@ // exceptions are caught at the boundary (cgo cannot unwind through Go). #include "edgetam_capi.h" #include "../sam3.h" +#include "edgetam_coreml.h" // producer encoder handle (edgetam_coreml_create/encode/destroy) #include #include @@ -17,6 +18,16 @@ struct EtTracker { std::shared_ptr model; sam3_state_ptr state; sam3_tracker_ptr tracker; + + // RFD 0011 U8 Wave 5 (encoder-ahead threading): a producer encoder handle + // (its OWN MLModel, per-thread ownership) + a fixed neck pool. Go owns slot + // free/ready coordination via channels, so NO lock here — each slot is written + // once by encode_slot (producer thread) then read once by track_slot (consumer + // thread). track_slot is the ONLY caller of sam3_coreml_set_prefetched_neck + // (global, consumer-thread-only). prod_enc is non-null only when CoreML is on. + edgetam_coreml_handle prod_enc = nullptr; + static const int POOL = 3; + std::vector neck[POOL][3]; // n0 256*256*256, n1 128*128*256, n2 64*64*256 }; // Wrap a caller RGB24 buffer as a sam3_image (copies into the owned vector — the @@ -64,6 +75,15 @@ extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, t->state = sam3_create_state(*t->model, p); sam3_visual_track_params vp; // defaults match sam3_edgetam_bench t->tracker = sam3_create_visual_tracker(*t->model, vp); + // Wave 5: producer encoder handle + neck pool (only when CoreML is on). + if (models_dir && *models_dir) { + std::string d = models_dir; + t->prod_enc = edgetam_coreml_create((d + "/edgetam_encoder_neck_nhwc.mlpackage").c_str(), /*ANE*/ 1); + const int Wd[3] = {256, 128, 64}; + for (int s = 0; s < EtTracker::POOL; ++s) + for (int i = 0; i < 3; ++i) + t->neck[s][i].assign((size_t)256 * Wd[i] * Wd[i], 0.f); + } return t.release(); } catch (...) { return nullptr; @@ -117,7 +137,66 @@ extern "C" void edgetam_capi_reset(edgetam_tracker_t h) { } extern "C" void edgetam_capi_destroy(edgetam_tracker_t h) { - delete static_cast(h); + auto* t = static_cast(h); + if (t && t->prod_enc) edgetam_coreml_destroy(t->prod_enc); + delete t; +} + +// ── RFD 0011 U8 Wave 5: encoder-ahead split ────────────────────────────────── +// Go drives the pipeline: a producer goroutine pops a free slot, calls +// encode_slot; a consumer goroutine calls track_slot then recycles the slot. The +// two run on different OS threads (per-thread MLModel ownership): encode_slot uses +// the producer encoder handle; track_slot consumes the prefetched neck and never +// encodes. Go's channels serialize each slot (write-then-read), so no lock here. + +extern "C" int edgetam_capi_pool_size(edgetam_tracker_t h) { + auto* t = static_cast(h); + return (t && t->prod_enc) ? EtTracker::POOL : 0; // 0 => threading unavailable +} + +extern "C" int edgetam_capi_encode_slot(edgetam_tracker_t h, int slot, + const uint8_t* rgb, int w, int hgt) { + auto* t = static_cast(h); + if (!t || !t->prod_enc || !rgb || slot < 0 || slot >= EtTracker::POOL) return 0; + try { + sam3_image img = make_image(rgb, w, hgt); + std::vector norm = sam3_coreml_preprocess_image(img, 1024); + return edgetam_coreml_encode(t->prod_enc, norm.data(), + t->neck[slot][0].data(), + t->neck[slot][1].data(), + t->neck[slot][2].data()) ? 1 : 0; + } catch (...) { + return 0; + } +} + +extern "C" et_result edgetam_capi_track_slot(edgetam_tracker_t h, int slot, + const uint8_t* rgb, int w, int hgt) { + et_result r; + std::memset(&r, 0, sizeof(r)); + auto* t = static_cast(h); + if (!t || !rgb || slot < 0 || slot >= EtTracker::POOL) return r; + try { + // Consumer thread only: hand the producer-encoded neck to the library so + // sam3_propagate_frame's encode step is skipped (no ggml/CoreML encode here). + sam3_coreml_set_prefetched_neck(t->neck[slot][0].data(), + t->neck[slot][1].data(), + t->neck[slot][2].data()); + sam3_image img = make_image(rgb, w, hgt); + sam3_result res = sam3_propagate_frame(*t->tracker, *t->state, *t->model, img); + if (res.detections.empty()) return r; + const sam3_detection& d = res.detections[0]; + const float W = (float)w, H = (float)hgt; + r.box.x0 = d.box.x0 / W; r.box.y0 = d.box.y0 / H; + r.box.x1 = d.box.x1 / W; r.box.y1 = d.box.y1 / H; + r.valid = 1; + r.obj_score = d.score; + r.mask_iou = d.iou_score; + r.state = (int)d.state; + return r; + } catch (...) { + return r; + } } extern "C" const char* edgetam_capi_version(void) { diff --git a/coreml/edgetam_capi.h b/coreml/edgetam_capi.h index 886c9da..a65af0f 100644 --- a/coreml/edgetam_capi.h +++ b/coreml/edgetam_capi.h @@ -44,6 +44,16 @@ int edgetam_capi_seed(edgetam_tracker_t t, const uint8_t* rgb, int w, int h, et_ // (box normalized [0,1]); valid==0 when the target is lost this frame. et_result edgetam_capi_track(edgetam_tracker_t t, const uint8_t* rgb, int w, int h); +// RFD 0011 U8 Wave 5 — encoder-ahead threading split. pool_size returns the slot +// count (0 if CoreML/threading is unavailable). encode_slot (PRODUCER thread) +// preprocesses+encodes rgb into slot's neck buffers via the producer's own encoder +// handle. track_slot (CONSUMER thread) consumes slot's prefetched neck + propagates. +// Go owns slot free/ready coordination (channels) — one writer then one reader per +// slot, so these are safe to overlap across the two threads on DIFFERENT slots. +int edgetam_capi_pool_size(edgetam_tracker_t t); +int edgetam_capi_encode_slot(edgetam_tracker_t t, int slot, const uint8_t* rgb, int w, int h); +et_result edgetam_capi_track_slot(edgetam_tracker_t t, int slot, const uint8_t* rgb, int w, int h); + void edgetam_capi_reset(edgetam_tracker_t t); void edgetam_capi_destroy(edgetam_tracker_t t); const char* edgetam_capi_version(void); // SAM3_VERSION, for cgo link sanity From 0df3d644a2b3e73271591ceb0db97ad3f8fa7e5c Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:25:45 -0700 Subject: [PATCH 10/14] =?UTF-8?q?test(coreml):=20capi=20encoder-ahead=20sm?= =?UTF-8?q?oke=20test=20=E2=86=92=2018.55=20fps=20@=200.979=20(RFD=200011?= =?UTF-8?q?=20U8=20Wave=205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drives edgetam_capi_encode_slot/track_slot via a producer/consumer thread pair — the exact C-ABI path cgo will use — on dt0004: 18.55 fps (vs 14.3 single-thread), goldeneval tracked_fraction 0.979, 0 wrong. De-risks the Go wiring: the capi split reproduces the standalone threaded bench's speedup with accuracy intact. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/CMakeLists.txt | 4 + examples/sam3_coreml_capi_threaded.cpp | 136 +++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 examples/sam3_coreml_capi_threaded.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 29b441c..cd8e9f1 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -31,6 +31,10 @@ if(SAM3_COREML) add_executable(sam3_coreml_tracker_threaded sam3_coreml_tracker_threaded.cpp) target_link_libraries(sam3_coreml_tracker_threaded PRIVATE sam3) target_compile_definitions(sam3_coreml_tracker_threaded PRIVATE SAM3_TIMING) + + # RFD 0011 U8 Wave 5: encoder-ahead via the cgo C-ABI (capi encode_slot/track_slot). + add_executable(sam3_coreml_capi_threaded sam3_coreml_capi_threaded.cpp) + target_link_libraries(sam3_coreml_capi_threaded PRIVATE sam3) endif() # SDL2 + ImGui are optional — build only if SDL2 is found. diff --git a/examples/sam3_coreml_capi_threaded.cpp b/examples/sam3_coreml_capi_threaded.cpp new file mode 100644 index 0000000..18e3d2a --- /dev/null +++ b/examples/sam3_coreml_capi_threaded.cpp @@ -0,0 +1,136 @@ +// RFD 0011 U8 Wave 5 — verify the capi encoder-ahead split end-to-end through the +// SAME C-ABI cgo will call (edgetam_capi_create/seed/encode_slot/track_slot), so a +// green fps + goldeneval here de-risks the Go wiring. Producer thread: encode_slot; +// consumer (main): track_slot. Same {"preds":{...}} JSON as sam3_edgetam_bench. +// +// Run (CoreML dir holds the 4 .mlpackages; ggml model is the seed checkpoint): +// sam3_coreml_capi_threaded --models-dir-coreml --ggml-model \ +// --video --prompt --n-frames 120 --dump-json +#include "../coreml/edgetam_capi.h" +#include "sam3.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool parse_box_json(const std::string& path, et_box& out) { + std::ifstream f(path); if (!f) return false; + std::stringstream ss; ss << f.rdbuf(); std::string s = ss.str(); + size_t lb = s.find('[', s.find("box")); if (lb == std::string::npos) return false; + size_t rb = s.find(']', lb); if (rb == std::string::npos) return false; + std::string in = s.substr(lb + 1, rb - lb - 1); + for (char& c : in) if (c == ',') c = ' '; + std::stringstream vs(in); float v[4]; int n = 0; + while (n < 4 && (vs >> v[n])) ++n; + if (n != 4) return false; + out = {v[0], v[1], v[2], v[3]}; return true; +} + +static const char* state_name(int s) { + switch (s) { case 0: return "tracked"; case 1: return "at_risk"; case 2: return "occluded"; + case 3: return "lost"; case 4: return "candidate_reacquire"; case 5: return "reacquired"; + default: return "lost"; } +} + +// Bounded blocking queue of pool-slot ids (SPSC producer<->consumer). +struct IQueue { + std::queue q; std::mutex m; std::condition_variable cv; + void push(int v) { { std::lock_guard l(m); q.push(v); } cv.notify_one(); } + int pop() { std::unique_lock l(m); cv.wait(l, [&] { return !q.empty(); }); int v = q.front(); q.pop(); return v; } +}; + +int main(int argc, char** argv) { + std::string coreml_dir, ggml_model = "models/edgetam_f16.ggml", video, prompt, dump; + int n_frames = 120; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto nx = [&] { return (i + 1 < argc) ? argv[++i] : ""; }; + if (a == "--models-dir-coreml") coreml_dir = nx(); + else if (a == "--ggml-model") ggml_model = nx(); + else if (a == "--video") video = nx(); + else if (a == "--prompt") prompt = nx(); + else if (a == "--n-frames") n_frames = atoi(nx()); + else if (a == "--dump-json") dump = nx(); + } + if (coreml_dir.empty() || video.empty() || prompt.empty()) { + fprintf(stderr, "need --models-dir-coreml --video --prompt\n"); return 1; + } + + edgetam_tracker_t h = edgetam_capi_create(coreml_dir.c_str(), ggml_model.c_str(), 1, 1); + if (!h) { fprintf(stderr, "ERROR: edgetam_capi_create failed\n"); return 1; } + const int POOL = edgetam_capi_pool_size(h); + if (POOL <= 0) { fprintf(stderr, "ERROR: threading unavailable (CoreML off?)\n"); return 1; } + + auto vinfo = sam3_get_video_info(video); + if (vinfo.n_frames <= 0) { fprintf(stderr, "ERROR: read video\n"); return 1; } + if (n_frames > vinfo.n_frames) n_frames = vinfo.n_frames; + et_box init_box; + if (!parse_box_json(prompt, init_box)) { fprintf(stderr, "ERROR: --prompt box.json\n"); return 1; } + + std::vector frames(n_frames); + for (int f = 0; f < n_frames; ++f) { + frames[f] = sam3_decode_video_frame(video, f); + if (frames[f].data.empty()) { fprintf(stderr, "ERROR: decode frame %d\n", f); return 1; } + } + const int w = frames[0].width, hgt = frames[0].height; + const float fw = (float)w, fh = (float)hgt; + + if (!edgetam_capi_seed(h, frames[0].data.data(), w, hgt, init_box)) { + fprintf(stderr, "ERROR: seed failed\n"); return 1; + } + + struct Rec { int frame; std::string state; float b[4]; }; + std::vector recs; recs.reserve(n_frames); + recs.push_back({0, "tracked", {init_box.x0 / fw, init_box.y0 / fh, init_box.x1 / fw, init_box.y1 / fh}}); + + IQueue freeq, readyq; + for (int i = 0; i < POOL; ++i) freeq.push(i); + + // Producer: encode frames 1..N-1 in order into free slots. + std::thread producer([&] { + for (int f = 1; f < n_frames; ++f) { + int slot = freeq.pop(); + edgetam_capi_encode_slot(h, slot, frames[f].data.data(), w, hgt); + readyq.push(slot); + } + }); + + // Consumer (main): track each frame using its prefetched neck. + auto t0 = std::chrono::high_resolution_clock::now(); + for (int f = 1; f < n_frames; ++f) { + int slot = readyq.pop(); + et_result r = edgetam_capi_track_slot(h, slot, frames[f].data.data(), w, hgt); + Rec rec{f, r.valid ? state_name(r.state) : "lost", + {r.box.x0, r.box.y0, r.box.x1, r.box.y1}}; + recs.push_back(rec); + freeq.push(slot); + } + auto t1 = std::chrono::high_resolution_clock::now(); + producer.join(); + double wall_ms = std::chrono::duration(t1 - t0).count(); + printf("\nCAPI THREADED: %.2f fps (%.1f ms over %d hold frames, encoder-ahead via capi)\n", + (n_frames - 1) * 1000.0 / wall_ms, wall_ms, n_frames - 1); + + if (!dump.empty()) { + std::ofstream o(dump); + o << "{\"preds\":{"; + for (size_t i = 0; i < recs.size(); ++i) { + const Rec& r = recs[i]; + o << (i ? "," : "") << "\"" << r.frame << "\":{\"bbox\":[" << r.b[0] << "," << r.b[1] + << "," << r.b[2] << "," << r.b[3] << "],\"state\":\"" << r.state << "\"}"; + } + o << "}}"; + fprintf(stderr, "wrote %s\n", dump.c_str()); + } + edgetam_capi_destroy(h); + return 0; +} From 3c2ee06eafc8d762c123da3e1acb7888b3a09d21 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:12:23 -0400 Subject: [PATCH 11/14] perf(coreml): per-chip compute-unit profile (env-tunable) + M4 sweep tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFD 0011 U8. Applies Egor's METHODOLOGY (per-module compute-unit sweep) to the M4 instead of chasing his M1 result. sam3_coreml_pipeline now parameterizes all 4 stage units (SAM3_{ENC,MA,DEC,MENC}_UNIT_N); the real tracker reads SAM3_COREML_{ENC,MA,DEC,MENC}_UNIT for per-chip tuning. M4 Pro sweep (isolated 20-rep median, 0=ALL 1=ANE 2=GPU 3=CPU): encoder ANE 11.0 | GPU 14.3 | CPU 37.7 | ALL 17.6 -> ANE mem-attn ANE 26.3 | GPU 17.2 | CPU 24.0 | ALL 33.3 -> GPU decoder ANE 7.8 | GPU 8.2 | CPU 13.9 | ALL 8.9 -> ANE memenc ANE 5.0 | GPU 3.9 | CPU 9.9 | ALL 6.7 -> GPU (isolated) Finding: our M1-era placement (enc ANE, mem-attn GPU, dec ANE, memenc ANE) is ALREADY M4-optimal — NOT inverted like the ORT/EfficientTAM stack (where the encoder flips to GPU), because our RepViT encoder is conv-heavy/ANE-friendly and runs as a standalone .mlpackage (no ORT graph partitioning, no -14). memenc's isolated GPU edge (3.9 vs 5.0) does NOT survive thermal noise end-to-end (A/B: GPU 20.5 vs ANE 19.7 at 200 frames, but GPU 13.7 vs ANE 15.3 at 300 frames — thermal throttling dominates), accuracy identical (0 wrong), so the default stays ANE. Knobs let a thermally-stable rig re-profile per chip. Defaults unchanged => behavior-preserving. dt0004 full-clip threaded 0.979, 0 wrong. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/sam3_coreml_pipeline.cpp | 15 +++++++++------ sam3.cpp | 25 +++++++++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/examples/sam3_coreml_pipeline.cpp b/examples/sam3_coreml_pipeline.cpp index 9946ad9..2f7a4b6 100644 --- a/examples/sam3_coreml_pipeline.cpp +++ b/examples/sam3_coreml_pipeline.cpp @@ -45,14 +45,17 @@ int main(int argc, char** argv){ const char* GE = envOr("SAM3_COREML_MODELS", "/Users/noah.johnson/iris/audits/goldenclip-eval/coreml_models"); auto P=[&](const char* f){ std::string s=GE; s+="/"; s+=f; return s; }; - // heterogeneous placement (encoder ANE, mem-attn GPU; decoder/memenc default ANE, overridable) - int U_DEC = envUnit("SAM3_DEC_UNIT_N", 1), U_ME = envUnit("SAM3_MENC_UNIT_N", 1); - auto enc = edgetam_coreml_create(P("edgetam_encoder_neck_nhwc.mlpackage").c_str(), 1); - auto ma = edgetam_coreml_create(P("edgetam_memory_attention.mlpackage").c_str(), 2); + // heterogeneous placement — ALL FOUR stages overridable for the M4 compute-unit + // sweep (RFD 0011 U8). 0=ALL 1=ANE 2=GPU 3=CPU. M1-era default: enc ANE, mem-attn + // GPU, decoder/memenc ANE — re-validate on M4 (the optimal units may invert). + int U_ENC = envUnit("SAM3_ENC_UNIT_N", 1), U_MA = envUnit("SAM3_MA_UNIT_N", 2); + int U_DEC = envUnit("SAM3_DEC_UNIT_N", 1), U_ME = envUnit("SAM3_MENC_UNIT_N", 1); + auto enc = edgetam_coreml_create(P("edgetam_encoder_neck_nhwc.mlpackage").c_str(), U_ENC); + auto ma = edgetam_coreml_create(P("edgetam_memory_attention.mlpackage").c_str(), U_MA); auto dec = edgetam_coreml_create(P("edgetam_mask_decoder_nhwc.mlpackage").c_str(), U_DEC); auto me = edgetam_coreml_create(P("edgetam_memory_encode.mlpackage").c_str(), U_ME); - if(!enc||!ma||!dec||!me){ fprintf(stderr,"model load failed; set SAM3_COREML_MODELS=\n"); return 1; } - printf("loaded enc(ANE)+memattn(GPU)+dec(unit%d)+memenc(unit%d) N=%d warmup=%d\n", U_DEC, U_ME, N, WARMUP); + if(!enc||!ma||!dec||!me){ fprintf(stderr,"model load failed (enc=%d ma=%d dec=%d me=%d); a NULL = that stage's unit rejected the model\n", !!enc,!!ma,!!dec,!!me); return 1; } + printf("loaded enc(unit%d)+memattn(unit%d)+dec(unit%d)+memenc(unit%d) N=%d warmup=%d\n", U_ENC, U_MA, U_DEC, U_ME, N, WARMUP); const size_t S_IMG=3*1024*1024, S_N0=256*256*256, S_N1=128*128*256, S_N2=64*64*256; const size_t S_CURR=4096*256, S_MEM=3648*64, S_PE=64*64*256, S_SP=256, S_MASKF=1024*1024; diff --git a/sam3.cpp b/sam3.cpp index 82fbb57..5f17c21 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -4983,6 +4983,19 @@ static void edgetam_build_repvit_graph(struct ggml_context* ctx, // SPSC: the bench serializes set→consume per frame, so no lock is needed here. static std::vector s_prefetch_neck[3]; static bool s_prefetch_ready = false; + +// RFD 0011 U8 — per-chip CoreML compute-unit profile (0=ALL 1=ANE 2=GPU 3=CPU). +// M4 Pro sweep (sam3_coreml_pipeline, isolated 20-rep median): encoder ANE 11ms +// (GPU 14, CPU 38), mem-attn GPU 17ms (ANE 26, CPU 24), decoder ANE 7.8ms (GPU 8.2), +// memenc ANE 5.0 / GPU 3.9. Defaults = the validated M4 optimum (enc ANE, mem-attn +// GPU, decoder ANE, memenc ANE — memenc's isolated GPU edge washes out in thermal +// noise end-to-end). NOT inverted vs M1 like the ORT/EfficientTAM stack. Override +// per chip via env (e.g. SAM3_COREML_MA_UNIT=1) if M1/M2/M3 profiles differ. +static int sam3_coreml_unit(const char* env, int def) { + const char* v = getenv(env); + return (v && *v) ? atoi(v) : def; +} + void sam3_coreml_set_prefetched_neck(const float* n0, const float* n1, const float* n2) { const int Wd[3] = {256, 128, 64}, D = 256; const float* src[3] = {n0, n1, n2}; @@ -5008,7 +5021,7 @@ static bool edgetam_encode_image_coreml(sam3_state& state, const sam3_model& mod if (!s_prefetch_ready && !s_enc) { // prefetch path needs no local encoder handle const char* mp = getenv("SAM3_COREML_MODEL"); if (!mp) { fprintf(stderr, "%s: SAM3_COREML_ENCODER set but SAM3_COREML_MODEL unset\n", __func__); return false; } - s_enc = edgetam_coreml_create(mp, /*compute_units=ANE*/ 1); + s_enc = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_ENC_UNIT", /*ANE*/ 1)); if (!s_enc) { fprintf(stderr, "%s: CoreML encoder load failed\n", __func__); return false; } fprintf(stderr, "%s: CoreML EdgeTAM encoder loaded (ANE): %s\n", __func__, mp); } @@ -11694,7 +11707,7 @@ static sam3_prop_output sam3_propagate_single( s_memattn_tried = true; const char* mp = getenv("SAM3_COREML_MEMATTN_MODEL"); if (mp) { - s_memattn = edgetam_coreml_create(mp, /*CPU_AND_GPU*/ 2); + s_memattn = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_MA_UNIT", /*GPU*/ 2)); if (s_memattn) fprintf(stderr, "%s: CoreML mem-attn loaded (GPU): %s\n", __func__, mp); } } @@ -11722,7 +11735,7 @@ static sam3_prop_output sam3_propagate_single( s_decoder_tried = true; const char* mp = getenv("SAM3_COREML_DECODER_MODEL"); if (mp) { - s_decoder = edgetam_coreml_create(mp, /*CPU_AND_NE*/ 1); + s_decoder = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_DEC_UNIT", /*ANE*/ 1)); if (s_decoder) fprintf(stderr, "%s: CoreML decoder loaded (ANE): %s\n", __func__, mp); } } @@ -12191,7 +12204,11 @@ static bool sam3_encode_memory( s_memenc_tried = true; const char* mp = getenv("SAM3_COREML_MEMENC_MODEL"); if (mp) { - s_memenc = edgetam_coreml_create(mp, /*CPU_AND_NE*/ 1); + // memenc GPU is ~1.1ms faster in isolation (M4 bench: 3.9 vs ANE 5.0) but the + // win is WITHIN thermal run-to-run noise end-to-end (A/B inconclusive) and + // accuracy is identical, so the default stays ANE (validated). Tune per-rig + // via SAM3_COREML_MENC_UNIT if a thermally-stable bench shows a real win. + s_memenc = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_MENC_UNIT", /*ANE*/ 1)); if (s_memenc) fprintf(stderr, "%s: CoreML memenc loaded (ANE): %s\n", __func__, mp); } } From 8ac99976a7d52ce5226abb8be93324741168cf43 Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Fri, 19 Jun 2026 16:46:25 -0400 Subject: [PATCH 12/14] feat(build): SAM3_VULKAN cross-vendor GPU backend + CoreML-free capi (RFD 0011 W0) Wires GGML_VULKAN behind a SAM3_VULKAN option (non-Apple) so the same ggml EdgeTAM engine runs on AMD/NVIDIA/Intel via Vulkan, and compiles the cgo C-ABI (edgetam_capi.cpp) on the Vulkan path WITHOUT the CoreML bridge: all edgetam_coreml_*/sam3_coreml_* calls + the encoder-ahead producer pool are now #ifdef SAM3_COREML-guarded, so a Vulkan build runs the pure-ggml path (which lands on the Vulkan device) and pool_size()==0 (synchronous Track, no threading). Mac SAM3_COREML build verified unchanged (guards are additive). Build + measure on real AMD/NVIDIA hardware is still required (W0.4 gate) before this is validated. Co-Authored-By: Claude Opus 4.8 (1M context) --- CMakeLists.txt | 19 +++++++++++++++++++ coreml/edgetam_capi.cpp | 22 +++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd5bed1..dc0029e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,16 @@ if(APPLE AND SAM3_METAL) set(GGML_METAL_EMBED_LIBRARY ON CACHE BOOL "" FORCE) endif() +# RFD 0011 (Windows/Linux cross-vendor GPU): ggml-Vulkan covers AMD/NVIDIA/Intel +# from one build. The Mac path uses Metal+CoreML; this is the non-Apple GPU path. +# The same cgo C-ABI (coreml/edgetam_capi.cpp) is reused below — its CoreML stage +# calls are #ifdef SAM3_COREML-guarded, so a Vulkan build runs the pure-ggml path +# (which lands on the Vulkan device). Requires the Vulkan SDK (glslc) at build. +option(SAM3_VULKAN "Enable the ggml-Vulkan cross-vendor GPU backend (non-Apple)" OFF) +if(NOT APPLE AND SAM3_VULKAN) + set(GGML_VULKAN ON CACHE BOOL "" FORCE) +endif() + add_subdirectory(ggml) # sam3 static library @@ -65,6 +75,15 @@ if(APPLE) endif() endif() +# RFD 0011 (Windows/Vulkan): reuse the cgo tracker C-ABI on the non-Apple Vulkan +# path WITHOUT the CoreML bridge. edgetam_capi.cpp's CoreML stage calls + producer- +# encoder pool are #ifdef SAM3_COREML-guarded, so here the tracker runs the pure- +# ggml path on the Vulkan device. SAM3_COREML is NOT defined, no .mm, no frameworks. +if(NOT APPLE AND SAM3_VULKAN) + target_sources(sam3 PRIVATE coreml/edgetam_capi.cpp) + target_include_directories(sam3 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/coreml) +endif() + # Examples (only when top-level) option(SAM3_BUILD_EXAMPLES "Build example executables" ON) if(SAM3_BUILD_EXAMPLES AND CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) diff --git a/coreml/edgetam_capi.cpp b/coreml/edgetam_capi.cpp index 3949aa5..5aca9bd 100644 --- a/coreml/edgetam_capi.cpp +++ b/coreml/edgetam_capi.cpp @@ -45,10 +45,14 @@ extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, const char* ggml_model, int /*use_samurai*/, int /*use_lifecycle*/) { try { +#ifdef SAM3_COREML // Enable the CoreML stages (encoder ANE + mem-attn GPU + decoder ANE) by // pointing the library's existing runtime gates at the .mlpackage dir. // The library's full-CoreML propagation path then runs with zero ggml // graph work per frame; ggml remains the seed + early-frame fallback. + // On a non-CoreML build (e.g. Windows/Vulkan, SAM3_VULKAN) this block is + // absent → models_dir is ignored and the tracker runs the pure-ggml path, + // which lands on whatever ggml GPU backend is compiled (Vulkan). if (models_dir && *models_dir) { std::string d = models_dir; setenv("SAM3_COREML_ENCODER", "1", 1); @@ -66,6 +70,9 @@ extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, setenv("SAM3_COREML_MEMENC_MODEL", (d + "/edgetam_memory_encode.mlpackage").c_str(), 1); setenv("SAM3_COREML_PAD_BANK", "1", 1); } +#else + (void)models_dir; // Vulkan/CPU build: pure-ggml path, no CoreML stage gates +#endif auto t = std::make_unique(); sam3_params p; p.model_path = ggml_model ? ggml_model : ""; @@ -75,7 +82,10 @@ extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, t->state = sam3_create_state(*t->model, p); sam3_visual_track_params vp; // defaults match sam3_edgetam_bench t->tracker = sam3_create_visual_tracker(*t->model, vp); - // Wave 5: producer encoder handle + neck pool (only when CoreML is on). +#ifdef SAM3_COREML + // Wave 5: producer encoder handle + neck pool (CoreML encoder-ahead threading). + // Absent on non-CoreML builds → prod_enc stays null → pool_size()==0 → + // the cgo session uses the synchronous (single-thread) Track path. if (models_dir && *models_dir) { std::string d = models_dir; t->prod_enc = edgetam_coreml_create((d + "/edgetam_encoder_neck_nhwc.mlpackage").c_str(), /*ANE*/ 1); @@ -84,6 +94,7 @@ extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, for (int i = 0; i < 3; ++i) t->neck[s][i].assign((size_t)256 * Wd[i] * Wd[i], 0.f); } +#endif return t.release(); } catch (...) { return nullptr; @@ -138,7 +149,9 @@ extern "C" void edgetam_capi_reset(edgetam_tracker_t h) { extern "C" void edgetam_capi_destroy(edgetam_tracker_t h) { auto* t = static_cast(h); +#ifdef SAM3_COREML if (t && t->prod_enc) edgetam_coreml_destroy(t->prod_enc); +#endif delete t; } @@ -158,6 +171,7 @@ extern "C" int edgetam_capi_encode_slot(edgetam_tracker_t h, int slot, const uint8_t* rgb, int w, int hgt) { auto* t = static_cast(h); if (!t || !t->prod_enc || !rgb || slot < 0 || slot >= EtTracker::POOL) return 0; +#ifdef SAM3_COREML try { sam3_image img = make_image(rgb, w, hgt); std::vector norm = sam3_coreml_preprocess_image(img, 1024); @@ -168,6 +182,10 @@ extern "C" int edgetam_capi_encode_slot(edgetam_tracker_t h, int slot, } catch (...) { return 0; } +#else + (void)rgb; (void)w; (void)hgt; // encoder-ahead threading is CoreML-only + return 0; +#endif } extern "C" et_result edgetam_capi_track_slot(edgetam_tracker_t h, int slot, @@ -177,11 +195,13 @@ extern "C" et_result edgetam_capi_track_slot(edgetam_tracker_t h, int slot, auto* t = static_cast(h); if (!t || !rgb || slot < 0 || slot >= EtTracker::POOL) return r; try { +#ifdef SAM3_COREML // Consumer thread only: hand the producer-encoded neck to the library so // sam3_propagate_frame's encode step is skipped (no ggml/CoreML encode here). sam3_coreml_set_prefetched_neck(t->neck[slot][0].data(), t->neck[slot][1].data(), t->neck[slot][2].data()); +#endif sam3_image img = make_image(rgb, w, hgt); sam3_result res = sam3_propagate_frame(*t->tracker, *t->state, *t->model, img); if (res.detections.empty()) return r; From 1688f6f24a6f2671bf99935f8fef0ff9f59b203c Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:37:39 -0400 Subject: [PATCH 13/14] feat(vulkan): select the ggml-Vulkan backend in the model loader (RFD 0011 W0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loader was #ifdef GGML_USE_METAL → Metal, else CPU — so a GGML_VULKAN build compiled the Vulkan backend but never used it (model loaded on CPU). Add the GGML_USE_VULKAN branch (ggml_backend_vk_init(0)) + the ggml-vulkan.h include. VALIDATED on Mac via MoltenVK (the only Vulkan device available here): model now loads "using Vulkan backend" on the M4 GPU, runs EdgeTAM end-to-end at 2.88 fps (encoder 102ms, mem-attn 119ms — note ggml-Vulkan mem-attn BEATS ggml-Metal's 184ms). MoltenVK has no matrix cores + translation overhead, so this is a worst-case proxy, NOT representative of native AMD/NVIDIA Vulkan — the real W0.4 per-vendor gate still needs that hardware. Mac SAM3_COREML build unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- sam3.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sam3.cpp b/sam3.cpp index 5f17c21..c46bb24 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -12,6 +12,12 @@ #include "ggml-metal.h" #endif +// RFD 0011 (Windows/Linux cross-vendor GPU): the ggml-Vulkan backend covers +// AMD/NVIDIA/Intel from one build (SAM3_VULKAN). The model loader selects it below. +#ifdef GGML_USE_VULKAN +#include "ggml-vulkan.h" +#endif + /* stb (implementation compiled here -- order is pinned) */ #define STB_IMAGE_IMPLEMENTATION #include "stb_image.h" @@ -3424,6 +3430,14 @@ std::shared_ptr sam3_load_model(const sam3_params& params) { fprintf(stderr, "%s: using Metal backend\n", __func__); model->backend = ggml_backend_metal_init(); } +#endif +#ifdef GGML_USE_VULKAN + // RFD 0011: cross-vendor GPU. ggml-Vulkan runs the same EdgeTAM graph on any + // Vulkan device (AMD/NVIDIA/Intel; Apple via MoltenVK). device 0 = first GPU. + if (params.use_gpu && !model->backend) { + fprintf(stderr, "%s: using Vulkan backend\n", __func__); + model->backend = ggml_backend_vk_init(0); + } #endif if (!model->backend) { fprintf(stderr, "%s: using CPU backend\n", __func__); From 8b94d6f0189ed3eb349fba45f384093d959cebef Mon Sep 17 00:00:00 2001 From: njayj <124410198+njayj@users.noreply.github.com> Date: Fri, 19 Jun 2026 18:34:16 -0400 Subject: [PATCH 14/14] feat(coreml): export the EdgeTAM mask via the C-ABI (RFD 0011 U8) The U2 fast path derived the bbox from the low-res logit grid and never built a mask bitmap, so the cgo tracker had no mask to ship. Now populate a CHEAP low-res (256x256) binary mask from po.mask_logits on the fast path (no full-res upscale -> no perf regression, ~12.5fps held) and attach it to the result detection unconditionally (was gated behind SAM3_FULLRES_MASK). New edgetam_capi_last_mask(out, cap, *w, *h) copies the last track's mask out. Verified: dt0004 last frame 256x256, 3.8% foreground. Co-Authored-By: Claude Opus 4.8 (1M context) --- coreml/edgetam_capi.cpp | 27 ++++++++++++++++++++++++-- coreml/edgetam_capi.h | 7 +++++++ examples/sam3_coreml_capi_threaded.cpp | 15 ++++++++++++++ sam3.cpp | 17 ++++++++++++++-- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/coreml/edgetam_capi.cpp b/coreml/edgetam_capi.cpp index 5aca9bd..1c99dcd 100644 --- a/coreml/edgetam_capi.cpp +++ b/coreml/edgetam_capi.cpp @@ -28,6 +28,11 @@ struct EtTracker { edgetam_coreml_handle prod_enc = nullptr; static const int POOL = 3; std::vector neck[POOL][3]; // n0 256*256*256, n1 128*128*256, n2 64*64*256 + + // RFD 0011 U8 mask export: the last track's binary mask (0/255, orig-frame + // resolution), stashed so the Go side can pull the pixels after a track() + // for State-Sync/frontend rendering. Cleared on a lost frame. + sam3_mask last_mask; }; // Wrap a caller RGB24 buffer as a sam3_image (copies into the owned vector — the @@ -125,8 +130,9 @@ extern "C" et_result edgetam_capi_track(edgetam_tracker_t h, const uint8_t* rgb, try { sam3_image img = make_image(rgb, w, hgt); sam3_result res = sam3_propagate_frame(*t->tracker, *t->state, *t->model, img); - if (res.detections.empty()) return r; // valid stays 0 -> lost + if (res.detections.empty()) { t->last_mask = sam3_mask{}; return r; } // lost -> clear mask const sam3_detection& d = res.detections[0]; + t->last_mask = d.mask; // stash for edgetam_capi_last_mask const float W = (float)w, H = (float)hgt; r.box.x0 = d.box.x0 / W; r.box.y0 = d.box.y0 / H; r.box.x1 = d.box.x1 / W; r.box.y1 = d.box.y1 / H; @@ -140,6 +146,22 @@ extern "C" et_result edgetam_capi_track(edgetam_tracker_t h, const uint8_t* rgb, } } +// Copies the last track's binary mask (0/255, row-major, w*h bytes) into `out` +// (capacity `cap`). Writes dims to *w,*h. Returns the mask byte count (w*h); if +// cap < w*h it copies nothing and returns w*h so the caller can size the buffer. +// 0 => no mask this frame (lost / not yet tracked). RFD 0011 U8 mask export. +extern "C" int edgetam_capi_last_mask(edgetam_tracker_t h, uint8_t* out, int cap, int* w, int* hgt) { + auto* t = static_cast(h); + if (!t) return 0; + const sam3_mask& m = t->last_mask; + const int n = (int)m.data.size(); + if (w) *w = m.width; + if (hgt) *hgt = m.height; + if (n == 0) return 0; + if (out && cap >= n) std::memcpy(out, m.data.data(), (size_t)n); + return n; +} + extern "C" void edgetam_capi_reset(edgetam_tracker_t h) { auto* t = static_cast(h); if (t && t->tracker) { @@ -204,8 +226,9 @@ extern "C" et_result edgetam_capi_track_slot(edgetam_tracker_t h, int slot, #endif sam3_image img = make_image(rgb, w, hgt); sam3_result res = sam3_propagate_frame(*t->tracker, *t->state, *t->model, img); - if (res.detections.empty()) return r; + if (res.detections.empty()) { t->last_mask = sam3_mask{}; return r; } const sam3_detection& d = res.detections[0]; + t->last_mask = d.mask; // stash for edgetam_capi_last_mask const float W = (float)w, H = (float)hgt; r.box.x0 = d.box.x0 / W; r.box.y0 = d.box.y0 / H; r.box.x1 = d.box.x1 / W; r.box.y1 = d.box.y1 / H; diff --git a/coreml/edgetam_capi.h b/coreml/edgetam_capi.h index a65af0f..4849ecd 100644 --- a/coreml/edgetam_capi.h +++ b/coreml/edgetam_capi.h @@ -44,6 +44,13 @@ int edgetam_capi_seed(edgetam_tracker_t t, const uint8_t* rgb, int w, int h, et_ // (box normalized [0,1]); valid==0 when the target is lost this frame. et_result edgetam_capi_track(edgetam_tracker_t t, const uint8_t* rgb, int w, int h); +// RFD 0011 U8 mask export — copy the LAST track's binary mask (0/255, row-major, +// w*h bytes, original-frame resolution) into `out` (capacity `cap`). Writes dims +// to *w,*h. Returns the byte count (w*h); if cap < w*h nothing is copied and w*h +// is returned so the caller can size its buffer. 0 => no mask (lost / pre-track). +// Valid after edgetam_capi_track / edgetam_capi_track_slot on the same handle. +int edgetam_capi_last_mask(edgetam_tracker_t t, uint8_t* out, int cap, int* w, int* h); + // RFD 0011 U8 Wave 5 — encoder-ahead threading split. pool_size returns the slot // count (0 if CoreML/threading is unavailable). encode_slot (PRODUCER thread) // preprocesses+encodes rgb into slot's neck buffers via the producer's own encoder diff --git a/examples/sam3_coreml_capi_threaded.cpp b/examples/sam3_coreml_capi_threaded.cpp index 18e3d2a..59fb6c2 100644 --- a/examples/sam3_coreml_capi_threaded.cpp +++ b/examples/sam3_coreml_capi_threaded.cpp @@ -116,6 +116,21 @@ int main(int argc, char** argv) { } auto t1 = std::chrono::high_resolution_clock::now(); producer.join(); + + // RFD 0011 U8 mask export check: pull the last track's mask via the C-ABI. + { + int mw = 0, mh = 0; + int need = edgetam_capi_last_mask(h, nullptr, 0, &mw, &mh); + if (need > 0) { + std::vector mbuf(need); + edgetam_capi_last_mask(h, mbuf.data(), need, &mw, &mh); + size_t fg = 0; for (auto v : mbuf) if (v) ++fg; + printf("MASK export: last frame %dx%d, %zu foreground px (%.1f%% of frame)\n", + mw, mh, fg, 100.0 * (double)fg / (double)need); + } else { + printf("MASK export: EMPTY on last frame (w=%d h=%d) — mask not populated on cgo path\n", mw, mh); + } + } double wall_ms = std::chrono::duration(t1 - t0).count(); printf("\nCAPI THREADED: %.2f fps (%.1f ms over %d hold frames, encoder-ahead via capi)\n", (n_frames - 1) * 1000.0 / wall_ms, wall_ms, n_frames - 1); diff --git a/sam3.cpp b/sam3.cpp index c46bb24..8947d16 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -13122,6 +13122,17 @@ sam3_result sam3_propagate_frame( pm[id].data.resize(state.orig_width * state.orig_height); for (int p = 0; p < (int)rs.size(); ++p) pm[id].data[p] = rs[p] > 0.0f ? 255 : 0; + } else { + // RFD 0011 U8 mask export: a cheap LOW-RES binary mask straight off the + // decoder's ~256x256 logit grid (no full-res upscale → keeps the U2 + // fast-path perf). pm carries it to the result detection (~13315) so the + // C-ABI/State-Sync can ship it; the frontend upsamples to the video. + const int mw = po[id].mask_w, mh = po[id].mask_h; + pm[id].width = mw; + pm[id].height = mh; + pm[id].data.resize((size_t)mw * mh); + for (int p = 0; p < mw * mh; ++p) + pm[id].data[p] = po[id].mask_logits[p] > 0.0f ? 255 : 0; } ml.last_score = po[id].iou_scores[0]; ml.last_seen = fi; @@ -13310,8 +13321,10 @@ sam3_result sam3_propagate_frame( det.mask.iou_score = score; auto pit = po.find(inst_id); if (pit != po.end()) det.mask.obj_score = pit->second.obj_score; - // Attach full-res mask pixels only if we built them (debug/overlay). - if (want_fullres) { + // Attach the mask pixels: low-res grid by default (RFD 0011 U8 export, for + // State-Sync/frontend display), full-res when SAM3_FULLRES_MASK. Both were + // built into pm above; emit whichever is present. + { auto it = pm.find(inst_id); if (it != pm.end()) { det.mask.width = it->second.width;