diff --git a/CMakeLists.txt b/CMakeLists.txt index 1aacee0..dc0029e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,16 @@ if(APPLE AND SAM3_METAL) set(GGML_METAL_EMBED_LIBRARY ON CACHE BOOL "" FORCE) endif() +# RFD 0011 (Windows/Linux cross-vendor GPU): ggml-Vulkan covers AMD/NVIDIA/Intel +# from one build. The Mac path uses Metal+CoreML; this is the non-Apple GPU path. +# The same cgo C-ABI (coreml/edgetam_capi.cpp) is reused below — its CoreML stage +# calls are #ifdef SAM3_COREML-guarded, so a Vulkan build runs the pure-ggml path +# (which lands on the Vulkan device). Requires the Vulkan SDK (glslc) at build. +option(SAM3_VULKAN "Enable the ggml-Vulkan cross-vendor GPU backend (non-Apple)" OFF) +if(NOT APPLE AND SAM3_VULKAN) + set(GGML_VULKAN ON CACHE BOOL "" FORCE) +endif() + add_subdirectory(ggml) # sam3 static library @@ -57,12 +67,23 @@ if(APPLE) set_source_files_properties(coreml/edgetam_coreml.mm PROPERTIES LANGUAGE CXX COMPILE_FLAGS "-x objective-c++ -fobjc-arc") + # RFD 0011 U8: extern "C" tracker C-ABI for cgo into iris/platform. + target_sources(sam3 PRIVATE coreml/edgetam_capi.cpp) target_include_directories(sam3 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/coreml) target_compile_definitions(sam3 PUBLIC SAM3_COREML) target_link_libraries(sam3 PUBLIC "-framework CoreML" "-framework Foundation") endif() endif() +# RFD 0011 (Windows/Vulkan): reuse the cgo tracker C-ABI on the non-Apple Vulkan +# path WITHOUT the CoreML bridge. edgetam_capi.cpp's CoreML stage calls + producer- +# encoder pool are #ifdef SAM3_COREML-guarded, so here the tracker runs the pure- +# ggml path on the Vulkan device. SAM3_COREML is NOT defined, no .mm, no frameworks. +if(NOT APPLE AND SAM3_VULKAN) + target_sources(sam3 PRIVATE coreml/edgetam_capi.cpp) + target_include_directories(sam3 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/coreml) +endif() + # Examples (only when top-level) option(SAM3_BUILD_EXAMPLES "Build example executables" ON) if(SAM3_BUILD_EXAMPLES AND CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) diff --git a/coreml/edgetam_capi.cpp b/coreml/edgetam_capi.cpp new file mode 100644 index 0000000..1c99dcd --- /dev/null +++ b/coreml/edgetam_capi.cpp @@ -0,0 +1,247 @@ +// RFD 0011 U8 — implementation of the extern "C" tracker C-ABI (edgetam_capi.h). +// Thin POD wrapper over the sam3.cpp visual tracker (sam3_create_visual_tracker / +// sam3_encode_image / sam3_tracker_add_instance / sam3_propagate_frame). All C++ +// exceptions are caught at the boundary (cgo cannot unwind through Go). +#include "edgetam_capi.h" +#include "../sam3.h" +#include "edgetam_coreml.h" // producer encoder handle (edgetam_coreml_create/encode/destroy) + +#include +#include +#include +#include +#include + +namespace { + +struct EtTracker { + std::shared_ptr model; + sam3_state_ptr state; + sam3_tracker_ptr tracker; + + // RFD 0011 U8 Wave 5 (encoder-ahead threading): a producer encoder handle + // (its OWN MLModel, per-thread ownership) + a fixed neck pool. Go owns slot + // free/ready coordination via channels, so NO lock here — each slot is written + // once by encode_slot (producer thread) then read once by track_slot (consumer + // thread). track_slot is the ONLY caller of sam3_coreml_set_prefetched_neck + // (global, consumer-thread-only). prod_enc is non-null only when CoreML is on. + edgetam_coreml_handle prod_enc = nullptr; + static const int POOL = 3; + std::vector neck[POOL][3]; // n0 256*256*256, n1 128*128*256, n2 64*64*256 + + // RFD 0011 U8 mask export: the last track's binary mask (0/255, orig-frame + // resolution), stashed so the Go side can pull the pixels after a track() + // for State-Sync/frontend rendering. Cleared on a lost frame. + sam3_mask last_mask; +}; + +// Wrap a caller RGB24 buffer as a sam3_image (copies into the owned vector — the +// library keeps no alias, and cgo must not let Go memory escape into C). +sam3_image make_image(const uint8_t* rgb, int w, int h) { + sam3_image img; + img.width = w; img.height = h; img.channels = 3; + img.data.assign(rgb, rgb + (size_t)w * h * 3); + return img; +} + +} // namespace + +extern "C" edgetam_tracker_t edgetam_capi_create(const char* models_dir, + const char* ggml_model, + int /*use_samurai*/, int /*use_lifecycle*/) { + try { +#ifdef SAM3_COREML + // Enable the CoreML stages (encoder ANE + mem-attn GPU + decoder ANE) by + // pointing the library's existing runtime gates at the .mlpackage dir. + // The library's full-CoreML propagation path then runs with zero ggml + // graph work per frame; ggml remains the seed + early-frame fallback. + // On a non-CoreML build (e.g. Windows/Vulkan, SAM3_VULKAN) this block is + // absent → models_dir is ignored and the tracker runs the pure-ggml path, + // which lands on whatever ggml GPU backend is compiled (Vulkan). + if (models_dir && *models_dir) { + std::string d = models_dir; + setenv("SAM3_COREML_ENCODER", "1", 1); + setenv("SAM3_COREML_MODEL", (d + "/edgetam_encoder_neck_nhwc.mlpackage").c_str(), 1); + setenv("SAM3_COREML_MEMATTN", "1", 1); + setenv("SAM3_COREML_MEMATTN_MODEL", (d + "/edgetam_memory_attention.mlpackage").c_str(), 1); + setenv("SAM3_COREML_DECODER", "1", 1); + setenv("SAM3_COREML_DECODER_MODEL", (d + "/edgetam_mask_decoder_nhwc.mlpackage").c_str(), 1); + // Memory-encode on ANE + a fixed-size (padded) memory bank. The CoreML + // mem-attention model has a static 3648-token input (7*512 + 16*4); PAD_BANK + // pads the live bank to that shape so MEMATTN actually runs on the ANE/GPU + // instead of silently falling back to the ~184ms ggml path. Without these two + // the tracker is correct but ~8x slower (measured 570ms vs 69ms/frame). + setenv("SAM3_COREML_MEMENC", "1", 1); + setenv("SAM3_COREML_MEMENC_MODEL", (d + "/edgetam_memory_encode.mlpackage").c_str(), 1); + setenv("SAM3_COREML_PAD_BANK", "1", 1); + } +#else + (void)models_dir; // Vulkan/CPU build: pure-ggml path, no CoreML stage gates +#endif + auto t = std::make_unique(); + sam3_params p; + p.model_path = ggml_model ? ggml_model : ""; + p.use_gpu = true; + t->model = sam3_load_model(p); + if (!t->model) return nullptr; + t->state = sam3_create_state(*t->model, p); + sam3_visual_track_params vp; // defaults match sam3_edgetam_bench + t->tracker = sam3_create_visual_tracker(*t->model, vp); +#ifdef SAM3_COREML + // Wave 5: producer encoder handle + neck pool (CoreML encoder-ahead threading). + // Absent on non-CoreML builds → prod_enc stays null → pool_size()==0 → + // the cgo session uses the synchronous (single-thread) Track path. + if (models_dir && *models_dir) { + std::string d = models_dir; + t->prod_enc = edgetam_coreml_create((d + "/edgetam_encoder_neck_nhwc.mlpackage").c_str(), /*ANE*/ 1); + const int Wd[3] = {256, 128, 64}; + for (int s = 0; s < EtTracker::POOL; ++s) + for (int i = 0; i < 3; ++i) + t->neck[s][i].assign((size_t)256 * Wd[i] * Wd[i], 0.f); + } +#endif + return t.release(); + } catch (...) { + return nullptr; + } +} + +extern "C" int edgetam_capi_seed(edgetam_tracker_t h, const uint8_t* rgb, int w, int hgt, et_box seed) { + auto* t = static_cast(h); + if (!t || !rgb) return 0; + try { + sam3_image img = make_image(rgb, w, hgt); + if (!sam3_encode_image(*t->state, *t->model, img)) return 0; + sam3_pvs_params pvs; + pvs.use_box = true; + pvs.box = {seed.x0, seed.y0, seed.x1, seed.y1}; + int id = sam3_tracker_add_instance(*t->tracker, *t->state, *t->model, pvs); + return id >= 0 ? 1 : 0; + } catch (...) { + return 0; + } +} + +extern "C" et_result edgetam_capi_track(edgetam_tracker_t h, const uint8_t* rgb, int w, int hgt) { + et_result r; + std::memset(&r, 0, sizeof(r)); + auto* t = static_cast(h); + if (!t || !rgb) return r; + try { + sam3_image img = make_image(rgb, w, hgt); + sam3_result res = sam3_propagate_frame(*t->tracker, *t->state, *t->model, img); + if (res.detections.empty()) { t->last_mask = sam3_mask{}; return r; } // lost -> clear mask + const sam3_detection& d = res.detections[0]; + t->last_mask = d.mask; // stash for edgetam_capi_last_mask + const float W = (float)w, H = (float)hgt; + r.box.x0 = d.box.x0 / W; r.box.y0 = d.box.y0 / H; + r.box.x1 = d.box.x1 / W; r.box.y1 = d.box.y1 / H; + r.valid = 1; + r.obj_score = d.score; + r.mask_iou = d.iou_score; + r.state = (int)d.state; + return r; + } catch (...) { + return r; + } +} + +// Copies the last track's binary mask (0/255, row-major, w*h bytes) into `out` +// (capacity `cap`). Writes dims to *w,*h. Returns the mask byte count (w*h); if +// cap < w*h it copies nothing and returns w*h so the caller can size the buffer. +// 0 => no mask this frame (lost / not yet tracked). RFD 0011 U8 mask export. +extern "C" int edgetam_capi_last_mask(edgetam_tracker_t h, uint8_t* out, int cap, int* w, int* hgt) { + auto* t = static_cast(h); + if (!t) return 0; + const sam3_mask& m = t->last_mask; + const int n = (int)m.data.size(); + if (w) *w = m.width; + if (hgt) *hgt = m.height; + if (n == 0) return 0; + if (out && cap >= n) std::memcpy(out, m.data.data(), (size_t)n); + return n; +} + +extern "C" void edgetam_capi_reset(edgetam_tracker_t h) { + auto* t = static_cast(h); + if (t && t->tracker) { + try { sam3_tracker_reset(*t->tracker); } catch (...) {} + } +} + +extern "C" void edgetam_capi_destroy(edgetam_tracker_t h) { + auto* t = static_cast(h); +#ifdef SAM3_COREML + if (t && t->prod_enc) edgetam_coreml_destroy(t->prod_enc); +#endif + delete t; +} + +// ── RFD 0011 U8 Wave 5: encoder-ahead split ────────────────────────────────── +// Go drives the pipeline: a producer goroutine pops a free slot, calls +// encode_slot; a consumer goroutine calls track_slot then recycles the slot. The +// two run on different OS threads (per-thread MLModel ownership): encode_slot uses +// the producer encoder handle; track_slot consumes the prefetched neck and never +// encodes. Go's channels serialize each slot (write-then-read), so no lock here. + +extern "C" int edgetam_capi_pool_size(edgetam_tracker_t h) { + auto* t = static_cast(h); + return (t && t->prod_enc) ? EtTracker::POOL : 0; // 0 => threading unavailable +} + +extern "C" int edgetam_capi_encode_slot(edgetam_tracker_t h, int slot, + const uint8_t* rgb, int w, int hgt) { + auto* t = static_cast(h); + if (!t || !t->prod_enc || !rgb || slot < 0 || slot >= EtTracker::POOL) return 0; +#ifdef SAM3_COREML + try { + sam3_image img = make_image(rgb, w, hgt); + std::vector norm = sam3_coreml_preprocess_image(img, 1024); + return edgetam_coreml_encode(t->prod_enc, norm.data(), + t->neck[slot][0].data(), + t->neck[slot][1].data(), + t->neck[slot][2].data()) ? 1 : 0; + } catch (...) { + return 0; + } +#else + (void)rgb; (void)w; (void)hgt; // encoder-ahead threading is CoreML-only + return 0; +#endif +} + +extern "C" et_result edgetam_capi_track_slot(edgetam_tracker_t h, int slot, + const uint8_t* rgb, int w, int hgt) { + et_result r; + std::memset(&r, 0, sizeof(r)); + auto* t = static_cast(h); + if (!t || !rgb || slot < 0 || slot >= EtTracker::POOL) return r; + try { +#ifdef SAM3_COREML + // Consumer thread only: hand the producer-encoded neck to the library so + // sam3_propagate_frame's encode step is skipped (no ggml/CoreML encode here). + sam3_coreml_set_prefetched_neck(t->neck[slot][0].data(), + t->neck[slot][1].data(), + t->neck[slot][2].data()); +#endif + sam3_image img = make_image(rgb, w, hgt); + sam3_result res = sam3_propagate_frame(*t->tracker, *t->state, *t->model, img); + if (res.detections.empty()) { t->last_mask = sam3_mask{}; return r; } + const sam3_detection& d = res.detections[0]; + t->last_mask = d.mask; // stash for edgetam_capi_last_mask + const float W = (float)w, H = (float)hgt; + r.box.x0 = d.box.x0 / W; r.box.y0 = d.box.y0 / H; + r.box.x1 = d.box.x1 / W; r.box.y1 = d.box.y1 / H; + r.valid = 1; + r.obj_score = d.score; + r.mask_iou = d.iou_score; + r.state = (int)d.state; + return r; + } catch (...) { + return r; + } +} + +extern "C" const char* edgetam_capi_version(void) { + return SAM3_VERSION; +} diff --git a/coreml/edgetam_capi.h b/coreml/edgetam_capi.h new file mode 100644 index 0000000..4849ecd --- /dev/null +++ b/coreml/edgetam_capi.h @@ -0,0 +1,72 @@ +// RFD 0011 U8 — extern "C" C-ABI over the sam3.cpp EdgeTAM visual tracker, for +// cgo into iris/platform. POD only crosses this boundary (cgo cannot call the +// std:: C++ API in sam3.h). The engine underneath is the existing visual tracker +// (memory bank + U3 SAMURAI + U4 lifecycle); CoreML stages are enabled per +// edgetam_capi_create's models_dir. The pure-CoreML 20 fps engine (Wave 1) can +// later swap behind this same ABI without changing the platform cgo side. +#ifndef EDGETAM_CAPI_H +#define EDGETAM_CAPI_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* edgetam_tracker_t; + +// Box corners. seed (edgetam_capi_seed) is in ORIGINAL-FRAME PIXELS; results +// (et_result.box) are NORMALIZED [0,1] (matches hold.Box.Valid on the Go side). +typedef struct { float x0, y0, x1, y1; } et_box; + +// state mirrors sam3.h TargetState: 0 TRACKED, 1 AT_RISK, 2 OCCLUDED, 3 LOST, +// 4 CANDIDATE_REACQUIRE, 5 REACQUIRED. +typedef struct { + et_box box; + int valid; // 1 if a credible detection this frame + float obj_score; + float mask_iou; + int state; +} et_result; + +// Create a tracker. ggml_model = path to edgetam_f16.ggml (required, the seed +// path + ggml fallback). models_dir = dir holding the 4 CoreML .mlpackages; if +// non-empty, the CoreML encoder/mem-attn/decoder stages are enabled. Returns +// NULL on failure. +edgetam_tracker_t edgetam_capi_create(const char* models_dir, const char* ggml_model, + int use_samurai, int use_lifecycle); + +// Seed the target from a box on frame 0. rgb = contiguous RGB24 (w*h*3 bytes). +// seed box in PIXELS. Returns 1 on success. +int edgetam_capi_seed(edgetam_tracker_t t, const uint8_t* rgb, int w, int h, et_box seed_px); + +// Track one frame. rgb = contiguous RGB24 (w*h*3). Returns the per-frame result +// (box normalized [0,1]); valid==0 when the target is lost this frame. +et_result edgetam_capi_track(edgetam_tracker_t t, const uint8_t* rgb, int w, int h); + +// RFD 0011 U8 mask export — copy the LAST track's binary mask (0/255, row-major, +// w*h bytes, original-frame resolution) into `out` (capacity `cap`). Writes dims +// to *w,*h. Returns the byte count (w*h); if cap < w*h nothing is copied and w*h +// is returned so the caller can size its buffer. 0 => no mask (lost / pre-track). +// Valid after edgetam_capi_track / edgetam_capi_track_slot on the same handle. +int edgetam_capi_last_mask(edgetam_tracker_t t, uint8_t* out, int cap, int* w, int* h); + +// RFD 0011 U8 Wave 5 — encoder-ahead threading split. pool_size returns the slot +// count (0 if CoreML/threading is unavailable). encode_slot (PRODUCER thread) +// preprocesses+encodes rgb into slot's neck buffers via the producer's own encoder +// handle. track_slot (CONSUMER thread) consumes slot's prefetched neck + propagates. +// Go owns slot free/ready coordination (channels) — one writer then one reader per +// slot, so these are safe to overlap across the two threads on DIFFERENT slots. +int edgetam_capi_pool_size(edgetam_tracker_t t); +int edgetam_capi_encode_slot(edgetam_tracker_t t, int slot, const uint8_t* rgb, int w, int h); +et_result edgetam_capi_track_slot(edgetam_tracker_t t, int slot, const uint8_t* rgb, int w, int h); + +void edgetam_capi_reset(edgetam_tracker_t t); +void edgetam_capi_destroy(edgetam_tracker_t t); +const char* edgetam_capi_version(void); // SAM3_VERSION, for cgo link sanity + +#ifdef __cplusplus +} +#endif + +#endif // EDGETAM_CAPI_H diff --git a/coreml/edgetam_coreml.mm b/coreml/edgetam_coreml.mm index 9343daa..98fa6ab 100644 --- a/coreml/edgetam_coreml.mm +++ b/coreml/edgetam_coreml.mm @@ -226,9 +226,11 @@ int edgetam_coreml_memencode(edgetam_coreml_handle handle, id out = [h->model predictionFromFeatures:fp error:&err]; if (err || !out) { NSLog(@"[edgetam_coreml] memencode predict: %@", err); return 0; } - // Tuple output flattened to var_486 (latents) + var_488 (pos), each [1,512,64]. - bool ok = copy_f32([[out featureValueForName:@"var_486"] multiArrayValue], mem_feats, (size_t)512*64) - && copy_f32([[out featureValueForName:@"var_488"] multiArrayValue], mem_pos, (size_t)512*64); + // Stable named outputs from convert_memenc_coreml.py: mem_feats (latents) + + // mem_pos (position), each [1,512,64]. (Was var_486/var_488 — coremltools + // auto-names change per export; the convert now pins these names.) + bool ok = copy_f32([[out featureValueForName:@"mem_feats"] multiArrayValue], mem_feats, (size_t)512*64) + && copy_f32([[out featureValueForName:@"mem_pos"] multiArrayValue], mem_pos, (size_t)512*64); return ok ? 1 : 0; } } diff --git a/coreml/export/convert_memenc_coreml.py b/coreml/export/convert_memenc_coreml.py index 1df9e20..386117e 100644 --- a/coreml/export/convert_memenc_coreml.py +++ b/coreml/export/convert_memenc_coreml.py @@ -104,13 +104,23 @@ def forward_2d(x): class MemEnc(nn.Module): """memory_encoder + spatial_perceiver, the per-frame memory-write compute (stage 4).""" - def __init__(self, mem_enc, perceiver): + def __init__(self, mem_enc, perceiver, sigmoid_scale=1.0, sigmoid_bias=0.0): super().__init__() self.mem_enc = mem_enc self.perceiver = perceiver + self.sigmoid_scale = sigmoid_scale + self.sigmoid_bias = sigmoid_bias def forward(self, pix_feat, mask_logits): - out = self.mem_enc(pix_feat, mask_logits, skip_mask_sigmoid=False) + # FIX (RFD 0011 U8): replicate _encode_new_memory's mask processing exactly — + # sigmoid(mask) * sigmoid_scale_for_mem_enc + sigmoid_bias_for_mem_enc + # (EdgeTAM: 20.0 / -10.0), THEN skip the encoder's own sigmoid. The prior + # export used skip_mask_sigmoid=False (sigmoid only, no scale/bias), feeding the + # encoder a [0,1] mask vs the tracker's [-10,10] — which broke the slot + # (goldeneval 0.000 / proto 0.022). With scale+bias baked in, the model takes + # RAW logits and matches the tracker (proto 1.000; C++ 4-stage 0.94 @ ~14fps). + mask_for_mem = torch.sigmoid(mask_logits) * self.sigmoid_scale + self.sigmoid_bias + out = self.mem_enc(pix_feat, mask_for_mem, skip_mask_sigmoid=True) feats = out["vision_features"] pos = out["vision_pos_enc"][0] feats, pos = self.perceiver(feats, pos) @@ -119,7 +129,9 @@ def forward(self, pix_feat, mask_logits): def main(): model = build_model() - w = MemEnc(model.memory_encoder, model.spatial_perceiver).eval() + w = MemEnc(model.memory_encoder, model.spatial_perceiver, + sigmoid_scale=float(model.sigmoid_scale_for_mem_enc), + sigmoid_bias=float(model.sigmoid_bias_for_mem_enc)).eval() pix_feat = torch.randn(1, C_PIX, H, W) mask = torch.randn(1, 1, 1024, 1024) @@ -157,6 +169,9 @@ def main(): frozen, inputs=[ct.TensorType(name="pix_feat", shape=pix_feat.shape, dtype=np.float32), ct.TensorType(name="mask_logits", shape=mask.shape, dtype=np.float32)], + # Stable output names so the .mm/cgo bridge doesn't depend on coremltools + # auto-naming (var_NNN), which changes per export. (feats, pos) order. + outputs=[ct.TensorType(name="mem_feats"), ct.TensorType(name="mem_pos")], minimum_deployment_target=ct.target.iOS17, compute_units=ct.ComputeUnit.ALL, convert_to="mlprogram", diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 70fb9be..cd8e9f1 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -26,6 +26,15 @@ target_compile_definitions(sam3_edgetam_bench PRIVATE SAM3_TIMING) if(SAM3_COREML) add_executable(sam3_coreml_pipeline sam3_coreml_pipeline.cpp) target_link_libraries(sam3_coreml_pipeline PRIVATE sam3) + + # RFD 0011 U8: threaded (encoder-ahead) real-bank tracker bench. + add_executable(sam3_coreml_tracker_threaded sam3_coreml_tracker_threaded.cpp) + target_link_libraries(sam3_coreml_tracker_threaded PRIVATE sam3) + target_compile_definitions(sam3_coreml_tracker_threaded PRIVATE SAM3_TIMING) + + # RFD 0011 U8 Wave 5: encoder-ahead via the cgo C-ABI (capi encode_slot/track_slot). + add_executable(sam3_coreml_capi_threaded sam3_coreml_capi_threaded.cpp) + target_link_libraries(sam3_coreml_capi_threaded PRIVATE sam3) endif() # SDL2 + ImGui are optional — build only if SDL2 is found. diff --git a/examples/sam3_coreml_capi_threaded.cpp b/examples/sam3_coreml_capi_threaded.cpp new file mode 100644 index 0000000..59fb6c2 --- /dev/null +++ b/examples/sam3_coreml_capi_threaded.cpp @@ -0,0 +1,151 @@ +// RFD 0011 U8 Wave 5 — verify the capi encoder-ahead split end-to-end through the +// SAME C-ABI cgo will call (edgetam_capi_create/seed/encode_slot/track_slot), so a +// green fps + goldeneval here de-risks the Go wiring. Producer thread: encode_slot; +// consumer (main): track_slot. Same {"preds":{...}} JSON as sam3_edgetam_bench. +// +// Run (CoreML dir holds the 4 .mlpackages; ggml model is the seed checkpoint): +// sam3_coreml_capi_threaded --models-dir-coreml --ggml-model \ +// --video --prompt --n-frames 120 --dump-json +#include "../coreml/edgetam_capi.h" +#include "sam3.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool parse_box_json(const std::string& path, et_box& out) { + std::ifstream f(path); if (!f) return false; + std::stringstream ss; ss << f.rdbuf(); std::string s = ss.str(); + size_t lb = s.find('[', s.find("box")); if (lb == std::string::npos) return false; + size_t rb = s.find(']', lb); if (rb == std::string::npos) return false; + std::string in = s.substr(lb + 1, rb - lb - 1); + for (char& c : in) if (c == ',') c = ' '; + std::stringstream vs(in); float v[4]; int n = 0; + while (n < 4 && (vs >> v[n])) ++n; + if (n != 4) return false; + out = {v[0], v[1], v[2], v[3]}; return true; +} + +static const char* state_name(int s) { + switch (s) { case 0: return "tracked"; case 1: return "at_risk"; case 2: return "occluded"; + case 3: return "lost"; case 4: return "candidate_reacquire"; case 5: return "reacquired"; + default: return "lost"; } +} + +// Bounded blocking queue of pool-slot ids (SPSC producer<->consumer). +struct IQueue { + std::queue q; std::mutex m; std::condition_variable cv; + void push(int v) { { std::lock_guard l(m); q.push(v); } cv.notify_one(); } + int pop() { std::unique_lock l(m); cv.wait(l, [&] { return !q.empty(); }); int v = q.front(); q.pop(); return v; } +}; + +int main(int argc, char** argv) { + std::string coreml_dir, ggml_model = "models/edgetam_f16.ggml", video, prompt, dump; + int n_frames = 120; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto nx = [&] { return (i + 1 < argc) ? argv[++i] : ""; }; + if (a == "--models-dir-coreml") coreml_dir = nx(); + else if (a == "--ggml-model") ggml_model = nx(); + else if (a == "--video") video = nx(); + else if (a == "--prompt") prompt = nx(); + else if (a == "--n-frames") n_frames = atoi(nx()); + else if (a == "--dump-json") dump = nx(); + } + if (coreml_dir.empty() || video.empty() || prompt.empty()) { + fprintf(stderr, "need --models-dir-coreml --video --prompt\n"); return 1; + } + + edgetam_tracker_t h = edgetam_capi_create(coreml_dir.c_str(), ggml_model.c_str(), 1, 1); + if (!h) { fprintf(stderr, "ERROR: edgetam_capi_create failed\n"); return 1; } + const int POOL = edgetam_capi_pool_size(h); + if (POOL <= 0) { fprintf(stderr, "ERROR: threading unavailable (CoreML off?)\n"); return 1; } + + auto vinfo = sam3_get_video_info(video); + if (vinfo.n_frames <= 0) { fprintf(stderr, "ERROR: read video\n"); return 1; } + if (n_frames > vinfo.n_frames) n_frames = vinfo.n_frames; + et_box init_box; + if (!parse_box_json(prompt, init_box)) { fprintf(stderr, "ERROR: --prompt box.json\n"); return 1; } + + std::vector frames(n_frames); + for (int f = 0; f < n_frames; ++f) { + frames[f] = sam3_decode_video_frame(video, f); + if (frames[f].data.empty()) { fprintf(stderr, "ERROR: decode frame %d\n", f); return 1; } + } + const int w = frames[0].width, hgt = frames[0].height; + const float fw = (float)w, fh = (float)hgt; + + if (!edgetam_capi_seed(h, frames[0].data.data(), w, hgt, init_box)) { + fprintf(stderr, "ERROR: seed failed\n"); return 1; + } + + struct Rec { int frame; std::string state; float b[4]; }; + std::vector recs; recs.reserve(n_frames); + recs.push_back({0, "tracked", {init_box.x0 / fw, init_box.y0 / fh, init_box.x1 / fw, init_box.y1 / fh}}); + + IQueue freeq, readyq; + for (int i = 0; i < POOL; ++i) freeq.push(i); + + // Producer: encode frames 1..N-1 in order into free slots. + std::thread producer([&] { + for (int f = 1; f < n_frames; ++f) { + int slot = freeq.pop(); + edgetam_capi_encode_slot(h, slot, frames[f].data.data(), w, hgt); + readyq.push(slot); + } + }); + + // Consumer (main): track each frame using its prefetched neck. + auto t0 = std::chrono::high_resolution_clock::now(); + for (int f = 1; f < n_frames; ++f) { + int slot = readyq.pop(); + et_result r = edgetam_capi_track_slot(h, slot, frames[f].data.data(), w, hgt); + Rec rec{f, r.valid ? state_name(r.state) : "lost", + {r.box.x0, r.box.y0, r.box.x1, r.box.y1}}; + recs.push_back(rec); + freeq.push(slot); + } + auto t1 = std::chrono::high_resolution_clock::now(); + producer.join(); + + // RFD 0011 U8 mask export check: pull the last track's mask via the C-ABI. + { + int mw = 0, mh = 0; + int need = edgetam_capi_last_mask(h, nullptr, 0, &mw, &mh); + if (need > 0) { + std::vector mbuf(need); + edgetam_capi_last_mask(h, mbuf.data(), need, &mw, &mh); + size_t fg = 0; for (auto v : mbuf) if (v) ++fg; + printf("MASK export: last frame %dx%d, %zu foreground px (%.1f%% of frame)\n", + mw, mh, fg, 100.0 * (double)fg / (double)need); + } else { + printf("MASK export: EMPTY on last frame (w=%d h=%d) — mask not populated on cgo path\n", mw, mh); + } + } + double wall_ms = std::chrono::duration(t1 - t0).count(); + printf("\nCAPI THREADED: %.2f fps (%.1f ms over %d hold frames, encoder-ahead via capi)\n", + (n_frames - 1) * 1000.0 / wall_ms, wall_ms, n_frames - 1); + + if (!dump.empty()) { + std::ofstream o(dump); + o << "{\"preds\":{"; + for (size_t i = 0; i < recs.size(); ++i) { + const Rec& r = recs[i]; + o << (i ? "," : "") << "\"" << r.frame << "\":{\"bbox\":[" << r.b[0] << "," << r.b[1] + << "," << r.b[2] << "," << r.b[3] << "],\"state\":\"" << r.state << "\"}"; + } + o << "}}"; + fprintf(stderr, "wrote %s\n", dump.c_str()); + } + edgetam_capi_destroy(h); + return 0; +} diff --git a/examples/sam3_coreml_pipeline.cpp b/examples/sam3_coreml_pipeline.cpp index 9946ad9..2f7a4b6 100644 --- a/examples/sam3_coreml_pipeline.cpp +++ b/examples/sam3_coreml_pipeline.cpp @@ -45,14 +45,17 @@ int main(int argc, char** argv){ const char* GE = envOr("SAM3_COREML_MODELS", "/Users/noah.johnson/iris/audits/goldenclip-eval/coreml_models"); auto P=[&](const char* f){ std::string s=GE; s+="/"; s+=f; return s; }; - // heterogeneous placement (encoder ANE, mem-attn GPU; decoder/memenc default ANE, overridable) - int U_DEC = envUnit("SAM3_DEC_UNIT_N", 1), U_ME = envUnit("SAM3_MENC_UNIT_N", 1); - auto enc = edgetam_coreml_create(P("edgetam_encoder_neck_nhwc.mlpackage").c_str(), 1); - auto ma = edgetam_coreml_create(P("edgetam_memory_attention.mlpackage").c_str(), 2); + // heterogeneous placement — ALL FOUR stages overridable for the M4 compute-unit + // sweep (RFD 0011 U8). 0=ALL 1=ANE 2=GPU 3=CPU. M1-era default: enc ANE, mem-attn + // GPU, decoder/memenc ANE — re-validate on M4 (the optimal units may invert). + int U_ENC = envUnit("SAM3_ENC_UNIT_N", 1), U_MA = envUnit("SAM3_MA_UNIT_N", 2); + int U_DEC = envUnit("SAM3_DEC_UNIT_N", 1), U_ME = envUnit("SAM3_MENC_UNIT_N", 1); + auto enc = edgetam_coreml_create(P("edgetam_encoder_neck_nhwc.mlpackage").c_str(), U_ENC); + auto ma = edgetam_coreml_create(P("edgetam_memory_attention.mlpackage").c_str(), U_MA); auto dec = edgetam_coreml_create(P("edgetam_mask_decoder_nhwc.mlpackage").c_str(), U_DEC); auto me = edgetam_coreml_create(P("edgetam_memory_encode.mlpackage").c_str(), U_ME); - if(!enc||!ma||!dec||!me){ fprintf(stderr,"model load failed; set SAM3_COREML_MODELS=\n"); return 1; } - printf("loaded enc(ANE)+memattn(GPU)+dec(unit%d)+memenc(unit%d) N=%d warmup=%d\n", U_DEC, U_ME, N, WARMUP); + if(!enc||!ma||!dec||!me){ fprintf(stderr,"model load failed (enc=%d ma=%d dec=%d me=%d); a NULL = that stage's unit rejected the model\n", !!enc,!!ma,!!dec,!!me); return 1; } + printf("loaded enc(unit%d)+memattn(unit%d)+dec(unit%d)+memenc(unit%d) N=%d warmup=%d\n", U_ENC, U_MA, U_DEC, U_ME, N, WARMUP); const size_t S_IMG=3*1024*1024, S_N0=256*256*256, S_N1=128*128*256, S_N2=64*64*256; const size_t S_CURR=4096*256, S_MEM=3648*64, S_PE=64*64*256, S_SP=256, S_MASKF=1024*1024; diff --git a/examples/sam3_coreml_tracker_threaded.cpp b/examples/sam3_coreml_tracker_threaded.cpp new file mode 100644 index 0000000..635c49f --- /dev/null +++ b/examples/sam3_coreml_tracker_threaded.cpp @@ -0,0 +1,197 @@ +// RFD 0011 U8 — THREADED EdgeTAM tracker bench (encoder-ahead pipelining). +// +// The real tracker (preprocess + 3648-token assembly + 4 CoreML stages + bank) is +// ~69 ms/frame single-threaded (~14.5 fps). The encoder leg (preprocess + CoreML +// encode, ~17 ms) is bank-independent, so a producer thread runs it for frame N+1 +// while the main thread runs the consumer (assembly + memattn + decode + memencode +// + bank, via sam3_propagate_frame) for frame N — overlapping the encoder to push +// toward 20 fps. The producer owns its own CoreML encoder handle (per-thread MLModel +// ownership); it hands each frame's 3 neck levels to the library via +// sam3_coreml_set_prefetched_neck, and the consumer's encode step consumes them +// (skipping preprocess + encode). Same goldeneval JSON as sam3_edgetam_bench. +// +// Run (all 4 stages on CoreML + PAD_BANK, like the single-threaded 14.5 fps config): +// SAM3_COREML_PAD_BANK=1 SAM3_COREML_ENCODER=1 SAM3_COREML_MODEL=.../encoder.mlpackage \ +// SAM3_COREML_MEMATTN=1 SAM3_COREML_MEMATTN_MODEL=.../memattn.mlpackage \ +// SAM3_COREML_DECODER=1 SAM3_COREML_DECODER_MODEL=.../decoder.mlpackage \ +// SAM3_COREML_MEMENC=1 SAM3_COREML_MEMENC_MODEL=.../memencode.mlpackage \ +// sam3_coreml_tracker_threaded --models-dir --video --n-frames 120 \ +// --prompt --dump-json +#include "sam3.h" +#include "../coreml/edgetam_coreml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +static std::string discover_edgetam_model(const std::string& dir) { +#ifndef _WIN32 + DIR* d = opendir(dir.c_str()); + if (!d) return ""; + std::string best; + for (struct dirent* e; (e = readdir(d)) != nullptr;) { + std::string f = e->d_name; + if (f.size() > 5 && f.substr(f.size() - 5) == ".ggml" && f.find("edgetam") != std::string::npos) { + best = dir + "/" + f; break; + } + } + closedir(d); + return best; +#else + return dir + "/edgetam_f16.ggml"; +#endif +} + +static bool parse_box_json(const std::string& path, sam3_box& out) { + std::ifstream f(path); if (!f) return false; + std::stringstream ss; ss << f.rdbuf(); std::string s = ss.str(); + size_t lb = s.find('[', s.find("box")); if (lb == std::string::npos) return false; + size_t rb = s.find(']', lb); if (rb == std::string::npos) return false; + std::string in = s.substr(lb + 1, rb - lb - 1); + for (char& c : in) if (c == ',') c = ' '; + std::stringstream vs(in); float v[4]; int n = 0; + while (n < 4 && (vs >> v[n])) ++n; + if (n != 4) return false; + out = {v[0], v[1], v[2], v[3]}; return true; +} + +// Bounded blocking queue of pool-slot indices (SPSC producer<->consumer). +struct IQueue { + std::queue q; std::mutex m; std::condition_variable cv; + void push(int v) { { std::lock_guard l(m); q.push(v); } cv.notify_one(); } + int pop() { std::unique_lock l(m); cv.wait(l, [&] { return !q.empty(); }); int v = q.front(); q.pop(); return v; } +}; + +int main(int argc, char** argv) { + std::string models_dir = "models/", video_path, prompt_path, dump_path; + int n_frames = 120, warmup = 35; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto nx = [&] { return (i + 1 < argc) ? argv[++i] : ""; }; + if (a == "--models-dir") models_dir = nx(); + else if (a == "--video") video_path = nx(); + else if (a == "--n-frames") n_frames = atoi(nx()); + else if (a == "--warmup") warmup = atoi(nx()); + else if (a == "--prompt") prompt_path = nx(); + else if (a == "--dump-json") dump_path = nx(); + } + const char* enc_model = getenv("SAM3_COREML_MODEL"); + if (!enc_model) { fprintf(stderr, "ERROR: SAM3_COREML_MODEL (encoder .mlpackage) required\n"); return 1; } + + std::string model_path = discover_edgetam_model(models_dir); + sam3_params params; params.model_path = model_path; params.use_gpu = true; + auto model = sam3_load_model(params); + if (!model) { fprintf(stderr, "ERROR: load model %s\n", model_path.c_str()); return 1; } + auto state = sam3_create_state(*model, params); + sam3_visual_track_params vtp; vtp.max_keep_alive = 1000; vtp.recondition_every = 16; + auto tracker = sam3_create_visual_tracker(*model, vtp); + + auto vinfo = sam3_get_video_info(video_path); + if (vinfo.n_frames <= 0) { fprintf(stderr, "ERROR: read video %s\n", video_path.c_str()); return 1; } + if (n_frames > vinfo.n_frames) n_frames = vinfo.n_frames; + sam3_box init_box; + if (prompt_path.empty() || !parse_box_json(prompt_path, init_box)) { + fprintf(stderr, "ERROR: --prompt box.json required\n"); return 1; + } + + std::vector frames(n_frames); + for (int f = 0; f < n_frames; ++f) { + frames[f] = sam3_decode_video_frame(video_path, f); + if (frames[f].data.empty()) { fprintf(stderr, "ERROR: decode frame %d\n", f); return 1; } + } + const float fw = (float)frames[0].width, fh = (float)frames[0].height; + const int IMG = 1024; // EdgeTAM encoder input size + + // Producer's own encoder handle (ANE) — per-thread MLModel ownership. + auto prod_enc = edgetam_coreml_create(enc_model, /*ANE*/ 1); + if (!prod_enc) { fprintf(stderr, "ERROR: producer encoder load %s\n", enc_model); return 1; } + + // Neck pool (channels-last [1,H,W,256]); POOL slots let the producer run ahead. + const int POOL = 3, D = 256, Wd[3] = {256, 128, 64}; + struct Slot { std::vector n[3]; int frame; }; + std::vector pool(POOL); + for (auto& s : pool) for (int i = 0; i < 3; ++i) s.n[i].assign((size_t)D * Wd[i] * Wd[i], 0.f); + IQueue freeq, readyq; + for (int i = 0; i < POOL; ++i) freeq.push(i); + + // Producer: preprocess + CoreML encode every frame (0..N-1), in order. + std::thread producer([&] { + for (int f = 0; f < n_frames; ++f) { + int slot = freeq.pop(); + std::vector img = sam3_coreml_preprocess_image(frames[f], IMG); + edgetam_coreml_encode(prod_enc, img.data(), + pool[slot].n[0].data(), pool[slot].n[1].data(), pool[slot].n[2].data()); + pool[slot].frame = f; + readyq.push(slot); + } + }); + + // Consumer (main): seed on frame 0, then propagate 1..N-1, all using prefetched necks. + struct Rec { int frame; bool tracked; std::string state; float b[4]; float conf; }; + std::vector recs; recs.reserve(n_frames); + + int slot0 = readyq.pop(); // frame 0 + sam3_coreml_set_prefetched_neck(pool[slot0].n[0].data(), pool[slot0].n[1].data(), pool[slot0].n[2].data()); + sam3_encode_image(*state, *model, frames[0]); + (void)sam3_take_frame_timing(); + sam3_pvs_params pvs; pvs.box = init_box; pvs.use_box = true; + int inst = sam3_tracker_add_instance(*tracker, *state, *model, pvs); + if (inst < 0) { fprintf(stderr, "ERROR: seed add_instance failed\n"); return 1; } + (void)sam3_take_frame_timing(); + freeq.push(slot0); + recs.push_back({0, true, "tracked", {init_box.x0 / fw, init_box.y0 / fh, init_box.x1 / fw, init_box.y1 / fh}, 1.f}); + + auto wall0 = std::chrono::high_resolution_clock::now(); + for (int f = 1; f < n_frames; ++f) { + int slot = readyq.pop(); // frame f (producer delivers in order) + sam3_coreml_set_prefetched_neck(pool[slot].n[0].data(), pool[slot].n[1].data(), pool[slot].n[2].data()); + sam3_result res = sam3_propagate_frame(*tracker, *state, *model, frames[f]); + (void)sam3_take_frame_timing(); + Rec r{f, false, "lost", {0, 0, 0, 0}, 0.f}; + if (!res.detections.empty()) { + const sam3_detection* d = &res.detections[0]; + for (const auto& x : res.detections) if (x.instance_id == inst) { d = &x; break; } + r.tracked = true; r.state = sam3_target_state_name(d->state); + r.b[0] = d->box.x0 / fw; r.b[1] = d->box.y0 / fh; r.b[2] = d->box.x1 / fw; r.b[3] = d->box.y1 / fh; + r.conf = d->score; + } + recs.push_back(r); + freeq.push(slot); + } + auto wall1 = std::chrono::high_resolution_clock::now(); + producer.join(); + double wall_ms = std::chrono::duration(wall1 - wall0).count(); + double wall_fps = (n_frames - 1) * 1000.0 / wall_ms; + + printf("\nTHREADED tracker: %.2f fps (%.1f ms over %d hold frames, encoder-ahead)\n", + wall_fps, wall_ms, n_frames - 1); + + if (!dump_path.empty()) { + std::ofstream o(dump_path); + o << "{\"preds\":{"; + for (size_t i = 0; i < recs.size(); ++i) { + const Rec& r = recs[i]; + o << (i ? "," : "") << "\"" << r.frame << "\":{\"bbox\":[" << r.b[0] << "," << r.b[1] + << "," << r.b[2] << "," << r.b[3] << "],\"state\":\"" << r.state << "\"}"; + } + o << "}}"; + fprintf(stderr, "wrote %s\n", dump_path.c_str()); + } + edgetam_coreml_destroy(prod_enc); + return 0; +} diff --git a/sam3.cpp b/sam3.cpp index 1506b32..8947d16 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -12,6 +12,12 @@ #include "ggml-metal.h" #endif +// RFD 0011 (Windows/Linux cross-vendor GPU): the ggml-Vulkan backend covers +// AMD/NVIDIA/Intel from one build (SAM3_VULKAN). The model loader selects it below. +#ifdef GGML_USE_VULKAN +#include "ggml-vulkan.h" +#endif + /* stb (implementation compiled here -- order is pinned) */ #define STB_IMAGE_IMPLEMENTATION #include "stb_image.h" @@ -3424,6 +3430,14 @@ std::shared_ptr sam3_load_model(const sam3_params& params) { fprintf(stderr, "%s: using Metal backend\n", __func__); model->backend = ggml_backend_metal_init(); } +#endif +#ifdef GGML_USE_VULKAN + // RFD 0011: cross-vendor GPU. ggml-Vulkan runs the same EdgeTAM graph on any + // Vulkan device (AMD/NVIDIA/Intel; Apple via MoltenVK). device 0 = first GPU. + if (params.use_gpu && !model->backend) { + fprintf(stderr, "%s: using Vulkan backend\n", __func__); + model->backend = ggml_backend_vk_init(0); + } #endif if (!model->backend) { fprintf(stderr, "%s: using CPU backend\n", __func__); @@ -4974,6 +4988,40 @@ static void edgetam_build_repvit_graph(struct ggml_context* ctx, #ifdef SAM3_COREML #include "edgetam_coreml.h" + +// RFD 0011 U8 (threaded encoder-ahead): the producer thread runs preprocess + +// CoreML encode for frame N+1 into s_prefetch_neck while the main thread runs the +// consumer (assembly+memattn+decode+memencode+bank) for frame N. When a prefetched +// neck is ready, the main thread's encode step copies it (skipping BOTH preprocess +// and the CoreML encode), so the ~17 ms encoder leg overlaps the ~52 ms consumer. +// SPSC: the bench serializes set→consume per frame, so no lock is needed here. +static std::vector s_prefetch_neck[3]; +static bool s_prefetch_ready = false; + +// RFD 0011 U8 — per-chip CoreML compute-unit profile (0=ALL 1=ANE 2=GPU 3=CPU). +// M4 Pro sweep (sam3_coreml_pipeline, isolated 20-rep median): encoder ANE 11ms +// (GPU 14, CPU 38), mem-attn GPU 17ms (ANE 26, CPU 24), decoder ANE 7.8ms (GPU 8.2), +// memenc ANE 5.0 / GPU 3.9. Defaults = the validated M4 optimum (enc ANE, mem-attn +// GPU, decoder ANE, memenc ANE — memenc's isolated GPU edge washes out in thermal +// noise end-to-end). NOT inverted vs M1 like the ORT/EfficientTAM stack. Override +// per chip via env (e.g. SAM3_COREML_MA_UNIT=1) if M1/M2/M3 profiles differ. +static int sam3_coreml_unit(const char* env, int def) { + const char* v = getenv(env); + return (v && *v) ? atoi(v) : def; +} + +void sam3_coreml_set_prefetched_neck(const float* n0, const float* n1, const float* n2) { + const int Wd[3] = {256, 128, 64}, D = 256; + const float* src[3] = {n0, n1, n2}; + for (int i = 0; i < 3; ++i) + s_prefetch_neck[i].assign(src[i], src[i] + (size_t)D * Wd[i] * Wd[i]); + s_prefetch_ready = true; +} +// Exposed for the producer thread (preprocess off the main thread). +std::vector sam3_coreml_preprocess_image(const sam3_image& image, int img_size) { + return sam2_preprocess_image(image, img_size); +} + // RFD 0011 (CoreML/ANE hybrid): run the EdgeTAM encoder on CoreML/ANE (~12 ms) // instead of the ggml/Metal RepViT (~122 ms) and load the ggml-matching 256-ch // neck features into state.neck_trk. The CoreML model exports neck(trunk(x))[0:3] @@ -4984,10 +5032,10 @@ static bool edgetam_encode_image_coreml(sam3_state& state, const sam3_model& mod const sam3_image& image, const std::vector& img_norm) { static edgetam_coreml_handle s_enc = nullptr; - if (!s_enc) { + if (!s_prefetch_ready && !s_enc) { // prefetch path needs no local encoder handle const char* mp = getenv("SAM3_COREML_MODEL"); if (!mp) { fprintf(stderr, "%s: SAM3_COREML_ENCODER set but SAM3_COREML_MODEL unset\n", __func__); return false; } - s_enc = edgetam_coreml_create(mp, /*compute_units=ANE*/ 1); + s_enc = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_ENC_UNIT", /*ANE*/ 1)); if (!s_enc) { fprintf(stderr, "%s: CoreML encoder load failed\n", __func__); return false; } fprintf(stderr, "%s: CoreML EdgeTAM encoder loaded (ANE): %s\n", __func__, mp); } @@ -5003,7 +5051,10 @@ static bool edgetam_encode_image_coreml(sam3_state& state, const sam3_model& mod static std::vector nbuf[3]; for (int i = 0; i < 3; ++i) nbuf[i].resize((size_t)D * Wd[i] * Hd[i]); - { + if (s_prefetch_ready) { // threaded: producer already encoded this frame off-thread + for (int i = 0; i < 3; ++i) nbuf[i].swap(s_prefetch_neck[i]); + s_prefetch_ready = false; + } else { SAM3_TIME_SCOPE(image_encoder_compute_ms); // the ANE inference itself if (!edgetam_coreml_encode(s_enc, img_norm.data(), nbuf[0].data(), nbuf[1].data(), nbuf[2].data())) @@ -5066,6 +5117,21 @@ static bool edgetam_encode_image(sam3_state& state, state.orig_width = image.width; state.orig_height = image.height; +#ifdef SAM3_COREML + // RFD 0011 (CoreML/ANE hybrid): when enabled, run the encoder on CoreML/ANE + // and skip the ggml RepViT+FPN graph entirely. Threaded (encoder-ahead): when a + // prefetched neck is ready the producer already did preprocess + encode off-thread, + // so skip preprocess here too — the helper consumes the prefetched neck. + if (getenv("SAM3_COREML_ENCODER")) { + std::vector cml_img; + if (!s_prefetch_ready) { + SAM3_TIME_SCOPE(preprocess_ms); + cml_img = sam2_preprocess_image(image, img_size); // NCHW [1,3,1024,1024] + } + return edgetam_encode_image_coreml(state, model, image, cml_img); + } +#endif + // ── Preprocess (same ImageNet normalization as SAM2) ───────────────── // RFD 0011 U0: preprocess timed separately from the graph build/alloc/compute. std::vector img_data; @@ -5074,16 +5140,6 @@ static bool edgetam_encode_image(sam3_state& state, img_data = sam2_preprocess_image(image, img_size); } -#ifdef SAM3_COREML - // RFD 0011 (CoreML/ANE hybrid): when enabled, run the encoder on CoreML/ANE - // and skip the ggml RepViT+FPN graph entirely. img_data is already NCHW - // [1,3,1024,1024] (ggml inp ne=[W,H,C] == numpy NCHW), so it hands straight - // to CoreML; only the 256-ch neck outputs need a layout permute (in helper). - if (getenv("SAM3_COREML_ENCODER")) { - return edgetam_encode_image_coreml(state, model, image, img_data); - } -#endif - // ── Build graph ────────────────────────────────────────────────────── // RFD 0011 U0: graph CONSTRUCTION region (image_encoder_build_ms). U1 will // build this graph once and reuse it, driving build_ms toward 0; that is @@ -11611,42 +11667,40 @@ static sam3_prop_output sam3_propagate_single( if (ptr_tpos[p] < 1) ptr_tpos[p] = 1; // minimum distance of 1 } +#ifdef SAM3_COREML + // RFD 0011 U8 Wave 1: force full mem-attn capacity (num_maskmem spatial slots + + // max_obj_ptrs pointer slots = 3648 tokens) from frame 1, so the CoreML mem-attn + // engages EVERY frame instead of falling back to ggml on early (not-yet-full) + // frames. Required to test a consistent all-CoreML pipeline (CoreML memencode + // slots must not be fed to the ggml mem-attn). Pad spatial slots by repeating the + // most-recent slot; pad pointers with no_obj_ptr at a far temporal distance. + if (getenv("SAM3_COREML_PAD_BANK") && use_perceiver && !slot_feats.empty()) { + while ((int)slot_feats.size() < hp.num_maskmem) { + slot_feats.push_back(slot_feats.back()); + slot_pes.push_back(slot_pes.back()); + spatial_tpos.push_back(spatial_tpos.back()); + } + if (model.no_obj_ptr) { + std::vector nop(D); + sam3_read_f32(model.no_obj_ptr, nop.data(), D); + while ((int)obj_ptrs.size() < hp.max_obj_ptrs) { + obj_ptrs.push_back(nop); + ptr_tpos.push_back(hp.max_obj_ptrs); + } + } + } +#endif auto pd = sam3_build_prompt_and_pos(model, slot_feats, slot_pes, spatial_tpos, obj_ptrs, ptr_tpos, H); // ── RoPE frequencies (cached) ────────────────────────────────────── sam3_ensure_tracker_pe_caches(tracker, hp, H); const int half_d = D / 2; // 128 const auto& rope_q_reord = tracker.cached_axial_cis_reord; - // For cross-attn K: build rope_k_data for all M_spatial tokens + // RFD 0011 U8 perf: rope_k_data feeds ONLY the ggml mem-attn fallback graph. + // The CoreML mem-attn path (below) uses cached_sinpe_256, never rope_k_data, + // so building it here is ~2 ms/frame of pure waste on the hot path. Moved to + // just before the ggml graph build, so it runs only when CoreML did NOT. std::vector rope_k_data; - if (pd.M_spatial > 0) { - rope_k_data.resize(2 * half_d * pd.M_spatial); - if (use_perceiver) { - // EdgeTAM perceiver: each frame has N_per_slot=512 tokens. - // First 256 (1D latents): identity RoPE (cos=1, sin=0). - // Last 256 (2D latents): 16x16 RoPE from cached_axial_cis_k16_reord. - const int N_1d = hp.perceiver_n_latents_1d; // 256 - const int N_2d = hp.perceiver_n_latents_2d; // 256 - const auto& rope_k16 = tracker.cached_axial_cis_k16_reord; // [2, 128, 256] - for (int s = 0; s < n_sel; ++s) { - float* dst = rope_k_data.data() + s * D * N_per_slot; - // 1D tokens: identity RoPE (cos=1, sin=0) in [2, half_d, N_1d] layout - // Layout: for token n, dim i: cos at [0 + i*2 + n*D], sin at [1 + i*2 + n*D] - for (int n = 0; n < N_1d; ++n) - for (int i = 0; i < half_d; ++i) { - dst[0 + i * 2 + n * D] = 1.0f; // cos = 1 - dst[1 + i * 2 + n * D] = 0.0f; // sin = 0 - } - // 2D tokens: copy 16x16 RoPE - float* dst_2d = dst + D * N_1d; - memcpy(dst_2d, rope_k16.data(), D * N_2d * sizeof(float)); - } - } else { - // Standard: repeat HxH axial CIS for each memory frame - for (int s = 0; s < pd.M_spatial / N; ++s) - memcpy(rope_k_data.data() + s * D * N, rope_q_reord.data(), D * N * sizeof(float)); - } - } // RFD 0011 (CoreML/ANE hybrid): run the memory attention on CoreML-GPU // (~18 ms vs ggml ~185 ms) at the fixed steady-state capacity (7 memory @@ -11667,7 +11721,7 @@ static sam3_prop_output sam3_propagate_single( s_memattn_tried = true; const char* mp = getenv("SAM3_COREML_MEMATTN_MODEL"); if (mp) { - s_memattn = edgetam_coreml_create(mp, /*CPU_AND_GPU*/ 2); + s_memattn = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_MA_UNIT", /*GPU*/ 2)); if (s_memattn) fprintf(stderr, "%s: CoreML mem-attn loaded (GPU): %s\n", __func__, mp); } } @@ -11695,7 +11749,7 @@ static sam3_prop_output sam3_propagate_single( s_decoder_tried = true; const char* mp = getenv("SAM3_COREML_DECODER_MODEL"); if (mp) { - s_decoder = edgetam_coreml_create(mp, /*CPU_AND_NE*/ 1); + s_decoder = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_DEC_UNIT", /*ANE*/ 1)); if (s_decoder) fprintf(stderr, "%s: CoreML decoder loaded (ANE): %s\n", __func__, mp); } } @@ -11731,6 +11785,36 @@ static sam3_prop_output sam3_propagate_single( } #endif + // RFD 0011 U8 perf: build rope_k_data HERE — only reached when the CoreML + // mem-attn/decoder path above did NOT return (fallback to the ggml graph). + if (pd.M_spatial > 0) { + rope_k_data.resize(2 * half_d * pd.M_spatial); + if (use_perceiver) { + // EdgeTAM perceiver: each frame has N_per_slot=512 tokens. + // First 256 (1D latents): identity RoPE (cos=1, sin=0). + // Last 256 (2D latents): 16x16 RoPE from cached_axial_cis_k16_reord. + const int N_1d = hp.perceiver_n_latents_1d; // 256 + const int N_2d = hp.perceiver_n_latents_2d; // 256 + const auto& rope_k16 = tracker.cached_axial_cis_k16_reord; // [2, 128, 256] + for (int s = 0; s < n_sel; ++s) { + float* dst = rope_k_data.data() + s * D * N_per_slot; + // 1D tokens: identity RoPE (cos=1, sin=0) in [2, half_d, N_1d] layout + for (int n = 0; n < N_1d; ++n) + for (int i = 0; i < half_d; ++i) { + dst[0 + i * 2 + n * D] = 1.0f; // cos = 1 + dst[1 + i * 2 + n * D] = 0.0f; // sin = 0 + } + // 2D tokens: copy 16x16 RoPE + float* dst_2d = dst + D * N_1d; + memcpy(dst_2d, rope_k16.data(), D * N_2d * sizeof(float)); + } + } else { + // Standard: repeat HxH axial CIS for each memory frame + for (int s = 0; s < pd.M_spatial / N; ++s) + memcpy(rope_k_data.data() + s * D * N, rope_q_reord.data(), D * N * sizeof(float)); + } + } + // ── Build graph ───────────────────────────────────────────────────── // RFD 0011 U0: mem-attn + SAM mask decoder graph CONSTRUCTION region // (mem_attn_build_ms). NOTE: mem-attn and the mask decoder are built into @@ -12106,6 +12190,83 @@ static bool sam3_encode_memory( // Mask preprocessing: mask_logits → HIGH_RES → sigmoid → scale/bias → INTERPOL auto m_hires = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES); +#ifdef SAM3_COREML + // RFD 0011 U8 Wave 1: CoreML memory-encode (memory_encoder + spatial_perceiver, + // ~6ms ANE vs ~24ms ggml). ⚠️ DO NOT USE IN THE ggml-ORCHESTRATED TRACKER. + // SPEED VERIFIED (7.8→10.86 fps, mem_encoder 24→0ms) but ACCURACY BROKEN + // (goldeneval dt0004 0.000). DIAGNOSED via the parity probe below: vs the ggml + // edgetam_perceiver_forward slot, pos cos=1.000 but **feats cos≈0.85** on + // IDENTICAL pix_feat+mask (pix_feat HWC→BCHW transpose verified — the H/W-swapped + // variant scores 0.43, far worse; mask sigmoid/scale is not the driver). So the + // gap is a genuine REPRESENTATION difference between the ggml perceiver and the + // CoreML spatial_perceiver (different magnitudes). The ggml assembly + tpos + + // mem-attn are calibrated to the ggml representation, so a 0.85-different slot + // breaks the track. CONCLUSION: 20fps needs a FULLY pure-CoreML tracker (the + // sam3_coreml_pipeline path + a real bank + ggml-seed), not this hybrid drop-in. + // Gated OFF by default (env SAM3_COREML_MEMENC); the ggml memencode below is the + // correct path for the hybrid. + // The exported model applies the mask sigmoid internally + // (convert_memenc_coreml.py: skip_mask_sigmoid=False), so it takes RAW logits at + // HIGH_RES — i.e. m_hires BEFORE the sigmoid below. pix_feat is neck2 transposed + // from ggml HWC (channel-fastest) to BCHW. Output is the perceiver slot + // (== the ggml path's perc_latents/perc_pos), stored identically. Returns early, + // skipping the per-frame ggml memencode graph entirely. + if (getenv("SAM3_COREML_MEMENC") && hp.has_perceiver) { + static edgetam_coreml_handle s_memenc = nullptr; + static bool s_memenc_tried = false; + if (!s_memenc && !s_memenc_tried) { + s_memenc_tried = true; + const char* mp = getenv("SAM3_COREML_MEMENC_MODEL"); + if (mp) { + // memenc GPU is ~1.1ms faster in isolation (M4 bench: 3.9 vs ANE 5.0) but the + // win is WITHIN thermal run-to-run noise end-to-end (A/B inconclusive) and + // accuracy is identical, so the default stays ANE (validated). Tune per-rig + // via SAM3_COREML_MENC_UNIT if a thermally-stable bench shows a real win. + s_memenc = edgetam_coreml_create(mp, sam3_coreml_unit("SAM3_COREML_MENC_UNIT", /*ANE*/ 1)); + if (s_memenc) fprintf(stderr, "%s: CoreML memenc loaded (ANE): %s\n", __func__, mp); + } + } + if (s_memenc) { + std::vector n2((size_t)D * H * H), pix_chw((size_t)D * H * H); + ggml_backend_tensor_get(state.neck_trk[2], n2.data(), 0, (size_t)D * H * H * sizeof(float)); + for (int h = 0; h < H; ++h) + for (int w = 0; w < H; ++w) + for (int c = 0; c < D; ++c) + pix_chw[(size_t)c * H * H + (size_t)h * H + w] = n2[(size_t)c + ((size_t)h * H + w) * D]; + const int N_perc = hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d; // 512 + std::vector mfeat((size_t)MD * N_perc), mpos((size_t)MD * N_perc); + if (edgetam_coreml_memencode(s_memenc, pix_chw.data(), m_hires.data(), mfeat.data(), mpos.data())) { + if (!tracker.ctx) { + struct ggml_init_params tp = {ggml_tensor_overhead() * 4096, nullptr, true}; + tracker.ctx = ggml_init(tp); + } + auto store_slot = [&](const std::vector& src) { + auto* t = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); + auto* b = ggml_backend_alloc_buffer(model.backend, (size_t)MD * N_perc * sizeof(float)); + struct ggml_tallocr a = ggml_tallocr_new(b); + ggml_tallocr_alloc(&a, t); + tracker.owned_buffers.push_back(b); + ggml_backend_tensor_set(t, src.data(), 0, (size_t)MD * N_perc * sizeof(float)); + return t; + }; + sam3_memory_slot slot; + slot.spatial_feats = store_slot(mfeat); + slot.spatial_pe = store_slot(mpos); + slot.frame_index = frame_idx; + slot.is_cond_frame = is_cond; + auto& bk = tracker.mem_banks[inst_id]; + bk.push_back(slot); + while ((int)bk.size() > sam3_mem_pool_cap(hp.num_maskmem)) { + bool removed = false; + for (auto it = bk.begin(); it != bk.end(); ++it) + if (!it->is_cond_frame) { bk.erase(it); removed = true; break; } + if (!removed) bk.erase(bk.begin() + 1); + } + return true; + } + } + } +#endif const float sig_scale = hp.sigmoid_scale(), sig_bias = hp.sigmoid_bias(); for (auto& v : m_hires) { float s = 1.0f / (1.0f + expf(-v)); v = s * sig_scale + sig_bias; } auto m_interp = sam3_bilinear_interpolate(m_hires.data(), HIGH_RES, HIGH_RES, INTERPOL, INTERPOL); @@ -12240,8 +12401,48 @@ static bool sam3_encode_memory( return false; } - // Store perceiver output: [MD=64, N_perc] (N_1d + N_2d latents) const int N_perc = hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d; +#ifdef SAM3_COREML + // RFD 0011 U8 Wave 1: memencode parity probe. Run the ggml path AND set + // SAM3_COREML_MEMENC_PARITY(+_MODEL) to cosine-compare the CoreML memencode + // slot against this ggml perceiver slot, per axis, to find the wrong one. + if (getenv("SAM3_COREML_MEMENC_PARITY")) { + static edgetam_coreml_handle s_pe = nullptr; + static bool s_pe_tried = false; + if (!s_pe && !s_pe_tried) { + s_pe_tried = true; + const char* mp = getenv("SAM3_COREML_MEMENC_MODEL"); + if (mp) s_pe = edgetam_coreml_create(mp, 1); + } + if (s_pe) { + auto raw = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES); + std::vector n2((size_t)D * H * H), pc((size_t)D * H * H); + ggml_backend_tensor_get(state.neck_trk[2], n2.data(), 0, (size_t)D * H * H * sizeof(float)); + for (int hh = 0; hh < H; ++hh) + for (int ww = 0; ww < H; ++ww) + for (int cc = 0; cc < D; ++cc) + pc[(size_t)cc * H * H + (size_t)hh * H + ww] = n2[(size_t)cc + ((size_t)hh * H + ww) * D]; + auto cosab = [](const std::vector& a, const std::vector& b) { + double d = 0, na = 0, nb = 0; size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) { d += (double)a[i] * b[i]; na += (double)a[i] * a[i]; nb += (double)b[i] * b[i]; } + return (na > 0 && nb > 0) ? d / (sqrt(na) * sqrt(nb)) : 0.0; + }; + std::vector mf((size_t)MD * N_perc), mp2((size_t)MD * N_perc); + edgetam_coreml_memencode(s_pe, pc.data(), raw.data(), mf.data(), mp2.data()); // current transpose (c,h,w) + // pix_feat with H/W swapped (write to (c,w,h)) — tests a spatial transpose bug + std::vector pcS((size_t)D * H * H); + for (int hh = 0; hh < H; ++hh) + for (int ww = 0; ww < H; ++ww) + for (int cc = 0; cc < D; ++cc) + pcS[(size_t)cc * H * H + (size_t)ww * H + hh] = n2[(size_t)cc + ((size_t)hh * H + ww) * D]; + std::vector mfS((size_t)MD * N_perc), mpS((size_t)MD * N_perc); + edgetam_coreml_memencode(s_pe, pcS.data(), raw.data(), mfS.data(), mpS.data()); + fprintf(stderr, "MEMENC_PARITY f%d: feats cos transpose=%.4f swapHW=%.4f pos cos=%.4f\n", + frame_idx, cosab(perc_latents, mf), cosab(perc_latents, mfS), cosab(perc_pos, mp2)); + } + } +#endif + // Store perceiver output: [MD=64, N_perc] (N_1d + N_2d latents) auto* st = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); auto* sb = ggml_backend_alloc_buffer(model.backend, MD * N_perc * sizeof(float)); struct ggml_tallocr ta_perc = ggml_tallocr_new(sb); @@ -12921,6 +13122,17 @@ sam3_result sam3_propagate_frame( pm[id].data.resize(state.orig_width * state.orig_height); for (int p = 0; p < (int)rs.size(); ++p) pm[id].data[p] = rs[p] > 0.0f ? 255 : 0; + } else { + // RFD 0011 U8 mask export: a cheap LOW-RES binary mask straight off the + // decoder's ~256x256 logit grid (no full-res upscale → keeps the U2 + // fast-path perf). pm carries it to the result detection (~13315) so the + // C-ABI/State-Sync can ship it; the frontend upsamples to the video. + const int mw = po[id].mask_w, mh = po[id].mask_h; + pm[id].width = mw; + pm[id].height = mh; + pm[id].data.resize((size_t)mw * mh); + for (int p = 0; p < mw * mh; ++p) + pm[id].data[p] = po[id].mask_logits[p] > 0.0f ? 255 : 0; } ml.last_score = po[id].iou_scores[0]; ml.last_seen = fi; @@ -13109,8 +13321,10 @@ sam3_result sam3_propagate_frame( det.mask.iou_score = score; auto pit = po.find(inst_id); if (pit != po.end()) det.mask.obj_score = pit->second.obj_score; - // Attach full-res mask pixels only if we built them (debug/overlay). - if (want_fullres) { + // Attach the mask pixels: low-res grid by default (RFD 0011 U8 export, for + // State-Sync/frontend display), full-res when SAM3_FULLRES_MASK. Both were + // built into pm above; emit whichever is present. + { auto it = pm.find(inst_id); if (it != pm.end()) { det.mask.width = it->second.width; diff --git a/sam3.h b/sam3.h index 79c33e5..809d4de 100644 --- a/sam3.h +++ b/sam3.h @@ -359,6 +359,18 @@ bool sam3_save_mask(const sam3_mask & mask, const std::string & path) sam3_image sam3_decode_video_frame(const std::string & video_path, int frame_index); sam3_video_info sam3_get_video_info(const std::string & video_path); +/* +** ── RFD 0011 U8: threaded encoder-ahead (CoreML) ───────────────────────── +** The producer thread preprocesses + CoreML-encodes frame N+1 off the main +** thread, then hands the 3 neck levels here; the next sam3_propagate_frame's +** encode step consumes them (skipping preprocess + encode) so the encoder leg +** overlaps the consumer. Built only with SAM3_COREML; no-ops otherwise. +*/ +void sam3_coreml_set_prefetched_neck(const float* neck0, + const float* neck1, + const float* neck2); +std::vector sam3_coreml_preprocess_image(const sam3_image& image, int img_size); + /***************************************************************************** ** Test and Debug API **