Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions apps/desktop/src/renderer/hooks/useRemoteControl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ export function useRemoteControl({
const heldButtonsRef = useRef<Set<MouseButton>>(new Set());
const heldKeysRef = useRef<Set<string>>(new Set());
const lastCursorUpdateRef = useRef(0);
// Pointer events fire before mouse events in Chromium. Storing the last
// pointer event's timestamp lets the mouse handler skip a double-fire.
const lastPointerEventRef = useRef(0);
const cursorThrottleMs = 16; // ~60fps throttle for cursor updates

// Check if we can send input (enabled, granted control, and capturing)
Expand Down Expand Up @@ -112,6 +115,9 @@ export function useRemoteControl({
(event: MouseEvent) => {
if (!canSendInput) return;

// Skip if a pointer event just fired (Chromium fires both).
if (Date.now() - lastPointerEventRef.current < 100) return;

const coords = getRelativeCoords(event);
if (!coords) return;

Expand Down Expand Up @@ -149,6 +155,9 @@ export function useRemoteControl({
(event: MouseEvent) => {
if (!canSendInput) return;

// Skip if a pointer event just fired.
if (Date.now() - lastPointerEventRef.current < 100) return;

const coords = getRelativeCoords(event);
if (!coords) return;

Expand Down Expand Up @@ -271,6 +280,33 @@ export function useRemoteControl({
[canSendInput]
);

// Pointer events fire before mouse events in Chromium, and on some trackpads
// only pointer events fire at all. Delegate to the mouse handlers, then mark
// the timestamp so the mouse-handler dedup skips the follow-up mouse event.
const handlePointerDown = useCallback(
(event: PointerEvent) => {
lastPointerEventRef.current = Date.now();
handleMouseDown(event as unknown as MouseEvent);
},
[handleMouseDown]
);

const handlePointerUp = useCallback(
(event: PointerEvent) => {
lastPointerEventRef.current = Date.now();
handleMouseUp(event as unknown as MouseEvent);
},
[handleMouseUp]
);

const handlePointerMove = useCallback(
(event: PointerEvent) => {
lastPointerEventRef.current = Date.now();
handleMouseMove(event as unknown as MouseEvent);
},
[handleMouseMove]
);

// Start capturing input
const startCapture = useCallback(() => {
setIsCapturing(true);
Expand All @@ -294,6 +330,10 @@ export function useRemoteControl({
container.addEventListener('mousemove', handleMouseMove);
container.addEventListener('mousedown', handleMouseDown);
container.addEventListener('mouseup', handleMouseUp);
// Pointer events cover trackpads that don't fire mouse events.
container.addEventListener('pointermove', handlePointerMove);
container.addEventListener('pointerdown', handlePointerDown);
container.addEventListener('pointerup', handlePointerUp);
container.addEventListener('wheel', handleWheel, { passive: false });
container.addEventListener('mouseleave', handleMouseLeave);
container.addEventListener('contextmenu', handleContextMenu);
Expand All @@ -310,6 +350,9 @@ export function useRemoteControl({
container.removeEventListener('mousemove', handleMouseMove);
container.removeEventListener('mousedown', handleMouseDown);
container.removeEventListener('mouseup', handleMouseUp);
container.removeEventListener('pointermove', handlePointerMove);
container.removeEventListener('pointerdown', handlePointerDown);
container.removeEventListener('pointerup', handlePointerUp);
container.removeEventListener('wheel', handleWheel);
container.removeEventListener('mouseleave', handleMouseLeave);
container.removeEventListener('contextmenu', handleContextMenu);
Expand All @@ -329,6 +372,9 @@ export function useRemoteControl({
handleWheel,
handleMouseLeave,
handleContextMenu,
handlePointerDown,
handlePointerUp,
handlePointerMove,
handleKeyDown,
handleKeyUp,
releaseHeldInput,
Expand Down
46 changes: 46 additions & 0 deletions apps/web/src/hooks/useRemoteControl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ export function useRemoteControl({
const heldButtonsRef = useRef<Set<MouseButton>>(new Set());
const heldKeysRef = useRef<Set<string>>(new Set());
const lastCursorUpdateRef = useRef(0);
// Pointer events fire before mouse events in Chromium. Storing the last
// pointer event's timestamp lets the mouse handler skip a double-fire.
const lastPointerEventRef = useRef(0);
const cursorThrottleMs = 16; // ~60fps throttle for cursor updates

// Check if we can send input (enabled, granted control, and capturing)
Expand Down Expand Up @@ -112,6 +115,9 @@ export function useRemoteControl({
(event: MouseEvent) => {
if (!canSendInput) return;

// Skip if a pointer event just fired (Chromium fires both).
if (Date.now() - lastPointerEventRef.current < 100) return;

const coords = getRelativeCoords(event);
if (!coords) return;

Expand Down Expand Up @@ -149,6 +155,9 @@ export function useRemoteControl({
(event: MouseEvent) => {
if (!canSendInput) return;

// Skip if a pointer event just fired.
if (Date.now() - lastPointerEventRef.current < 100) return;

const coords = getRelativeCoords(event);
if (!coords) return;

Expand Down Expand Up @@ -271,6 +280,33 @@ export function useRemoteControl({
[canSendInput]
);

// Pointer events fire before mouse events in Chromium, and on some trackpads
// only pointer events fire at all. Delegate to the mouse handlers, then mark
// the timestamp so the mouse-handler dedup skips the follow-up mouse event.
const handlePointerDown = useCallback(
(event: PointerEvent) => {
lastPointerEventRef.current = Date.now();
handleMouseDown(event as unknown as MouseEvent);
},
[handleMouseDown]
);

const handlePointerUp = useCallback(
(event: PointerEvent) => {
lastPointerEventRef.current = Date.now();
handleMouseUp(event as unknown as MouseEvent);
},
[handleMouseUp]
);

const handlePointerMove = useCallback(
(event: PointerEvent) => {
lastPointerEventRef.current = Date.now();
handleMouseMove(event as unknown as MouseEvent);
},
[handleMouseMove]
);

// Start capturing input
const startCapture = useCallback(() => {
setIsCapturing(true);
Expand All @@ -294,6 +330,10 @@ export function useRemoteControl({
container.addEventListener('mousemove', handleMouseMove);
container.addEventListener('mousedown', handleMouseDown);
container.addEventListener('mouseup', handleMouseUp);
// Pointer events cover trackpads that don't fire mouse events.
container.addEventListener('pointermove', handlePointerMove);
container.addEventListener('pointerdown', handlePointerDown);
container.addEventListener('pointerup', handlePointerUp);
container.addEventListener('wheel', handleWheel, { passive: false });
container.addEventListener('mouseleave', handleMouseLeave);
container.addEventListener('contextmenu', handleContextMenu);
Expand All @@ -310,6 +350,9 @@ export function useRemoteControl({
container.removeEventListener('mousemove', handleMouseMove);
container.removeEventListener('mousedown', handleMouseDown);
container.removeEventListener('mouseup', handleMouseUp);
container.removeEventListener('pointermove', handlePointerMove);
container.removeEventListener('pointerdown', handlePointerDown);
container.removeEventListener('pointerup', handlePointerUp);
container.removeEventListener('wheel', handleWheel);
container.removeEventListener('mouseleave', handleMouseLeave);
container.removeEventListener('contextmenu', handleContextMenu);
Expand All @@ -329,6 +372,9 @@ export function useRemoteControl({
handleWheel,
handleMouseLeave,
handleContextMenu,
handlePointerDown,
handlePointerUp,
handlePointerMove,
handleKeyDown,
handleKeyUp,
releaseHeldInput,
Expand Down
61 changes: 43 additions & 18 deletions packages/remote-input/src/injector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ export class RemoteInputInjector {
/** Where the local pointer was before a remote click borrowed it. */
private borrowedFrom: { x: number; y: number } | null = null;

// Every async operation that results in an OS-level button press or release
// chains through this promise. disable() and emergencyStop() wait on it so
// releaseAll always runs *after* trackHeldState has recorded the press.
private pendingInject: Promise<void> | null = null;

constructor(options: RemoteInputInjectorOptions = {}) {
this.selection = options.selection ?? getInputBackendSelection();
this.makeBackend = options.createBackend ?? createInputBackend;
Expand Down Expand Up @@ -163,9 +168,12 @@ export class RemoteInputInjector {

disable(): void {
this.enabled = false;
// Anything still held must come back up, or the host is left mid-drag with
// a physically stuck button and no way to recover short of a reboot.
void this.releaseAll('injection disabled');
// Wait for any in-flight inject to finish (so trackHeldState has run),
// then let go of everything. releaseAll() is deliberately not gated on
// `enabled` — releasing is always safe and must never be skipped.
void (this.pendingInject ?? Promise.resolve()).then(() =>
this.releaseAll('injection disabled')
);
this.logger.log('[RemoteInput] Injection disabled');
}

Expand Down Expand Up @@ -377,23 +385,35 @@ export class RemoteInputInjector {
return;
}

const validation = validateInputEvent(event);
if (!validation.ok) {
// reason is always set when ok is false
this.reject(validation.reason ?? 'invalid-key', event, validation.detail);
this.logger.warn('[RemoteInput] Refused event', {
reason: validation.reason,
detail: validation.detail,
});
return;
}

if (!this.rateLimiter.shouldAllow()) {
this.reject('rate-limited', event, 'event rate ceiling exceeded');
return;
}
// Serialize every injection so disable() and emergencyStop() can wait for
// us to finish (including trackHeldState) before they release everything.
// Without this, releaseAll can check heldButtons while dispatch is
// mid-await — nut-js has already pressed the button at the OS level, but
// trackHeldState hasn't recorded it yet — and the button stays stuck.
const prev = this.pendingInject;
let resolve: (() => void) | undefined;
this.pendingInject = new Promise<void>((r) => {
resolve = r;
});

try {
await prev;

const validation = validateInputEvent(event);
if (!validation.ok) {
this.reject(validation.reason ?? 'invalid-key', event, validation.detail);
this.logger.warn('[RemoteInput] Refused event', {
reason: validation.reason,
detail: validation.detail,
});
return;
}

if (!this.rateLimiter.shouldAllow()) {
this.reject('rate-limited', event, 'event rate ceiling exceeded');
return;
}

await this.dispatch(event);
this.stats.injected += 1;
this.trackHeldState(event);
Expand All @@ -405,6 +425,8 @@ export class RemoteInputInjector {
action: 'action' in event ? event.action : undefined,
error: error instanceof Error ? error.message : String(error),
});
} finally {
resolve?.();
}
}

Expand All @@ -416,6 +438,9 @@ export class RemoteInputInjector {
async emergencyStop(): Promise<void> {
this.logger.log('[RemoteInput] Emergency stop');
this.enabled = false;
// Must wait for any in-flight inject to finish before releasing, for the
// same reason disable() does: trackHeldState might not have run yet.
if (this.pendingInject) await this.pendingInject;
await this.releaseAll('emergency stop');

try {
Expand Down
Loading