-
Notifications
You must be signed in to change notification settings - Fork 660
feat: Multiprocessing support in packaged desktop apps #6662
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
21 commits
Select commit
Hold shift + click to select a range
79df82e
feat(build): macOS runner intercepts multiprocessing child re-execs
ndonkoHenri 8a06e44
feat(build): Windows runner intercepts multiprocessing child re-execs
ndonkoHenri 955fbbe
feat(build): Linux runner intercepts multiprocessing child re-execs
ndonkoHenri ba85721
fix(build): run the app module as the real __main__; point multiproce…
ndonkoHenri c485b71
feat(create): guard ft.run() with __main__ in the app scaffold
ndonkoHenri eef9908
refactor(build): splice all dynamic boot-script values via jsonEncode
ndonkoHenri a0ed920
docs: Multiprocessing cookbook recipe
ndonkoHenri 6cf2905
add changelog entry
ndonkoHenri 35894b6
Merge branch 'flet-0.86' into multiprocessing
ndonkoHenri 596b497
fix(build): open console.log as UTF-8
ndonkoHenri 6c65a66
fix: hide the console window of the git version-fallback on Windows
ndonkoHenri e7187d6
add cookbook recipe and examples
ndonkoHenri 1522f5d
docs: add detailed usage guides to CLI command docstrings
ndonkoHenri 09dc4bf
docs: update README of build-template with installation, platform sup…
ndonkoHenri 876a006
breaking changes in version folders
ndonkoHenri 07687fa
update changelog
ndonkoHenri 132324c
Merge branch 'flet-0.86' into multiprocessing
ndonkoHenri 2aeb08a
fix(build): splice {module_name} into the boot script via jsonEncode too
ndonkoHenri 66f6dec
docs: drop leftover debug prints from the persistent-worker example
ndonkoHenri c44b7ed
docs: fix relative links broken by the breaking-changes version folders
ndonkoHenri 42b5268
Bump python build release and serious_python
FeodorFitsner File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
sdk/python/examples/cookbook/multiprocessing/cancel_task.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| import multiprocessing | ||
| import time | ||
|
|
||
| import flet as ft | ||
|
|
||
|
|
||
| def long_job(seconds: int) -> None: | ||
| """Burn CPU for roughly `seconds` seconds. | ||
|
|
||
| Stands in for work you cannot interrupt from Python itself, e.g. a tight | ||
| loop inside a C extension library. Runs in a worker process, which is why | ||
| it can be cancelled at any point — threads can't be stopped like that. | ||
| """ | ||
| deadline = time.monotonic() + seconds | ||
| while time.monotonic() < deadline: | ||
| sum(range(1_000_000)) | ||
|
|
||
|
|
||
| def main(page: ft.Page): | ||
| process = None | ||
|
|
||
| def watch(p: multiprocessing.Process): | ||
| """Wait (on a background thread) for the worker to end, then report | ||
| whether it finished on its own or was cancelled.""" | ||
| p.join() | ||
| if p.exitcode == 0: | ||
| status.value = "Process finished normally." | ||
| else: | ||
| # terminate() ends the child with a negative exit code (-SIGTERM). | ||
| status.value = f"Process was terminated (exit code {p.exitcode})." | ||
| start.disabled = False | ||
| cancel.disabled = True | ||
| page.update() | ||
|
|
||
| def start_job(): | ||
| nonlocal process | ||
| process = multiprocessing.Process(target=long_job, args=(30,), daemon=True) | ||
| process.start() | ||
| status.value = f"Running in process {process.pid}…" | ||
| start.disabled = True | ||
| cancel.disabled = False | ||
| page.update() | ||
| page.run_thread(watch, process) | ||
|
|
||
| def cancel_job(): | ||
| if process is not None and process.is_alive(): | ||
| process.terminate() # threads can't do this | ||
|
|
||
| page.add( | ||
| ft.SafeArea( | ||
| content=ft.Column( | ||
| controls=[ | ||
| ft.Row( | ||
| controls=[ | ||
| start := ft.Button("Start 30s job", on_click=start_job), | ||
| cancel := ft.Button( | ||
| "Cancel", on_click=cancel_job, disabled=True | ||
| ), | ||
| ] | ||
| ), | ||
| status := ft.Text(), | ||
| ] | ||
| ) | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| ft.run(main) |
46 changes: 46 additions & 0 deletions
46
sdk/python/examples/cookbook/multiprocessing/parallel_sort.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| import random | ||
| from concurrent.futures import ProcessPoolExecutor, as_completed | ||
|
|
||
| import flet as ft | ||
|
|
||
|
|
||
| def sort_chunk(chunk: list[float]) -> list[float]: | ||
| """Sort one chunk of data.""" | ||
| return sorted(chunk) | ||
|
|
||
|
|
||
| def main(page: ft.Page): | ||
| def run_sort(): | ||
| """Orchestrate the pool from a background thread, updating the UI | ||
| from the main process as results come in.""" | ||
| chunks = [[random.random() for _ in range(250_000)] for _ in range(8)] | ||
| completed = 0 | ||
| # Without max_workers, the pool sizes itself to the machine's CPUs. | ||
| with ProcessPoolExecutor() as pool: | ||
| futures = [pool.submit(sort_chunk, c) for c in chunks] | ||
| # as_completed() yields each future as soon as its worker is done. | ||
| for _ in as_completed(futures): | ||
| completed += 1 | ||
| progress.value = completed / len(futures) | ||
| status.value = f"Sorted {completed}/{len(futures)} chunks" | ||
| page.update() | ||
| status.value = "Done!" | ||
| page.update() | ||
|
|
||
| page.add( | ||
| ft.SafeArea( | ||
| content=ft.Column( | ||
| controls=[ | ||
| ft.Button( | ||
| "Start sorting", on_click=lambda: page.run_thread(run_sort) | ||
| ), | ||
| status := ft.Text("Idle"), | ||
| progress := ft.ProgressBar(value=0, width=300), | ||
| ] | ||
| ) | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| ft.run(main) | ||
65 changes: 65 additions & 0 deletions
65
sdk/python/examples/cookbook/multiprocessing/persistent_worker.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| import multiprocessing | ||
| import time | ||
|
|
||
| import flet as ft | ||
|
|
||
|
|
||
| def calc_worker(conn) -> None: | ||
| """Serve calculation jobs over the pipe until told to stop. | ||
|
|
||
| Runs in a worker process that stays alive across jobs, so the process | ||
| startup cost is paid once. Expensive setup (loading models, opening | ||
| datasets, warming caches) could be done once here, before the loop, | ||
| and reused by every job. Receiving `None` is the shutdown signal. | ||
| """ | ||
| while (job := conn.recv()) is not None: | ||
| started = time.perf_counter() | ||
| result = sum(i * i for i in range(job)) | ||
| conn.send((result, time.perf_counter() - started)) | ||
| conn.close() | ||
|
|
||
|
|
||
| def main(page: ft.Page): | ||
| # One end of the pipe goes to the worker, the other stays with the UI. | ||
| parent_conn, child_conn = multiprocessing.Pipe() | ||
| worker = multiprocessing.Process( | ||
| target=calc_worker, args=(child_conn,), daemon=True | ||
| ) | ||
| worker.start() | ||
|
|
||
| def submit(): | ||
| # One job in flight at a time: the button stays disabled until the | ||
| # worker replies (a bare Pipe is not multiplexed). | ||
| button.disabled = True | ||
| status.value = "Working…" | ||
| page.update() | ||
| page.run_thread(request) | ||
|
|
||
| def request(): | ||
| """Send a job to the worker and wait for its reply. | ||
|
|
||
| Should be run on a background thread: conn.recv() blocks until the worker | ||
| answers, so it must stay off the UI event loop. | ||
| """ | ||
| parent_conn.send(100_000_000) | ||
| result, duration = parent_conn.recv() | ||
| status.value = ( | ||
| f"sum of squares = {result}\n{duration:.2f}s in process {worker.pid}" | ||
| ) | ||
| button.disabled = False | ||
| page.update() | ||
|
|
||
| page.add( | ||
| ft.SafeArea( | ||
| content=ft.Column( | ||
| controls=[ | ||
| button := ft.Button("Compute in worker", on_click=submit), | ||
| status := ft.Text(f"Worker ready (pid {worker.pid})"), | ||
| ] | ||
| ) | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| ft.run(main) |
58 changes: 58 additions & 0 deletions
58
sdk/python/examples/cookbook/multiprocessing/worker_progress.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| import multiprocessing | ||
|
|
||
| import flet as ft | ||
|
|
||
|
|
||
| def crunch(progress_queue: multiprocessing.Queue, steps: int) -> None: | ||
| """Do `steps` slices of CPU-heavy work, reporting progress after each one. | ||
|
|
||
| Runs in a worker process, which has no access to the page — the queue is | ||
| the only channel back to the UI. Values are fractions 0..1; a final `None` | ||
| tells the consumer there is nothing more to read. | ||
| """ | ||
| for i in range(steps): | ||
| sum(range(50_000_000)) # one slice of real work | ||
| progress_queue.put((i + 1) / steps) | ||
| progress_queue.put(None) # sentinel: no more updates | ||
|
|
||
|
|
||
| def main(page: ft.Page): | ||
| def start(): | ||
| button.disabled = True | ||
| status.value = "Starting worker…" | ||
| page.update() | ||
| queue = multiprocessing.Queue() | ||
| worker = multiprocessing.Process(target=crunch, args=(queue, 20), daemon=True) | ||
| worker.start() | ||
| page.run_thread(drain, queue, worker) | ||
|
|
||
| def drain(queue: multiprocessing.Queue, worker: multiprocessing.Process): | ||
| """Forward the worker's progress reports to the UI. | ||
|
|
||
| Runs on a background thread: queue.get() blocks until the worker | ||
| reports again, so it must stay off the UI event loop. | ||
| """ | ||
| while (value := queue.get()) is not None: | ||
| progress.value = value | ||
| status.value = f"Crunching… {value:.0%}" | ||
| page.update() | ||
| worker.join() | ||
| status.value = "Done!" | ||
| button.disabled = False | ||
| page.update() | ||
|
|
||
| page.add( | ||
| ft.SafeArea( | ||
| content=ft.Column( | ||
| controls=[ | ||
| button := ft.Button("Start", on_click=start), | ||
| progress := ft.ProgressBar(value=0, width=300), | ||
| status := ft.Text(), | ||
| ] | ||
| ) | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| ft.run(main) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.