diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd50497..048bddd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,54 +2,124 @@ name: CI on: push: - branches: ["main", "develop", "workspace-esparso"] + branches: [main, develop] pull_request: - branches: ["main", "develop"] + branches: [main, develop] + workflow_dispatch: # allows running the workflow manually jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Ruff + # keep in sync with the `ruff` pin in pyproject.toml's `dev` extra + run: pip install "ruff>=0.16,<0.17" + + - name: Run Ruff + run: ruff check src/ tests/ examples/ scripts/ + test: + # dissmodel + every optional geospatial extra that does not depend on + # repositories still under construction (brmangue-dissmodel is left out + # on purpose). Starts at 3.11 because the `zarr` extra needs + # zarr-python 3, which requires Python >= 3.11 (same floor as disscube). + name: test (py${{ matrix.python-version }}) runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: fail-fast: false matrix: - include: - # Núcleo puro: sem dissmodel nem nenhum extra geoespacial. - # Garante que engine.py, disk/workspace.py, disk/convergence.py - # continuam com zero dependências em runtime. - - name: "core (sem extras)" - extras: "dev" - # dissmodel + todos os extras geoespaciais opcionais que não - # dependem de repositórios externos ainda em construção - # (brmangue-dissmodel fica de fora de propósito). - - name: "full (dissmodel + geo extras)" - extras: "dev,dissmodel,geotiff,zarr,zarr-test,geomosaic,dissmodel-ca" + python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev,dissmodel,geotiff,zarr,zarr-test,geomosaic,dissmodel-ca]" + + # Coverage XML is produced but not uploaded anywhere yet. + - name: Run tests with coverage + run: pytest tests/ --cov=haloexec --cov-report=term-missing --cov-report=xml + + test-minimal: + # Core install without optional extras: guarantees engine.py, + # disk/workspace.py and disk/convergence.py keep zero runtime + # dependencies beyond numpy, and that tests needing optional packages + # are skipped, not failed. Runs on 3.10, the lowest version the core + # supports (requires-python), which the full job cannot cover. + name: test (core install, no extras, py3.10) + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run tests + run: pytest tests/ -rs + + docs: + # 1) builds the mkdocs site in strict mode (broken links/anchors fail); + # 2) extracts every ```python block from docs/ and runs it, to catch + # broken quickstarts (import errors, wrong mixin/inheritance order, + # renamed APIs). + name: docs (strict build + code blocks) + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 with: python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml - - name: Install package (${{ matrix.name }}) - run: pip install -e ".[${{ matrix.extras }}]" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev,docs,dissmodel,geotiff,zarr,zarr-test,geomosaic,dissmodel-ca]" - - name: Run test suite - run: pytest -v + - name: Build site (strict) + run: mkdocs build --strict - - name: Extract and smoke-test Python blocks from docs/ - if: matrix.name == 'full (dissmodel + geo extras)' + - name: Run Python blocks from docs/ run: | python - <<'EOF' import re, subprocess, sys, pathlib docs = pathlib.Path("docs").glob("*.md") failures = [] - # Palavras que indicam um bloco ilustrativo (referencia dado - # externo que não existe neste repo, ou escala grande demais - # pra rodar em CI) -- pulamos, não são candidatos a bug de - # import/herança, que é o que este smoke-test mira. + # Markers of illustrative blocks (external data not in this repo, + # or a scale too large for CI) -- skipped, since they are not the + # import/inheritance bugs this smoke test targets. SKIP_MARKERS = ("50_000", "study_area.tif", "tile_001.zarr", "/tmp/my_large_workspace") for doc in docs: @@ -60,7 +130,7 @@ jobs: continue if "import" not in block: continue - print(f"--- {doc.name} bloco {i} ---") + print(f"--- {doc.name} block {i} ---") result = subprocess.run( [sys.executable, "-c", block], capture_output=True, text=True, timeout=120, @@ -70,9 +140,9 @@ jobs: print(result.stderr) if failures: - print(f"\n{len(failures)} bloco(s) de código quebrado(s) em docs/:") + print(f"\n{len(failures)} broken code block(s) in docs/:") for name, i, err in failures: - print(f" - {name} bloco {i}") + print(f" - {name} block {i}") sys.exit(1) - print("Todos os blocos de código testáveis em docs/ rodaram sem erro.") + print("All testable code blocks in docs/ ran without error.") EOF diff --git a/.github/workflows/docs_deploy.yml b/.github/workflows/docs_deploy.yml new file mode 100644 index 0000000..eaa6321 --- /dev/null +++ b/.github/workflows/docs_deploy.yml @@ -0,0 +1,24 @@ +name: Deploy documentation to GitHub Pages + +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + contents: write # needed to push to gh-pages (ghp-import) + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -e ".[docs]" + + - name: Deploy to GitHub Pages + run: mkdocs gh-deploy --force --strict diff --git a/.gitignore b/.gitignore index 84034d6..05201b4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,10 @@ __pycache__/ .pytest_cache/ *.egg-info/ .venv -raster_map_frames/ \ No newline at end of file +raster_map_frames/ +.coverage +coverage.xml +htmlcov/ +build/ +dist/ +site/ diff --git a/README.md b/README.md index 16ab5a2..3275579 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # haloexec +[![CI](https://github.com/DisSModel/haloexec/actions/workflows/ci.yml/badge.svg)](https://github.com/DisSModel/haloexec/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + Cellular Automaton execution engine using **Domain Decomposition** with **Halo Zones** (Ghost Cell Pattern), integrated with real [dissmodel](https://pypi.org/project/dissmodel/) via a pip dependency — diff --git a/docs/README.md b/docs/README.md index b0b675e..519958c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -113,7 +113,7 @@ env.run() ### 2. Out-of-Core Disk-Backed Simulation -When the domain exceeds physical memory, initialize a [`MemmapRasterWorkspace`](../src/haloexec/disk/workspace.py#L81) and run out-of-core: +When the domain exceeds physical memory, initialize a [`MemmapRasterWorkspace`](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/workspace.py#L81) and run out-of-core: ```python from pathlib import Path diff --git a/docs/api_reference.md b/docs/api_reference.md index bcc233e..aee3092 100644 --- a/docs/api_reference.md +++ b/docs/api_reference.md @@ -23,11 +23,11 @@ This document provides a comprehensive, exhaustive reference for all public clas - [DiskChunkedSyncRasterModel](#diskchunkedsyncrastermodel) - [haloexec.disk.cellular_automaton](#haloexecdiskcellular_automaton) - [DiskChunkedRasterCellularAutomaton](#diskchunkedrastercellularautomaton) -- [haloexec.disk.io.geotiff](#haloexecdiskio_geotiff) +- [haloexec.disk.io.geotiff](#haloexecdiskiogeotiff) - [load_geotiff_into_workspace](#load_geotiff_into_workspace) - [load_geotiffs_into_workspace](#load_geotiffs_into_workspace) - [save_workspace_to_geotiff](#save_workspace_to_geotiff) -- [haloexec.disk.io.zarr](#haloexecdiskio_zarr) +- [haloexec.disk.io.zarr](#haloexecdiskiozarr) - [load_zarr_into_workspace](#load_zarr_into_workspace) - [load_zarr_tiles_into_workspace](#load_zarr_tiles_into_workspace) - [haloexec.ram.cellular_automaton](#haloexecramcellular_automaton) diff --git a/docs/architecture.md b/docs/architecture.md index aba9dd2..e87fa22 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,12 +69,12 @@ graph TD ## 3. Component Deep Dive by Layer ### Layer 0: Pure Core Primitives (`haloexec.engine`) -- **[Block](../src/haloexec/engine.py#L21)**: An immutable, hashable dataclass representing a rectangular domain slice $[r_0:r_1, c_0:c_1)$. Provides properties `.shape` and `.core` (ready-to-use tuple of slice objects). -- **[make_blocks](../src/haloexec/engine.py#L41)**: Pure generator/function that subdivides arbitrary 2D dimensions $(H, W)$ into regular blocks of target size $(b_h, b_w)$, naturally accommodating edge remainders. -- **[resolve_boundary_value](../src/haloexec/engine.py#L55)**: Sentinel resolution utility mapping array names to boundary nodata values, with automatic fallback for `_past` temporal layers. +- **[Block](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/engine.py#L21)**: An immutable, hashable dataclass representing a rectangular domain slice $[r_0:r_1, c_0:c_1)$. Provides properties `.shape` and `.core` (ready-to-use tuple of slice objects). +- **[make_blocks](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/engine.py#L41)**: Pure generator/function that subdivides arbitrary 2D dimensions $(H, W)$ into regular blocks of target size $(b_h, b_w)$, naturally accommodating edge remainders. +- **[resolve_boundary_value](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/engine.py#L55)**: Sentinel resolution utility mapping array names to boundary nodata values, with automatic fallback for `_past` temporal layers. ### Layer 1: Storage & Workspace Management (`haloexec.disk.workspace`) -- **[MemmapRasterWorkspace](../src/haloexec/disk/workspace.py#L81)**: Manages binary disk arrays on the filesystem. +- **[MemmapRasterWorkspace](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/workspace.py#L81)**: Manages binary disk arrays on the filesystem. - **Directory Hierarchy**: ```text workspace_root/ @@ -97,11 +97,11 @@ graph TD - `checkpoint(step)`: Atomically persists execution state. ### Layer 2: Spatial Ingestion & Egress (`haloexec.disk.io`) -- **[geotiff.py](../src/haloexec/disk/io/geotiff.py)**: +- **[geotiff.py](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/io/geotiff.py)**: - Streams GeoTIFF and VRT rasters into a `MemmapRasterWorkspace` block-by-block using `rasterio.windows.Window`. - Validates coordinate reference systems (CRS) and dimension consistency across multi-file inputs. - Exports workspace slots to Cloud-Optimized GeoTIFFs (COG) with block-aligned tiling. -- **[zarr.py](../src/haloexec/disk/io/zarr.py)**: +- **[zarr.py](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/io/zarr.py)**: - Streams directly from Zarr groups or arrays (compatible with `disscube` data cubes). - Normalizes coordinate axis orders using Zarr v3 dimension metadata (`arr.metadata.dimension_names`) to prevent silent transpose bugs. - Assembles multi-tile footprints (`load_zarr_tiles_into_workspace`) into continuous seamless rasters. @@ -166,7 +166,7 @@ By ensuring that `self.backend` and `self.shape` point to `block_backend` during ### 4.3 Decimation Adapter Pattern (`WorkspaceRasterBackend`) Visualizing multi-million-cell rasters with `matplotlib` or interactive dashboards causes severe performance degradation and memory spikes. -Rather than copying and downsampling the entire raster in RAM, [WorkspaceRasterBackend](../src/haloexec/disk/backend.py#L17) implements an on-the-fly strided view: +Rather than copying and downsampling the entire raster in RAM, [WorkspaceRasterBackend](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/backend.py#L16) implements an on-the-fly strided view: ```python # haloexec/disk/backend.py:L57 diff --git a/docs/javascripts/mathjax.js b/docs/javascripts/mathjax.js new file mode 100644 index 0000000..d9482f6 --- /dev/null +++ b/docs/javascripts/mathjax.js @@ -0,0 +1,20 @@ +// MathJax configuration for pymdownx.arithmatex (generic mode). +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true, + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex", + }, +}; + +document$.subscribe(() => { + MathJax.startup.output.clearCache(); + MathJax.typesetClear(); + MathJax.texReset(); + MathJax.typesetPromise(); +}); diff --git a/docs/theory_and_concepts.md b/docs/theory_and_concepts.md index 982e6f3..40427ea 100644 --- a/docs/theory_and_concepts.md +++ b/docs/theory_and_concepts.md @@ -135,7 +135,7 @@ A ubiquitous practice in numerical computing is zero-padding (`np.pad(..., const If unmanaged, zero-padding external boundaries injects artificial river channels or sea-level cells along the edge of the study area, generating phantom colonization and divergent model dynamics. ### 4.2 Per-Array Sentinel Mapping -`haloexec` prevents this failure via [resolve_boundary_value](../src/haloexec/engine.py#L55): +`haloexec` prevents this failure via [resolve_boundary_value](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/engine.py#L55): - `boundary_value` accepts a dictionary mapping array names to their true domain `nodata` sentinel: ```python boundary_value = {"uso": 0, "alt": -9999.0, "solo": -1} @@ -149,7 +149,7 @@ If unmanaged, zero-padding external boundaries injects artificial river channels When scaling to regional or national extents (e.g., a 40,000 $\times$ 40,000 grid encompassing 1.6 billion cells, or 6.4 GB per float32 layer), allocating global in-memory NumPy arrays or performing global `np.pad` results in immediate Out-Of-Memory (OOM) termination. ### 5.1 Memory-Mapped I/O (`numpy.memmap`) -[`MemmapRasterWorkspace`](../src/haloexec/disk/workspace.py#L81) stores spatial arrays directly on the filesystem as uncompressed binary disk files (`.dat`) accessed through POSIX `mmap()` system calls. +[`MemmapRasterWorkspace`](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/workspace.py#L81) stores spatial arrays directly on the filesystem as uncompressed binary disk files (`.dat`) accessed through POSIX `mmap()` system calls. Under `mmap`: 1. The kernel maps the file on disk into the virtual address space of the process. @@ -174,7 +174,7 @@ In a 40,000 $\times$ 40,000 Conway's Game of Life simulation (1.6 billion cells This proves that `haloexec`'s memory footprint is $O(b_h \cdot b_w)$—proportional strictly to the block size, and **strictly independent of the global domain size** $H \times W$. ### 5.3 Filesystem Sparsity Economics -When creating new arrays, [`MemmapRasterWorkspace.create()`](../src/haloexec/disk/workspace.py#L145) uses `mode="w+"` to truncate and set the file length without writing zeros across the disk blocks. +When creating new arrays, [`MemmapRasterWorkspace.create()`](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/workspace.py#L144) uses `mode="w+"` to truncate and set the file length without writing zeros across the disk blocks. On modern filesystems (ext4, XFS, APFS, NTFS), this creates **POSIX sparse files**: - Unwritten disk blocks occupy zero physical storage on disk. @@ -230,7 +230,7 @@ backend.arrays["alt_past"] = backend.arrays["alt"].copy() ``` If applied naively to a disk-backed memory-mapped workspace, this call would materialize the entire multi-gigabyte array in RAM, crashing the system. -`haloexec` resolves this via [write_block_to_read_slot](../src/haloexec/disk/workspace.py#L269): +`haloexec` resolves this via [write_block_to_read_slot](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/workspace.py#L269): 1. Historical arrays `_past` are allocated directly within both disk slots. 2. In `pre_execute()` and `post_execute()`, synchronization copies data **block-by-block** directly within the **active read slot**: $$\text{Slot}_{\text{read}}[name\_past][block] \longleftarrow \text{Slot}_{\text{read}}[name][block]$$ @@ -248,10 +248,10 @@ Certain spatial processes cannot be bounded by a finite, localized halo radius: Resolving these dynamics with a static halo would require setting $h = \max(H, W)$, degenerating into a monolithic execution that exhausts memory. ### 7.2 Gauss-Seidel Iterative Convergence -For these unbounded processes, [`sweep_until_convergence`](../src/haloexec/disk/convergence.py#L46) replaces the fixed halo ping-pong mechanism with an **iterative relaxation sweep**: +For these unbounded processes, [`sweep_until_convergence`](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/convergence.py#L46) replaces the fixed halo ping-pong mechanism with an **iterative relaxation sweep**: - Uses a minimal halo ($h = 1$). - Traverses all blocks in the domain repeatedly. -- Updates are written **immediately in-place** to the active read buffer via [write_block_core_in_place](../src/haloexec/disk/workspace.py#L279). +- Updates are written **immediately in-place** to the active read buffer via [write_block_core_in_place](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/workspace.py#L279). ``` Sweep k: @@ -279,7 +279,7 @@ and the state space $\mathcal{S}$ is finite (e.g., binary connectivity $\{0, 1\} ## 8. Spatial Ingestion & Coordinate Anomalies ### 8.1 GeoTIFF Windowed Streaming -Rather than loading large GeoTIFF or VRT rasters monolithically into memory, [`load_geotiffs_into_workspace`](../src/haloexec/disk/io/geotiff.py#L74) maps each block's bounding box to a `rasterio.windows.Window`: +Rather than loading large GeoTIFF or VRT rasters monolithically into memory, [`load_geotiffs_into_workspace`](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/io/geotiff.py#L75) maps each block's bounding box to a `rasterio.windows.Window`: $$\text{Window}(\text{col\_off} = c_0, \; \text{row\_off} = r_0, \; \text{width} = c_1 - c_0, \; \text{height} = r_1 - r_0)$$ @@ -292,7 +292,7 @@ When ingesting data cubes from modern cloud stores (such as `disscube` or xarray $$\text{shape}(arr) = (N, N) \equiv (N, N)$$ fails to detect inverted axes. This causes silent, catastrophic transpose corruption (e.g., swapping latitude and longitude). -To safeguard data integrity, [`load_zarr_into_workspace`](../src/haloexec/disk/io/zarr.py#L86) interrogates native Zarr v3 dimension metadata (`arr.metadata.dimension_names`): +To safeguard data integrity, [`load_zarr_into_workspace`](https://github.com/DisSModel/haloexec/blob/main/src/haloexec/disk/io/zarr.py#L93) interrogates native Zarr v3 dimension metadata (`arr.metadata.dimension_names`): 1. Resolves actual dimension positions: e.g., identifying whether `"y"` is at axis 0 or axis 1. 2. Generates canonical index projections during chunk extraction. 3. Automatically transposes extracted sub-arrays to standard `(y, x)` canonical orientation before committing blocks to the workspace. diff --git a/examples/gol/gol_patterns_haloexec.py b/examples/gol/gol_patterns_haloexec.py index 77381fe..96a56c5 100644 --- a/examples/gol/gol_patterns_haloexec.py +++ b/examples/gol/gol_patterns_haloexec.py @@ -22,13 +22,12 @@ from __future__ import annotations import numpy as np - from dissmodel.core import Environment from dissmodel.geo import raster_grid from dissmodel.visualization.raster_map import RasterMap +from dissmodel_ca.models.game_of_life import PATTERNS from haloexec import HaloChunkedRasterCellularAutomaton -from dissmodel_ca.models.game_of_life import PATTERNS # --------------------------------------------------------------------------- diff --git a/examples/gol/teste_escala_30m.py b/examples/gol/teste_escala_30m.py index a91dbf0..7c3df83 100644 --- a/examples/gol/teste_escala_30m.py +++ b/examples/gol/teste_escala_30m.py @@ -12,19 +12,17 @@ """ import time -import resource from pathlib import Path import numpy as np import rasterio -from rasterio.windows import Window -from rasterio.transform import from_origin - from dissmodel.core import Environment from dissmodel.geo import raster_grid from dissmodel_ca.models.game_of_life_raster import GameOfLife +from rasterio.transform import from_origin +from rasterio.windows import Window -from haloexec import MemmapRasterWorkspace, DiskChunkedRasterCellularAutomaton, load_geotiff_into_workspace +from haloexec import DiskChunkedRasterCellularAutomaton, MemmapRasterWorkspace, load_geotiff_into_workspace class GameOfLifeHalo(DiskChunkedRasterCellularAutomaton, GameOfLife): @@ -121,7 +119,7 @@ def main(): resultado_disco = ws.snapshot("state") - print(f"\n=== resumo de memoria ===") + print("\n=== resumo de memoria ===") print(f"tamanho de um array completo: {HEIGHT*WIDTH/1024**2:.1f} MB") print(f"RssAnon final: {rss_pos_execucao['RssAnon']:.1f} MB " f"(razao sobre 1 array: {rss_pos_execucao['RssAnon']/(HEIGHT*WIDTH/1024**2):.2f}x)") @@ -131,7 +129,7 @@ def main(): # 30M celulas uint8 cabem em RAM como array UNICO (~30MB) -- o que # nao cabe/nao deveria ser feito e materializar durante GERACAO e # CARGA do arquivo, que ja foi provado acima via RssAnon. - print(f"\n[extra] gerando referencia monolitica na mesma escala para prova de equivalencia...") + print("\n[extra] gerando referencia monolitica na mesma escala para prova de equivalencia...") with rasterio.open(str(tif_path)) as ds: estado0 = ds.read(1) # aqui SIM materializamos, de proposito, so para a referencia golden backend_mono = raster_grid(rows=HEIGHT, cols=WIDTH, attrs={"state": estado0.copy()}) diff --git a/examples/gol/teste_escala_com_rastermap.py b/examples/gol/teste_escala_com_rastermap.py index efe64d9..eedc44b 100644 --- a/examples/gol/teste_escala_com_rastermap.py +++ b/examples/gol/teste_escala_com_rastermap.py @@ -15,17 +15,16 @@ import numpy as np import rasterio -from rasterio.windows import Window -from rasterio.transform import from_origin - from dissmodel.core import Environment from dissmodel_ca.models.game_of_life_raster import GameOfLife +from rasterio.transform import from_origin +from rasterio.windows import Window from haloexec import ( - MemmapRasterWorkspace, DiskChunkedRasterCellularAutomaton, - load_geotiff_into_workspace, + MemmapRasterWorkspace, WorkspaceRasterBackend, + load_geotiff_into_workspace, ) from haloexec.visualization import CheckpointRasterMap @@ -82,7 +81,7 @@ def main() -> None: tmp.mkdir(parents=True, exist_ok=True) tif_path = tmp / "estado_inicial.tif" - print(f"=== Teste de Escala com Visualização por Checkpoints ===") + print("=== Teste de Escala com Visualização por Checkpoints ===") print(f"Grade: {HEIGHT}x{WIDTH} = {HEIGHT*WIDTH:,} células (~{HEIGHT*WIDTH/1024**2:.1f} MB por array uint8)") print(f"Salvando quadros PNG apenas nos passos: {ANOS_PARA_SALVAR}\n") diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..dbe4597 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,50 @@ +site_name: haloexec +site_description: Domain decomposition with halo zones for large-scale cellular automata +site_url: https://dissmodel.github.io/haloexec/ +repo_url: https://github.com/DisSModel/haloexec +repo_name: haloexec +edit_uri: edit/main/docs/ + +theme: + name: material + language: en + palette: + - scheme: default + primary: teal + accent: green + features: + - content.code.copy + - navigation.sections + +plugins: + - search + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.highlight + - pymdownx.inlinehilite + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.arithmatex: + generic: true + # Renders GitHub-style callouts (> [!WARNING]) as admonitions, so the + # same Markdown reads correctly both on GitHub and on the site. + - github-callouts + - toc: + permalink: true + +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js + +# docs/README.md is the landing page on GitHub; mkdocs uses it as index. +nav: + - Home: README.md + - Theory & Core Concepts: theory_and_concepts.md + - Architecture & Design Patterns: architecture.md + - API Reference: api_reference.md + - Tutorials & Recipes: tutorials_and_recipes.md diff --git a/pyproject.toml b/pyproject.toml index aeccfdf..1ac2896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,12 +10,34 @@ readme = "README.md" license = "MIT" license-files = ["LICENSE"] requires-python = ">=3.10" +authors = [ + { name = "Sérgio Costa" } +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering :: GIS", +] dependencies = [ "numpy>=1.24", ] [project.optional-dependencies] -dev = ["pytest>=7.0"] +dev = [ + "pytest>=7.0", + "pytest-cov", + "ruff>=0.16,<0.17", # pinned minor: ruff's default rule set grows across versions +] +# Documentation site (mkdocs.yml); built and deployed by +# .github/workflows/docs_deploy.yml. +docs = [ + "mkdocs>=1.6,<2", # MkDocs 2.0 drops the plugin/theme system mkdocs-material needs + "mkdocs-material>=9.5", + "markdown-callouts>=0.4", # github-callouts extension (> [!WARNING]) +] # Opcional: só necessário para HaloChunkedRasterCellularAutomaton, # HaloChunkedSyncRasterModel, DiskChunkedSyncRasterModel # (dissmodel_ca.py, sync_model.py, disk_sync_model.py) -- os únicos 3 @@ -28,12 +50,13 @@ dissmodel = ["dissmodel>=0.6.3"] geotiff = ["rasterio>=1.3"] # Opcional: só necessário para zarr_io.py (carregamento de Zarr direto # para MemmapRasterWorkspace — segunda opção de entrada, pensada para -# consumir DerivedVariable do disscube). -zarr = ["zarr>=2.16"] +# consumir DerivedVariable do disscube). Requer zarr-python 3 (API +# create_array/dimension_names), que por sua vez exige Python >= 3.11. +zarr = ["zarr>=3"] # Opcional: só necessário para tests/test_zarr_axis_order_regression.py, # que reproduz o padrão real de escrita do disscube (VariableWriter usa # xarray.to_zarr()) para validar a correção de ordem de eixos. -zarr-test = ["zarr>=2.16", "xarray>=2023.1"] +zarr-test = ["zarr>=3", "xarray>=2024.10"] # Opcional: só necessário para tests/test_geomosaic_integration.py, que # prova que load_geotiff_into_workspace lê corretamente um mosaico # produzido pelo geomosaic (pacote separado, sem relação de runtime @@ -53,3 +76,15 @@ where = ["src"] # scripts de exemplo em examples/ seriam coletados automaticamente por # um `pytest` sem argumento na raiz do repositório. testpaths = ["tests"] + +[tool.ruff] +# Same policy as DisSModel/dissmodel and DisSModel/disscube: no explicit +# `select`, lint with ruff's own default rule set for the pinned minor +# version (see the `ruff` pin in the `dev` extra). If bumping that pin +# changes CI results, reconcile by fixing the code or, for a rule that is a +# bad fit, adding it to an explicit `ignore` here — not by re-pinning +# silently. +# line-length is informational: E501 is not in the default rules. +line-length = 120 +target-version = "py310" +exclude = ["docs", "site", "build"] diff --git a/scripts/generate_and_benchmark.py b/scripts/generate_and_benchmark.py index bf7778d..fc253cc 100644 --- a/scripts/generate_and_benchmark.py +++ b/scripts/generate_and_benchmark.py @@ -24,7 +24,6 @@ from __future__ import annotations import argparse -import resource import shutil import time from pathlib import Path @@ -133,9 +132,9 @@ def run_benchmark( print(f"RssFile (cache de páginas mmap, reclamável): {m_depois['RssFile']:.1f} MB") print(f"VmRSS total (soma dos dois, é o que ru_maxrss mediria): {m_depois['VmRSS']:.1f} MB") print(f"Tamanho de um array completo em disco: {grid_bytes / 1024**2:.1f} MB") - print(f"→ RssAnon é a métrica correta para 'quanto o processo materializou " - f"de fato'; RssFile cresce com o volume TOCADO acumulado (cache), " - f"não com o que está retido de uma vez.") + print("→ RssAnon é a métrica correta para 'quanto o processo materializou " + "de fato'; RssFile cresce com o volume TOCADO acumulado (cache), " + "não com o que está retido de uma vez.") if not keep: shutil.rmtree(root) diff --git a/src/haloexec/__init__.py b/src/haloexec/__init__.py index c00f7e3..496455e 100644 --- a/src/haloexec/__init__.py +++ b/src/haloexec/__init__.py @@ -1,25 +1,25 @@ -from .engine import Block, make_blocks, resolve_boundary_value -from .disk.workspace import MemmapRasterWorkspace from .disk.backend import WorkspaceRasterBackend +from .disk.convergence import sweep_until_convergence from .disk.io.geotiff import ( load_geotiff_into_workspace, load_geotiffs_into_workspace, save_workspace_to_geotiff, ) from .disk.io.zarr import load_zarr_into_workspace, load_zarr_tiles_into_workspace -from .disk.convergence import sweep_until_convergence +from .disk.workspace import MemmapRasterWorkspace +from .engine import Block, make_blocks, resolve_boundary_value __all__ = [ "Block", - "make_blocks", - "resolve_boundary_value", "MemmapRasterWorkspace", "WorkspaceRasterBackend", "load_geotiff_into_workspace", "load_geotiffs_into_workspace", - "save_workspace_to_geotiff", "load_zarr_into_workspace", "load_zarr_tiles_into_workspace", + "make_blocks", + "resolve_boundary_value", + "save_workspace_to_geotiff", "sweep_until_convergence", ] @@ -30,16 +30,16 @@ # (pip install "haloexec[dissmodel]"). Mesmo padrão usado em # pymangue/__init__.py para CMMAModel. try: + from .disk.cellular_automaton import DiskChunkedRasterCellularAutomaton + from .disk.sync_model import DiskChunkedSyncRasterModel, workspace_arrays_for_sync_model from .ram.cellular_automaton import HaloChunkedRasterCellularAutomaton from .ram.sync_model import HaloChunkedSyncRasterModel - from .disk.sync_model import DiskChunkedSyncRasterModel, workspace_arrays_for_sync_model - from .disk.cellular_automaton import DiskChunkedRasterCellularAutomaton __all__ += [ + "DiskChunkedRasterCellularAutomaton", + "DiskChunkedSyncRasterModel", "HaloChunkedRasterCellularAutomaton", "HaloChunkedSyncRasterModel", - "DiskChunkedSyncRasterModel", "workspace_arrays_for_sync_model", - "DiskChunkedRasterCellularAutomaton", ] except ImportError: pass diff --git a/src/haloexec/disk/backend.py b/src/haloexec/disk/backend.py index f46aba7..ff2559a 100644 --- a/src/haloexec/disk/backend.py +++ b/src/haloexec/disk/backend.py @@ -8,7 +8,6 @@ from __future__ import annotations -from typing import Any import numpy as np from .workspace import MemmapRasterWorkspace @@ -39,7 +38,7 @@ def __init__( self, workspace: MemmapRasterWorkspace, stride: int = 1, - nodata_value: float | int | None = None, + nodata_value: float | None = None, ) -> None: self.workspace = workspace self.stride = max(1, int(stride)) diff --git a/src/haloexec/disk/cellular_automaton.py b/src/haloexec/disk/cellular_automaton.py index 80f1f0f..d3bc647 100644 --- a/src/haloexec/disk/cellular_automaton.py +++ b/src/haloexec/disk/cellular_automaton.py @@ -19,12 +19,9 @@ from __future__ import annotations -import numpy as np - from dissmodel.geo.raster.backend import RasterBackend from .workspace import MemmapRasterWorkspace -from ..engine import resolve_boundary_value class DiskChunkedRasterCellularAutomaton: diff --git a/src/haloexec/disk/convergence.py b/src/haloexec/disk/convergence.py index 7395ee7..788780c 100644 --- a/src/haloexec/disk/convergence.py +++ b/src/haloexec/disk/convergence.py @@ -40,7 +40,7 @@ import numpy as np -from .workspace import MemmapRasterWorkspace, Block +from .workspace import Block, MemmapRasterWorkspace def sweep_until_convergence( diff --git a/src/haloexec/disk/io/geotiff.py b/src/haloexec/disk/io/geotiff.py index ea327ec..606f093 100644 --- a/src/haloexec/disk/io/geotiff.py +++ b/src/haloexec/disk/io/geotiff.py @@ -25,6 +25,7 @@ from __future__ import annotations from pathlib import Path +from typing import Any import numpy as np @@ -192,6 +193,7 @@ def save_workspace_to_geotiff( raise ImportError("rasterio é necessário — pip install -e '.[geotiff]'") import warnings + from rasterio.transform import from_origin path = Path(path) diff --git a/src/haloexec/disk/io/zarr.py b/src/haloexec/disk/io/zarr.py index b416ecb..982d99f 100644 --- a/src/haloexec/disk/io/zarr.py +++ b/src/haloexec/disk/io/zarr.py @@ -54,11 +54,18 @@ def _resolve_axis_order(arr, expected_names: tuple[str, ...]) -> tuple[int, ...] tem o MESMO shape nos dois casos — a checagem de shape sozinha não detecta a troca; é silenciosa, não trava. - Retorna None se dimension_names não estiver disponível (Zarr sem + No Zarr v2 (zarr-python 2.x, o único disponível em Python 3.10) + não existe dimension_names: o xarray grava os nomes no atributo + `_ARRAY_DIMENSIONS`, que é lido como fallback. + + Retorna None se nenhum dos dois estiver disponível (Zarr sem metadado de dimensão — não há como verificar, assume-se a ordem como está, mesmo comportamento de antes desta correção). """ dims = getattr(getattr(arr, "metadata", None), "dimension_names", None) + if not dims: + attrs = getattr(arr, "attrs", None) + dims = attrs.get("_ARRAY_DIMENSIONS") if attrs is not None else None if not dims: return None dims = tuple(dims) @@ -184,7 +191,7 @@ def load_zarr_into_workspace( # raw ainda está na ordem relativa em disco (menos o # eixo de tempo, já reduzido pela indexação inteira # acima) -- transpõe para (y, x) canônico. - def _shift(pos): + def _shift(pos, time_disk_axis=time_disk_axis): return pos - 1 if (time_disk_axis is not None and time_disk_axis < pos) else pos data = np.transpose(raw, (_shift(y_disk_axis), _shift(x_disk_axis))) else: diff --git a/src/haloexec/disk/sync_model.py b/src/haloexec/disk/sync_model.py index 6747a19..10bbfdd 100644 --- a/src/haloexec/disk/sync_model.py +++ b/src/haloexec/disk/sync_model.py @@ -49,7 +49,6 @@ class FloodModelDiskHalo(DiskChunkedSyncRasterModel, FloodModel): from __future__ import annotations import numpy as np - from dissmodel.geo.raster.backend import RasterBackend from .workspace import MemmapRasterWorkspace diff --git a/src/haloexec/disk/workspace.py b/src/haloexec/disk/workspace.py index 1a85103..9cbebd9 100644 --- a/src/haloexec/disk/workspace.py +++ b/src/haloexec/disk/workspace.py @@ -150,7 +150,7 @@ def create( block_h: int, block_w: int, halo: int = 1, - ) -> "MemmapRasterWorkspace": + ) -> MemmapRasterWorkspace: """Cria um workspace novo, com os dois slots do double-buffer. Os arquivos `.dat` nascem ESPARSOS: só as regiões efetivamente @@ -314,7 +314,7 @@ def flush(self) -> None: for mm in slot.values(): mm.flush() - def as_backend(self, stride: int = 1, nodata_value: float | int | None = None): + def as_backend(self, stride: int = 1, nodata_value: float | None = None): """Devolve um adaptador WorkspaceRasterBackend para uso com RasterMap/dissmodel.""" from .backend import WorkspaceRasterBackend return WorkspaceRasterBackend(self, stride=stride, nodata_value=nodata_value) diff --git a/src/haloexec/ram/cellular_automaton.py b/src/haloexec/ram/cellular_automaton.py index 31f5a65..52b82eb 100644 --- a/src/haloexec/ram/cellular_automaton.py +++ b/src/haloexec/ram/cellular_automaton.py @@ -37,7 +37,6 @@ from __future__ import annotations import numpy as np - from dissmodel.geo.raster.backend import RasterBackend from dissmodel.geo.raster.cellular_automaton import RasterCellularAutomaton diff --git a/src/haloexec/ram/sync_model.py b/src/haloexec/ram/sync_model.py index 745a0e9..84b16ac 100644 --- a/src/haloexec/ram/sync_model.py +++ b/src/haloexec/ram/sync_model.py @@ -37,7 +37,6 @@ class FloodModelHalo(HaloChunkedSyncRasterModel, FloodModel): from __future__ import annotations import numpy as np - from dissmodel.geo.raster.backend import RasterBackend from ..engine import make_blocks, resolve_boundary_value diff --git a/src/haloexec/visualization.py b/src/haloexec/visualization.py index b8781b6..4d2c381 100644 --- a/src/haloexec/visualization.py +++ b/src/haloexec/visualization.py @@ -7,7 +7,8 @@ from __future__ import annotations -from typing import Any, Iterable +from collections.abc import Iterable +from typing import Any try: from dissmodel.visualization.raster_map import RasterMap diff --git a/tests/test_convergence.py b/tests/test_convergence.py index 905a1fe..ec92fbe 100644 --- a/tests/test_convergence.py +++ b/tests/test_convergence.py @@ -87,7 +87,7 @@ def test_sweep_until_convergence_stress_random_seeds(tmp_path, seed): seeds, permeable = _labyrinth_scenario(35, 35, seed) golden = _run_monolithic(seeds, permeable) - chunked, info = _run_chunked(tmp_path, seeds, permeable, block_h=7, block_w=7) + chunked, _info = _run_chunked(tmp_path, seeds, permeable, block_h=7, block_w=7) assert np.array_equal(golden, chunked) diff --git a/tests/test_equivalence.py b/tests/test_equivalence.py index 03424ca..53b6be3 100644 --- a/tests/test_equivalence.py +++ b/tests/test_equivalence.py @@ -18,6 +18,8 @@ import numpy as np import pytest +pytest.importorskip("dissmodel") + from dissmodel.core import Environment from dissmodel.geo.raster.backend import RasterBackend from dissmodel.geo.raster.cellular_automaton import RasterCellularAutomaton diff --git a/tests/test_geotiff_io_equivalence.py b/tests/test_geotiff_io_equivalence.py index aa6121b..20fc4b5 100644 --- a/tests/test_geotiff_io_equivalence.py +++ b/tests/test_geotiff_io_equivalence.py @@ -14,7 +14,6 @@ from haloexec import MemmapRasterWorkspace, load_geotiff_into_workspace - BAND_SPEC = [ ("uso", "int16", 0), ("alt", "float32", -9999.0), diff --git a/tests/test_gol_patterns_example.py b/tests/test_gol_patterns_example.py index cd5ff97..8ff2103 100644 --- a/tests/test_gol_patterns_example.py +++ b/tests/test_gol_patterns_example.py @@ -27,9 +27,9 @@ from dissmodel.core import Environment from dissmodel.geo import raster_grid from dissmodel.geo.raster.cellular_automaton import RasterCellularAutomaton +from dissmodel_ca.models.game_of_life import PATTERNS from haloexec import HaloChunkedRasterCellularAutomaton -from dissmodel_ca.models.game_of_life import PATTERNS ROWS, COLS = 40, 40 GENERATIONS = 16 diff --git a/tests/test_visualization_adapter.py b/tests/test_visualization_adapter.py index 1aabc0c..191901e 100644 --- a/tests/test_visualization_adapter.py +++ b/tests/test_visualization_adapter.py @@ -3,6 +3,7 @@ """ from pathlib import Path + import numpy as np import pytest @@ -11,6 +12,7 @@ pytest.importorskip("dissmodel") from dissmodel.core import Environment from dissmodel_ca.models.game_of_life_raster import GameOfLife + from haloexec import DiskChunkedRasterCellularAutomaton from haloexec.visualization import CheckpointRasterMap @@ -96,9 +98,10 @@ def test_checkpoint_raster_map_filtering(tmp_path: Path, monkeypatch): def test_save_workspace_to_geotiff_roundtrip(tmp_path: Path): - from haloexec import save_workspace_to_geotiff, load_geotiff_into_workspace import rasterio + from haloexec import save_workspace_to_geotiff + shape = (60, 80) ws1 = MemmapRasterWorkspace.create( root=tmp_path / "ws1", diff --git a/tests/test_zarr_axis_order_regression.py b/tests/test_zarr_axis_order_regression.py index a0c5f8c..5c28c73 100644 --- a/tests/test_zarr_axis_order_regression.py +++ b/tests/test_zarr_axis_order_regression.py @@ -111,3 +111,27 @@ def test_load_zarr_handles_txy_axis_order_temporal(tmp_path): load_zarr_into_workspace(ws, str(store), variable_map={"mangue": "mangue"}, time_index=1) assert np.array_equal(ws.snapshot("mangue"), serie[1]) + + +def test_load_zarr_handles_xy_axis_order_zarr_v2_format(tmp_path): + """Store no FORMATO Zarr v2 (ex.: gravado por xarray/disscube mais + antigos, ou com zarr_format=2): não existe metadata.dimension_names, + o xarray guarda os nomes no atributo `_ARRAY_DIMENSIONS`. Mesmo + lendo com zarr-python 3, sem o fallback para esse atributo o array + quadrado (x, y) era carregado TRANSPOSTO, silenciosamente.""" + n = 6 + data = np.arange(n * n).reshape(n, n).astype("int16") + store = tmp_path / "v2.zarr" + da = xr.DataArray(data.T, dims=("x", "y"), name="uso") + da.to_dataset(name="uso").to_zarr(str(store), mode="w", consolidated=False, zarr_format=2) + + ws = MemmapRasterWorkspace.create( + root=tmp_path / "workspace", shape=(n, n), + arrays={"uso": np.int16}, block_h=3, block_w=3, halo=1, + ) + load_zarr_into_workspace(ws, str(store), variable_map={"uso": "uso"}) + + assert np.array_equal(ws.snapshot("uso"), data), ( + "Zarr formato v2 com eixos (x, y) carregado transposto -- " + "fallback para _ARRAY_DIMENSIONS ausente" + ) diff --git a/tests/test_zarr_tiles_integration.py b/tests/test_zarr_tiles_integration.py index b50234b..c635468 100644 --- a/tests/test_zarr_tiles_integration.py +++ b/tests/test_zarr_tiles_integration.py @@ -23,7 +23,7 @@ zarr = pytest.importorskip("zarr") -from haloexec import ( # noqa: E402 +from haloexec import ( Block, MemmapRasterWorkspace, load_zarr_tiles_into_workspace, @@ -238,7 +238,7 @@ def test_tile_outside_workspace_raises(tmp_path): def test_layout_shape_matches_disscube(tmp_path): """Fixa que as chaves que este loader exige são as que o CubeClient.tile_layout() produz. Sem o disscube instalado, pula.""" - disscube = pytest.importorskip("disscube") + pytest.importorskip("disscube") from disscube.client import CubeClient from disscube.models import DerivedVariable, GridSpec, SpatialSource