diff --git a/.claude/agents/api-backward-compatibility-specialist.md b/.claude/agents/api-backward-compatibility-specialist.md new file mode 100644 index 0000000000..ab3aac110b --- /dev/null +++ b/.claude/agents/api-backward-compatibility-specialist.md @@ -0,0 +1,21 @@ +--- +name: api-backward-compatibility-specialist +description: Protects users and integrators by ensuring API changes are backwards compatible, properly versioned, and well-documented +--- + +# Agent: API & Backward Compatibility Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/api-backward-compatibility-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/architecture-domain-specialist.md b/.claude/agents/architecture-domain-specialist.md new file mode 100644 index 0000000000..4cbdad7a06 --- /dev/null +++ b/.claude/agents/architecture-domain-specialist.md @@ -0,0 +1,21 @@ +--- +name: architecture-domain-specialist +description: Guards domain model, invariants, and architecture to maintain model clarity and prevent erosion of core principles +--- + +# Agent: Architecture & Domain Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/architecture-domain-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/coordinator.md b/.claude/agents/coordinator.md new file mode 100644 index 0000000000..9077ea39d4 --- /dev/null +++ b/.claude/agents/coordinator.md @@ -0,0 +1,21 @@ +--- +name: coordinator +description: Meta-agent that manages agent lifecycle, enforces structural standards, and maintains coherence across the agent system +--- + +# Agent: Coordinator + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/coordinator.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/data-time-semantics-specialist.md b/.claude/agents/data-time-semantics-specialist.md new file mode 100644 index 0000000000..b01e477c4a --- /dev/null +++ b/.claude/agents/data-time-semantics-specialist.md @@ -0,0 +1,21 @@ +--- +name: data-time-semantics-specialist +description: Prevents subtle bugs in time handling, units, and data semantics with focus on timezone-aware operations and unit conversions +--- + +# Agent: Data & Time Semantics Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/data-time-semantics-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/documentation-developer-experience-specialist.md b/.claude/agents/documentation-developer-experience-specialist.md new file mode 100644 index 0000000000..9d0831d298 --- /dev/null +++ b/.claude/agents/documentation-developer-experience-specialist.md @@ -0,0 +1,21 @@ +--- +name: documentation-developer-experience-specialist +description: Ensures excellent documentation, clear error messages, and smooth developer workflows to keep FlexMeasures accessible +--- + +# Agent: Documentation & Developer Experience Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/documentation-developer-experience-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/performance-scalability-specialist.md b/.claude/agents/performance-scalability-specialist.md new file mode 100644 index 0000000000..9786881e2f --- /dev/null +++ b/.claude/agents/performance-scalability-specialist.md @@ -0,0 +1,21 @@ +--- +name: performance-scalability-specialist +description: Identifies performance bottlenecks, inefficient algorithms, and scalability issues to keep FlexMeasures fast under load +--- + +# Agent: Performance & Scalability Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/performance-scalability-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/test-specialist.md b/.claude/agents/test-specialist.md new file mode 100644 index 0000000000..39fcafcadf --- /dev/null +++ b/.claude/agents/test-specialist.md @@ -0,0 +1,21 @@ +--- +name: test-specialist +description: Focuses on test coverage, quality, and testing best practices without modifying production code +--- + +# Agent: Test Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/test-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/tooling-ci-specialist.md b/.claude/agents/tooling-ci-specialist.md new file mode 100644 index 0000000000..02bf4b0620 --- /dev/null +++ b/.claude/agents/tooling-ci-specialist.md @@ -0,0 +1,21 @@ +--- +name: tooling-ci-specialist +description: Reviews GitHub Actions workflows, pre-commit hooks, and CI/CD pipelines to ensure automation reliability +--- + +# Agent: Tooling & CI Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/tooling-ci-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.claude/agents/ui-specialist.md b/.claude/agents/ui-specialist.md new file mode 100644 index 0000000000..a985cf6f09 --- /dev/null +++ b/.claude/agents/ui-specialist.md @@ -0,0 +1,21 @@ +--- +name: ui-specialist +description: Guards UI consistency, permission patterns, JavaScript interaction patterns, and template quality in the FlexMeasures web interface +--- + +# Agent: UI Specialist + +This is a thin Claude Code pointer file. Agent logic is maintained once, for both Claude Code +and GitHub Copilot, in the `.github` folder: + +- **`.github/agents/ui-specialist.md`** — the full agent definition: role, scope, review checklist, + domain knowledge, interaction rules, and self-improvement notes. Read this file in full before + acting as this agent. When agent behavior needs to change, edit that file, not this one. +- **`.github/instructions/`** — project-wide conventions shared by every agent (atomic commits, + changelog entries, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, + timezone awareness, UI terminology). +- **`.github/workflows/copilot-setup-steps.yml`** — the reference environment setup (system + packages, Python/uv setup, database, environment variables) used for GitHub Copilot's cloud + agents. Claude Code agents run in their own sandboxed environment, not this one, so treat this + file as a reference for expected dependencies and services rather than a script to execute + verbatim. diff --git a/.github/agents/api-backward-compatibility-specialist.md b/.github/agents/api-backward-compatibility-specialist.md index da57b072fe..3d2627e493 100644 --- a/.github/agents/api-backward-compatibility-specialist.md +++ b/.github/agents/api-backward-compatibility-specialist.md @@ -219,6 +219,65 @@ value = params.get("sensor_to_save") - **Architecture Specialist**: Enforces "schema as source of truth" invariant - **API Specialist**: Verifies API documentation matches format +#### Data Format Mismatch Pattern + +**Problem** (PR #2072 - Constraint Analysis): + +Data transformations between API layers can silently use incompatible key types: +- Layer 1 produces asset-keyed results +- Layer 2 expects sensor-keyed results +- No schema validation catches the mismatch +- Silent data corruption in API response + +**Manifestation**: +```python +# Layer 1: Produces asset_id keyed dict +results = {asset_id: [values]} # ✅ Correct + +# Layer 2: Function assumes sensor_id keys +def _sensor_keyed_to_asset_keyed(sensor_keyed_results): # ❌ Misleading name + # Actually receives asset-keyed data + # Treats asset_ids as sensor_ids + # Returns corrupted mapping +``` + +**Prevention Checklist**: + +1. **Function naming must indicate format**: Use clear names like `_asset_keyed_to_list()` not `_transform_results()` +2. **Marshmallow schemas for transforms**: Add explicit response schemas, don't rely on inline OpenAPI: + ```python + class ConstraintAnalysisResponseSchema(Schema): + """Validates end-to-end data format""" + asset_id = fields.Int(required=True) + data = fields.Nested(ConstraintDataSchema, many=True) + ``` +3. **Integration tests verify data flow**: Test that end-to-end transformations preserve data semantics: + ```python + def test_constraint_analysis_returns_asset_keyed_results(): + result = constraint_analysis_transform(asset_data) + assert all(isinstance(k, int) for k in result.keys()), "Keys must be asset IDs" + # NOT just: assert result is not None + ``` +4. **Document key types in docstrings**: Be explicit about what each layer expects/produces: + ```python + def analyze_constraints(data): + """Transform constraint results. + + Args: + data: asset_id -> constraint_list mapping + + Returns: + dict: asset_id -> formatted_constraints mapping (keys are asset IDs, not sensor IDs) + """ + ``` + +**Why This Matters for Backward Compatibility**: +- Silent data corruption breaks client contracts subtly +- Clients may succeed but get wrong data +- Testing may pass with mismatched formats if tests don't validate keys +- Makes it hard to reason about what the API actually returns +- Requires coordination with clients to fix (breaking change) + ### CLI Command Changes - [ ] **Argument changes**: Adding required args breaks scripts diff --git a/.github/agents/architecture-domain-specialist.md b/.github/agents/architecture-domain-specialist.md index 0133406645..6e65db4a21 100644 --- a/.github/agents/architecture-domain-specialist.md +++ b/.github/agents/architecture-domain-specialist.md @@ -577,6 +577,93 @@ After each assignment: - Added guidance on ``` +### Asset ID Keying Pattern (PR #2072) + +When scheduling results or constraint analysis shift from sensor-keyed to asset-keyed organization, you must prevent silent data corruption across data flow boundaries. + +**Problem**: A storage scheduler optimization changed constraint analysis results from sensor-keyed to asset-keyed organization. Without careful attention, Layer 1 (storage) produces asset-keyed dicts while Layer 2 (API) treats keys as sensor IDs, silently corrupting results. + +**Pattern**: + +1. **Identify all data flow stages**: + - **Storage layer**: How does the scheduler compute and store results? + - **API transformation**: How are results converted for the API response? + - **Client expectations**: What format do clients expect? + + Example: Constraint analysis produces Dict[int, Dict] where int is asset_id. API must transform this for clients while maintaining correctness. + +2. **Prevent format mismatches at boundaries**: + - Layer 1 produces asset-keyed dict (e.g., `{asset_id: {...}}`) + - Layer 2 MUST expect asset-keyed dict (not assume sensor-keyed) + - Add Marshmallow schemas to validate the transformation + - No silent conversions without schema validation + + Example pattern: + ```python + # ❌ Wrong: storage produces asset-keyed, API treats as sensor-keyed + constraint_result = {1: {}} # asset_id 1 from scheduler + for sensor_id, data in constraint_result.items(): # Treats key as sensor_id! + ... + + # ✅ Correct: explicitly document and validate transformation + @dataclass + class ConstraintDataPerAsset: + """Storage layer produces Dict[asset_id, ...]""" + asset_id_keyed_result: Dict[int, Dict] + + class ConstraintResponseSchema(Schema): + """API transforms to client format but documents asset keying""" + assets = fields.List(fields.Nested(...)) # Asset-keyed response + ``` + +3. **Update domain invariants**: + - Asset ID is the authoritative key (not sensor ID or device index) + - Multiple sensors may belong to the same asset + - Constraint analysis results grouped by asset, not sensor + - Document this in docstrings and type hints + + Example invariant addition to docstrings: + ```python + def get_constraints_for_assets( + asset_ids: List[int], + ... + ) -> Dict[int, ConstraintData]: + """Get constraint analysis results grouped by asset ID. + + Key domain invariant: Results are keyed by asset_id, not sensor_id. + Multiple sensors may belong to the same asset, but constraints + are computed and reported per-asset for scheduling purposes. + + Returns: + Dict mapping asset_id -> ConstraintData + """ + ``` + +4. **Test the transformation end-to-end**: + - Storage layer produces one format (may be optimized for efficiency) + - API layer transforms to client format (asset-keyed) + - Integration tests verify both format correctness and no data loss + + Example test structure: + ```python + def test_constraint_results_keyed_by_asset(): + """Verify storage→API transformation preserves asset keying""" + # Storage layer produces asset-keyed result + result = scheduler.get_constraints() + assert all(isinstance(k, int) for k in result.keys()) # asset IDs + + # API layer transforms for response + response = transform_for_api(result) + assert response["assets"] # asset-keyed response structure + + # Verify no data loss or corruption + assert len(response["assets"]) == len(result) + ``` + +**Why it matters**: Asset-keyed results differ fundamentally from sensor-keyed results. A storage scheduler with 10 assets but 30 sensors uses asset-keyed for efficiency, but the API must still make this explicit to prevent Layer 2 from misinterpreting keys. Silent misinterpretation corrupts scheduling results without throwing errors. + +**Review trigger**: Any scheduler or constraint analysis code that changes result keying (from device-index to asset-id, or sensor-keyed to asset-keyed) — Add this documentation pattern and require tests that verify the transformation is not corrupted. + ### Lessons Learned **Session 2026-03-24 (PR #2058 — add account_id to DataSource)**: @@ -591,3 +678,10 @@ After each assignment: - **Schema parity gap**: The PR added `account_id` to `BeliefsSearchConfigSchema` but not to `Input` (io.py). These two schemas both expose `Sensor.search_beliefs` parameters; omitting a parameter from one creates a silent gap. The architecture agent must check both schemas on any search_beliefs parameter addition. - **Documentation vs. implementation mismatch**: The `reporting.rst` docs stated reporters can filter by `account_id`, but this only works if `Input` also has the field. Docs that outrun schema support mislead users. Always verify the full schema chain before documenting a feature. - **DataSource account_id=None for non-user sources**: The existing invariant (reporters/schedulers/forecasters have `account_id=None`) limits the usefulness of `account_id` filtering: it only matches user-type sources. PRs adding `account_id` filters should either document this limitation explicitly or reconsider the invariant. + +**Session 2026-XX (PR #2072 — storage scheduler asset keying optimization)**: + +- **Data flow format mismatches**: Storage scheduler optimization changed constraint analysis from sensor-keyed to asset-keyed results. The risk is high: Layer 1 produces asset-keyed dict, Layer 2 silently treats keys as sensor IDs, corrupting results without errors. This pattern must be documented and tested end-to-end. +- **Multi-sensor per asset invariant**: Asset ID is the authoritative key, not sensor ID. Multiple sensors belong to the same asset; constraint results are grouped by asset for scheduling purposes. Docstrings and type hints must make this explicit to prevent misuse. +- **Silent data corruption risk**: Unlike exceptions, format mismatches silently corrupt data. When keying changes (sensor→asset, device-index→asset-id), integration tests must verify the full transformation (storage format → API format) maintains data correctness and no loss. +- **Added Asset ID Keying Pattern**: New section in instructions documents the pattern, data flow stages, format validation, domain invariants, and end-to-end testing requirements. diff --git a/.github/agents/documentation-developer-experience-specialist.md b/.github/agents/documentation-developer-experience-specialist.md index 29a1d9eaec..74dd245afd 100644 --- a/.github/agents/documentation-developer-experience-specialist.md +++ b/.github/agents/documentation-developer-experience-specialist.md @@ -376,6 +376,77 @@ make update-docs - Testing all imports work before finalizing - ~500 lines for complete feature coverage +- **PR #2072 (Scheduling Constraints Terminology)**: Cross-document terminology updates require systematic consistency checks: + - Terminology changes must be applied consistently across all affected documents + - Field references and terminology need updates in: docstrings, API docs, feature guides, and changelogs + - Validate completeness with grep searches to catch orphaned references + - Follow update order: code source → documentation → changelog + +### Cross-Document Consistency Pattern + +When updating API terminology that appears in multiple places (e.g., removing fields, renaming concepts, changing constraint names): + +1. **Identify all affected documents**: + ```bash + grep -r "old_term" documentation/ + grep -r "old_term" flexmeasures/ + ``` + Look for: API docs, feature guides, changelog, docstrings, type hints, error messages + +2. **Update in order of authority** (source of truth first): + - Code docstrings and type hints (source of truth) + - Inline comments in code explaining the terms + - Feature documentation (`documentation/features/`) + - API reference and endpoints (`documentation/api/`) + - API changelog (`documentation/api/change_log.rst`) + - Deprecation warnings (if applicable) + +3. **Ensure consistency**: + - Use same terminology in all files + - Same capitalization and formatting + - Same context and examples + - Field references match across all docs + +4. **Verify completeness**: + ```bash + # After updates, verify old term is replaced everywhere + grep -r "old_term" documentation/ flexmeasures/ + # Should return zero matches (except in changelog when documenting deprecation) + ``` + +5. **Document the change**: + - Changelog entry explaining what changed and why + - Migration path if it's a breaking change + - Note affected endpoint versions + +6. **Commit strategy**: + - Commit docs and code changes together in logical groups + - Separate changelog commit if it's substantial + - Use atomic commits: one file or one logical change per commit + +**Example workflow:** + +```bash +# 1. Search for all references to old terminology +grep -r "scheduling_result" documentation/ flexmeasures/ + +# 2. Update code docstrings first (they are the source of truth) +# - Update parameter descriptions +# - Update field names in class docstrings +# - Update return value documentation + +# 3. Update feature guides and API docs +# - Replace field references in examples +# - Update field descriptions in API documentation +# - Update endpoint descriptions + +# 4. Update changelog with clear migration notes + +# 5. Final verification +grep -r "scheduling_result" documentation/ flexmeasures/ +# Should show only changelog entries mentioning the removal +``` + ### Continuous Improvement - Monitor user questions (docs should answer them) diff --git a/.github/agents/test-specialist.md b/.github/agents/test-specialist.md index 2ebdf13979..9bd90677f6 100644 --- a/.github/agents/test-specialist.md +++ b/.github/agents/test-specialist.md @@ -605,6 +605,43 @@ When fixing failing tests, ALWAYS follow this test-driven approach: - What pattern or pitfall should be remembered? - What verification step was missing? +### Data Format Transformation Testing + +When testing API layers that transform data (e.g., sensor-keyed → asset-keyed): + +**Session learned (2026-03-24, PR #2072 — constraint analysis storage scheduler)**: +- 1281 tests verified for constraint analysis changes +- Tested asset-keyed vs sensor-keyed data formats +- Confirmed storage scheduler interaction correctness + +**1. Verify key types explicitly**: +- ✅ Type check first: `assert isinstance(result, dict), "Result must be a dict"` +- ✅ Then verify key types: `assert all(isinstance(k, int) for k in result.keys()), "Keys must be asset IDs"` +- ❌ `assert result is not None` (silent type mismatches when result is list, string, etc.) + +**2. Test both directions if bidirectional**: +- Forward: input → output format +- Reverse: output can be deserialized correctly +- Verify data semantics survive round-trip without corruption + +**3. Use integration tests for transforms**: +- Test actual job.meta serialization/deserialization +- Verify end-to-end from storage to API response +- Don't rely on mock-only tests for format validation +- Test with real database fixtures, not just stubs + +**4. Prevent silent data corruption**: +- Test assertions should verify data semantics, not just null checks +- Example: If transforming sensor data to asset-keyed format, assert that asset IDs are correct (not sensor IDs) +- Check constraint violations reference correct entity types +- Verify cross-references maintain semantic integrity + +**Pattern Detection Tips**: +- Format transforms often hide type errors (int vs string keys) +- Mocks don't catch serialization/deserialization bugs +- Data corruption is silent — assertions on intermediate values only +- Integration tests catch edge cases mocks miss + ## Interaction Rules - When a failing test reveals a production bug, fix the production code and escalate the area to the relevant domain specialist (Architecture, API, Data & Time) for a broader review. diff --git a/.github/instructions/docstrings.instructions.md b/.github/instructions/docstrings.instructions.md index 5c44e6e351..602887b027 100644 --- a/.github/instructions/docstrings.instructions.md +++ b/.github/instructions/docstrings.instructions.md @@ -17,7 +17,7 @@ def function_name(param1: str, param2: int) -> bool: :param param1: Description of param1. :param param2: Description of param2. - :return: Description of return value. + :returns: Description of return value. :raises ValueError: When param1 is empty. Example:: @@ -35,6 +35,7 @@ def function_name(param1: str, param2: int) -> bool: - Use `Example::` (double colon) to introduce a doctest block. - Complement type hints — don't duplicate them in the docstring text. - Use exactly one space after punctuation (no double spaces after periods). +- Use line breaks only after punctuation (this facilitates review commenting and text searching). ## Click CLI commands diff --git a/.github/instructions/feature-branch-sync.instructions.md b/.github/instructions/feature-branch-sync.instructions.md new file mode 100644 index 0000000000..926d917fac --- /dev/null +++ b/.github/instructions/feature-branch-sync.instructions.md @@ -0,0 +1,42 @@ +--- +applyTo: "**" +--- +# Feature Branch Synchronization + +Feature branches must be kept synchronized with `origin/main` before implementing code changes. + +## Check branch status + +Before starting implementation work, verify the branch is up to date: + +```bash +git log --oneline origin/main...HEAD --left-right +``` + +If you see < markers, origin/main has commits the branch lacks — a fresh merge is needed. + +```bash +# ❌ Don't just check git status (it only tells you about uncommitted changes) +git status # shows "nothing to commit" even if behind main + +# ✅ Do check the commit graph +git log --left-right origin/main...HEAD +``` + +## Merge before implementation + +```bash +git fetch origin +git merge origin/main +# Resolve any conflicts +git add . +git commit -m "Merge origin/main into feature branch" +``` + +This ensures your implementation starts from the latest state of the repository. + +## Why this matters + +- Merging later causes merge conflicts to compound +- Large late merges are harder to review +- Feature work should build on current main, not diverge diff --git a/AGENTS.md b/AGENTS.md index c1b9d3d2d6..c683e88dbd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1234,6 +1234,12 @@ Track and document when the Lead: - **Key insight**: "Inspecting code is not a substitute for a green test — write the test first and let it prove or disprove the concern." +**Specific lesson learned (2026-06 feature branch sync)**: +- **Session**: Computing first unmet targets +- **Discovery**: Feature branch was 10+ commits behind `origin/main`; need explicit process rule +- **Prevention**: Added `.github/instructions/feature-branch-sync.instructions.md` to guide all agents +- **Key insight**: "Branch status checks must use git log, not git status — the latter only shows uncommitted changes" + Update this file to prevent repeating the same mistakes. ## Session Close Checklist (MANDATORY) @@ -1296,6 +1302,7 @@ This is a regression (see Regression Prevention section). You MUST: ### Pre-Commit Verification +- [ ] **Branch in sync with main**: Run `git log --oneline origin/main...HEAD --left-right` — if `<` markers exist, `origin/main` has commits the branch lacks; merge before proceeding. - [ ] **All hooks pass**: `pre-commit run --all-files` (see `.github/instructions/pre-commit-hooks.instructions.md`) - [ ] **Changes committed**: If hooks modified files, changes included in commit diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 19803bca98..ac092c48cd 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,6 +5,14 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace. +v3.0-32 | July XX, 2026 +"""""""""""""""""""""""" + +- [**Breaking change**] For a finished scheduling job, the ``result`` field of ``GET /api/v3_0/jobs/`` is now always an object, instead of the boolean ``true`` it used to return unconditionally on success. + For a ``StorageScheduler`` job, it holds soft state-of-charge constraint analysis: ``unresolved`` and ``resolved`` arrays (each keyed by asset ID) with ``soc-minima``/``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are simply empty (``{"unresolved": [], "resolved": []}``) when no such constraints were defined. + Scheduling jobs using a different scheduler (e.g. ``ProcessScheduler``) return an empty object (``{}``) for now, pending their own result specification. + This may affect external integrators, such as custom scripts, FlexMeasures plugins and API client code (the ``flexmeasures-client`` package is not affected), that check ``result === true`` (or ``result is True`` in Python) unconditionally on a finished scheduling job; such clients should instead check for a truthy ``result``, or explicitly handle the object shape. + v3.0-31 | 2026-06-01 """""""""""""""""""" diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst index 39ac273083..c3efd0df28 100644 --- a/documentation/api/v3_0.rst +++ b/documentation/api/v3_0.rst @@ -10,7 +10,7 @@ A quick overview of the available endpoints. For more details, click their names .. The qrefs make links very similar to the openapi plugin, but we have to run a sed command after the fact to make them exactly alike (see the update-docs poe task) .. qrefflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.accounts, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health, flexmeasures.api.v3_0.public + :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.jobs, flexmeasures.api.v3_0.accounts, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health, flexmeasures.api.v3_0.public :order: path :include-empty-docstring: diff --git a/documentation/changelog.rst b/documentation/changelog.rst index e37cd914aa..b87665d4a0 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,6 +17,7 @@ New features * Sensor references in flex-model and flex-context support various ways of filtering by source [see `PR #2209 `_] * Let storage scheduling infer missing ``power-capacity`` from directional device capacities before falling back to site capacity, and default the missing opposite capacity to zero when only a non-zero ``consumption-capacity`` or ``production-capacity`` is configured [see `PR #2222 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] +* Add soft constraint analysis (``unresolved`` and ``resolved`` SoC constraints per asset) to scheduling job results: the ``result`` field of ``GET /api/v3_0/jobs/`` for a finished scheduling job is now always an object instead of the boolean ``true`` it used to return unconditionally on success; a ``StorageScheduler`` job populates it with this analysis (empty arrays when no ``soc-minima``/``soc-maxima`` were defined), while other schedulers return an empty object for now [see `PR #2072 `_] Infrastructure / Support ---------------------- @@ -66,11 +67,11 @@ v0.33.0 | June 1, 2026 New features ------------- * Added API and UI support for copying assets and their subtrees [see `PR #2017 `_ and `PR #2120 `_] +* Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * Improve UX after deleting a child asset through the UI [see `PR #2119 `_] * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_ and `PR #2151 `_] * Support sensor references for efficiency fields in storage flex-models [see `PR #2142 `_] * Introduce the ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` to save schedules to [see `PR #2190 `_ and `PR #2213 `_] -* Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * Add ``flexmeasures jobs inspect-job`` CLI command to show job status and metadata information (similar to the job status endpoint in the API) [see `PR #2202 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] * Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 6399d867af..d966ba301c 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -323,8 +323,164 @@ You can add new shiftable-process schedules with the CLI command ``flexmeasures .. note:: Currently, the ``ProcessScheduler`` uses only the ``consumption-price`` field of the flex-context, so it ignores any site capacities and inflexible devices. -Work on other schedulers --------------------------- +The schedule +------------ + +A schedule produced by FlexMeasures is a series of power values for each flexible device (represented by its power sensor), covering the scheduling window at the scheduling resolution. + +For detailed constraint analysis (unresolved constraints and margins), use the ``GET /api/v3_0/jobs/`` endpoint, which provides structured information about constraints organized by asset. See the :ref:`scheduling_constraint_results` section below for details. + + +.. _scheduling_constraint_results: + +Accessing constraint results +----------------------------- + +When a schedule is computed for a device with state-of-charge constraints, FlexMeasures analyzes whether the constraints can be met. + +Use the **jobs endpoint** (``GET /api/v3_0/jobs/``) to retrieve detailed constraint analysis for all assets involved in the scheduling job, organized by asset ID. +This endpoint is useful when you want to inspect constraint violations without retrieving the full schedule. + +Multi-asset scheduling workflow +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Consider a site (asset ID 123) with four assets, each with a power sensor: + +- **Sensors 1 & 2**: Inflexible devices (e.g. PV panel and building load) +- **Sensors 3 & 4**: Flexible devices (e.g. a battery and an EV charger), + each with a state-of-charge sensor (sensors 5 and 6, respectively) + +The scheduling workflow looks like this: + +1. **Trigger the schedule** for site asset 123 via + ``POST /api/v3_0/assets/123/schedules/trigger``. + The endpoint returns a job UUID, e.g. ``"5d28df1b-9f16-4177-ae43-6e750d80fad3"``. + +2. **Retrieve the scheduled power series** for the flexible devices once scheduling is done, + via ``GET /api/v3_0/sensors/3/schedules/`` and ``GET /api/v3_0/sensors/4/schedules/``. + Each response contains the power setpoints for that device: + + .. code-block:: json + + { + "values": [0.5, 1.0, 1.5, 0.0], + "start": "2024-01-15T08:00:00+00:00", + "duration": "PT4H", + "unit": "kW" + } + +3. **Retrieve constraint analysis** for all flexible assets via ``GET /api/v3_0/jobs/``. + The ``result`` field in the response shows whether the state-of-charge targets for sensors 5 and 6 could be met, and by how much. + For a finished ``StorageScheduler`` job, ``result`` is always an object with ``unresolved`` and ``resolved`` constraint analysis (as shown below); + both arrays are simply empty when the flex model defines no ``soc-minima``/``soc-maxima``, or when a scheduler other than ``StorageScheduler`` was used. + +The constraint results distinguish between: + +- Constraints that were **unresolved**: Soft constraints that could not be satisfied during optimization, with the shortfall or excess reported as their **violation**. +- Constraints that were **resolved**: Soft constraints that were satisfied, with the headroom remaining reported as their **margin**. + +For each device, the ``soc-minima``/``soc-maxima`` value under ``unresolved`` or ``resolved`` is a **list** of entries — one per violated slot (unresolved) or per met slot with its margin (resolved), ordered chronologically. +By default, every violated or met slot is listed (this is not currently configurable via the API). +Each list entry includes: + +- ``datetime``: ISO 8601 UTC timestamp of that slot. +- ``violation`` (unresolved only): Magnitude of the violation at that slot (shortage for minima, excess for maxima). +- ``margin`` (resolved only): Headroom remaining at that slot. + + +Example: Constraint results from a battery scheduling job +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Suppose you schedule a battery device (asset ID 42) with the following constraints: + +- **soc-minima**: Battery must stay above 60 kWh +- **soc-maxima**: Battery must not exceed 100 kWh + +If the optimization cannot satisfy the minimum constraint at 10:30 UTC (falling short by 20 kWh) and again at 10:45 UTC (falling short by 15 kWh), +but does satisfy the maximum constraint with margins of 40 kWh at 11:00 UTC and 35 kWh at 12:00 UTC, the constraint results would show: + +**Response via GET /api/v3_0/jobs/:** + +.. code-block:: json + + { + "status": "FINISHED", + "message": "Scheduling job finished.", + "result": { + "unresolved": [ + { + "asset": 42, + "soc-minima": [ + { + "datetime": "2024-01-15T10:30:00+00:00", + "violation": "20.0 kWh" + }, + { + "datetime": "2024-01-15T10:45:00+00:00", + "violation": "15.0 kWh" + } + ] + } + ], + "resolved": [ + { + "asset": 42, + "soc-maxima": [ + { + "datetime": "2024-01-15T11:00:00+00:00", + "margin": "40.0 kWh" + }, + { + "datetime": "2024-01-15T12:00:00+00:00", + "margin": "35.0 kWh" + } + ] + } + ] + } + } + + +Interpreting constraint results for optimization decisions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**When constraints are all met:** + +An empty ``unresolved`` array indicates successful optimization. +However, check the margins in ``resolved`` to understand how tight the constraints were: + +- Large margins (e.g., 50 kWh) suggest the device has significant flexibility headroom. +- Small margins (e.g., 5 kWh) indicate the constraints were nearly violated. +- Zero margin would mean the device hit the exact constraint limit. + +*Use case*: If you see very small margins, you may want to relax constraints or provide additional flexibility to create a more robust schedule. + +**When constraints are unresolved:** + +Unresolved constraints indicate the optimization problem was over-constrained. Common causes: + +- Conflicting constraints, such as a high minimum on too short notice. +- Insufficient headroom within the grid capacity, caused by inflexible devices. + +The ``violation`` values tell you how much shortfall exists: + +- For ``soc-minima`` violations: The shortage in kWh. The device could not charge enough. +- For ``soc-maxima`` violations: The excess in kWh. The device could not discharge enough. + +*Use case*: If a battery is reporting 20 kWh shortage for a planned trip, you may need to: + +- Allow more time for charging. +- Install a larger battery. +- Reduce the minimum SoC requirement. +- Stretch the minimum SoC requirement over a longer time period (using the ``duration`` field) to continue charging in case the user plugs out later than expected. +- Warn the user about the shortfall. + +**When no constraints are defined:** + +If ``unresolved`` and ``resolved`` are both empty, no state-of-charge constraints were set. + +.. note:: Hard constraints (``soc-targets``) are never reported in results because the scheduler enforces them strictly by definition. + If a hard constraint cannot be met, the entire scheduling job will fail, not produce results with violations. We believe the two schedulers (and their flex-models) we describe here are covering a lot of use cases already. Here are some thoughts on further innovation: diff --git a/flexmeasures/api/v3_0/jobs.py b/flexmeasures/api/v3_0/jobs.py index 261032c295..fb7c41d6d8 100644 --- a/flexmeasures/api/v3_0/jobs.py +++ b/flexmeasures/api/v3_0/jobs.py @@ -84,6 +84,20 @@ def get_job_status(self, job_id: str, **kwargs): Failed jobs also include traceback information when the worker stored it with the job result. + + For a finished scheduling job, ``result`` is an object. For a + ``StorageScheduler`` job it holds soft state-of-charge constraint + analysis: ``unresolved`` lists constraints the scheduler could not + satisfy, and ``resolved`` lists constraints that were satisfied + with some margin. Each device entry's ``soc-minima``/``soc-maxima`` + value is a list, holding one entry per violated slot (for + ``unresolved``) or per met slot with its margin (for ``resolved``), + ordered chronologically. Both arrays are empty when the flex model + defines no ``soc-minima``/``soc-maxima``, or when a scheduler other + than ``StorageScheduler`` was used. This is the only place + constraint analysis is available — the sensor schedule endpoint + (``GET /api/v3_0/sensors//schedules/``) returns power + values only. security: - ApiKeyAuth: [] parameters: @@ -118,7 +132,14 @@ def get_job_status(self, job_id: str, **kwargs): type: string description: Human-readable description of the job status. result: - description: Return value of the job function, or null when not yet available. + description: > + Return value of the job function, or null when not yet + available. For a finished scheduling job, this is an + object; a ``StorageScheduler`` job populates it with + ``unresolved``/``resolved`` soft state-of-charge + constraint analysis (empty arrays when the flex model + defines no ``soc-minima``/``soc-maxima``, or when a + scheduler other than ``StorageScheduler`` was used). nullable: true func_name: type: string @@ -163,7 +184,15 @@ def get_job_status(self, job_id: str, **kwargs): value: status: FINISHED message: "Scheduling job has finished." - result: null + result: + unresolved: + - asset: 42 + soc-minima: + - datetime: "2024-01-01T10:00:00+00:00" + violation: "260.0 kWh" + - datetime: "2024-01-01T10:15:00+00:00" + violation: "180.0 kWh" + resolved: [] func_name: "flexmeasures.data.services.scheduling.create_schedule" origin: scheduling enqueued_at: "2026-04-28T10:00:00+00:00" @@ -227,17 +256,16 @@ def get_job_status(self, job_id: str, **kwargs): except Exception: # noqa: BLE001 result = None - return ( - dict( - status=status_name, - message=job_status_description(job), - result=result, - func_name=job.func_name, - origin=job.origin, - enqueued_at=_isoformat_or_none(job.enqueued_at), - started_at=_isoformat_or_none(job.started_at), - ended_at=_isoformat_or_none(job.ended_at), - exc_info=failed_job_exc_info(job), - ), - 200, + response = dict( + status=status_name, + message=job_status_description(job), + result=result, + func_name=job.func_name, + origin=job.origin, + enqueued_at=_isoformat_or_none(job.enqueued_at), + started_at=_isoformat_or_none(job.started_at), + ended_at=_isoformat_or_none(job.ended_at), + exc_info=failed_job_exc_info(job), ) + + return response, 200 diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 1c5e53d653..e28a7f0c4a 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1080,6 +1080,11 @@ def get_schedule( # noqa: C901 as database values and what is seen in UI charts. The values will indicate exactly what is stored, which is itself determined by the sensor's ``consumption_is_positive`` attribute (if set) or by the scheduler's default storage convention (production positive in the database). + + **Constraint analysis** + + For detailed constraint analysis (unmet and resolved constraints), use the + [GET /api/v3_0/jobs/](#/Jobs/get_api_v3_0_jobs__uuid_) endpoint. security: - ApiKeyAuth: [] parameters: @@ -1318,7 +1323,15 @@ def get_schedule( # noqa: C901 ) d, s = request_processed(scheduler_info_msg) - return dict(scheduler_info=scheduler_info, **response, **d), s + response_body = dict( + scheduler_info=scheduler_info, + **response, + **d, + ) + return ( + response_body, + s, + ) @route("/", methods=["GET"]) @use_kwargs({"sensor": SensorIdField(data_key="id")}, location="path") diff --git a/flexmeasures/api/v3_0/tests/test_jobs_api.py b/flexmeasures/api/v3_0/tests/test_jobs_api.py index 28454db63f..b9f43d73ca 100644 --- a/flexmeasures/api/v3_0/tests/test_jobs_api.py +++ b/flexmeasures/api/v3_0/tests/test_jobs_api.py @@ -230,11 +230,88 @@ def test_get_job_status_finished( assert data["enqueued_at"] is not None assert data["started_at"] is not None assert data["ended_at"] is not None - # scheduling jobs return True on success; result must be present in the response - assert data["result"] is not None + # every finished scheduling job now returns an object (not the boolean + # True it used to return unconditionally); this is a StorageScheduler + # job, so `result` is the soft SoC constraint analysis dict, and since + # this flex model defines no soc-minima/soc-maxima, both arrays are + # simply empty here + assert data["result"] == {"unresolved": [], "resolved": []} assert data["exc_info"] is None +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_get_job_status_finished_with_unresolved_soc_minima( + app, + add_market_prices, + add_battery_assets, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, + requesting_user, + db, +): + """A finished StorageScheduler job whose flex model defines an unreachable + soc-minima (as a soft constraint, via a breach price) should surface that + violation directly under `result`, replacing the boolean `True` that a + finished scheduling job would otherwise return. The old separate + `scheduling_result` field must no longer be present. + """ + sensor = add_battery_assets["Test battery"].sensors[0] + message = message_for_trigger_schedule() + # soc-max is 40 kWh, so a soc-minima target beyond that is unreachable + # regardless of power capacity. + message["flex-model"]["soc-minima"] = [ + {"value": 1000, "datetime": "2015-01-01T12:00:00+01:00"} + ] + message["flex-context"] = { + "soc-minima-breach-price": "1 EUR/kWh", # makes it a soft constraint + } + + with app.test_client() as client: + trigger_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + assert trigger_response.status_code == 200 + job_id = trigger_response.json["schedule"] + + # run the scheduling job + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) + assert job.is_finished + + # query the generic job endpoint + response = client.get( + url_for("JobAPI:get_job_status", uuid=job_id), + ) + + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + data = response.json + assert data["status"] == "FINISHED" + + result = data["result"] + assert isinstance(result, dict) + assert set(result.keys()) == {"unresolved", "resolved"} + + asset_id = add_battery_assets["Test battery"].id + unresolved_entry = next( + (e for e in result["unresolved"] if e["asset"] == asset_id), None + ) + assert unresolved_entry is not None, "Expected an unresolved soc-minima entry" + assert "soc-minima" in unresolved_entry + assert isinstance(unresolved_entry["soc-minima"], list) + assert len(unresolved_entry["soc-minima"]) >= 1 + for violation in unresolved_entry["soc-minima"]: + assert "datetime" in violation + assert "violation" in violation + + # the old separate field must really be gone + assert "scheduling_result" not in data + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 18596d64aa..069f0c9e81 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -246,7 +246,8 @@ def test_upload_sensor_data_with_unit_conversion_success( len(beliefs) == expected_num_beliefs ), f"Fetched {len(beliefs)} beliefs from the database, expecting {expected_num_beliefs}." - assert [b.event_value for b in beliefs] == expected_event_values + # approximate equality: unit conversion can differ slightly by pint/numpy version + assert [b.event_value for b in beliefs] == pytest.approx(expected_event_values) @pytest.mark.parametrize( diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index f908d9c775..0ff8ef6c11 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1348,12 +1348,12 @@ def add_schedule( # noqa C901 **MsgStyle.SUCCESS, ) else: - success = make_schedule( + make_schedule( asset_or_sensor=get_asset_or_sensor_ref(asset_or_sensor), dry_run=dry_run, **scheduling_kwargs, ) - if success and not dry_run: + if not dry_run: click.secho("New schedule is stored.", **MsgStyle.SUCCESS) diff --git a/flexmeasures/cli/tests/test_data_add_fresh_db.py b/flexmeasures/cli/tests/test_data_add_fresh_db.py index 079b5ca66f..20354a3246 100644 --- a/flexmeasures/cli/tests/test_data_add_fresh_db.py +++ b/flexmeasures/cli/tests/test_data_add_fresh_db.py @@ -301,6 +301,10 @@ def test_add_process( # call command result = runner.invoke(add_schedule, cli_input) check_command_ran_without_error(result) + # ProcessScheduler's make_schedule() call returns an empty dict (not the + # boolean True), which used to be falsy enough to silently suppress this + # message; confirm the message still appears. + assert "New schedule is stored." in result.output process_power_sensor = db.session.get(Sensor, process_power_sensor_id) schedule = process_power_sensor.search_beliefs() diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4c59a0ffa9..7a1cbdd804 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -36,6 +36,7 @@ MultiSensorFlexModelSchema, ) from flexmeasures.data.schemas.sensors import SensorReference, VariableQuantityField +from flexmeasures.data.services.scheduling_result import SchedulingJobResult from flexmeasures.utils.calculations import ( integrate_time_series, ) @@ -46,6 +47,10 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] +#: Key used to store and retrieve the ``SchedulingJobResult`` in RQ job metadata +#: and in the multi-result list returned by ``StorageScheduler.compute()``. +SCHEDULING_RESULT_KEY = "scheduling_result" + class MetaStorageScheduler(Scheduler): """This class defines the constraints of a schedule for a storage device from the @@ -1664,58 +1669,260 @@ def _build_soc_schedule( soc_at_start: list[float], device_constraints: list, resolution: timedelta, - ) -> dict: + ) -> tuple[dict, dict]: """Build the state-of-charge schedule for each device that has a state-of-charge sensor. + Also computes the MWh SoC for devices that have ``soc-minima`` or ``soc-maxima`` constraints + (even without a state-of-charge sensor) so that unresolved targets can be checked later. + Converts the integrated power schedule from MWh to the sensor's unit. For sensors with a '%' unit, the soc-max flex-model field is used as capacity. If soc-max is missing or zero for a '%' sensor, the schedule is skipped with a warning. Note: soc-max is a QuantityField (not a VariableQuantityField), so it is always a float - after deserialization and cannot be a sensor reference. The isinstance guard below is - therefore a defensive check for forward-compatibility. + after deserialization and cannot be a sensor reference. + The isinstance guard below is therefore a defensive check for forward-compatibility. + + :returns: Tuple of (soc_schedule keyed by SoC sensor in sensor unit, + soc_schedule_mwh keyed by device index in MWh). """ soc_schedule = {} + soc_schedule_mwh = {} for d, flex_model_d in enumerate(flex_model): state_of_charge_sensor = flex_model_d.get("state_of_charge", None) if isinstance(state_of_charge_sensor, SensorReference): state_of_charge_sensor = state_of_charge_sensor.sensor - if not isinstance(state_of_charge_sensor, Sensor): + has_soc_sensor = isinstance(state_of_charge_sensor, Sensor) + has_soc_minima_maxima = ( + flex_model_d.get("soc_minima") is not None + or flex_model_d.get("soc_maxima") is not None + ) + # Skip devices that neither have a SoC sensor nor soc-minima/soc-maxima constraints + if not has_soc_sensor and not has_soc_minima_maxima: continue - soc_unit = state_of_charge_sensor.unit - capacity = None - if soc_unit == "%": - soc_max = flex_model_d.get("soc_max") - if isinstance(soc_max, (Sensor, SensorReference)): - raise ValueError( - f"Cannot convert state-of-charge schedule to '%' unit for sensor {state_of_charge_sensor.id}: " - "soc-max as a sensor reference is not supported for '%' unit conversion. " - "Skipping state-of-charge schedule." - ) - if not soc_max: - raise ValueError( - f"Cannot convert state-of-charge schedule to '%' unit for sensor {state_of_charge_sensor.id}: " - "soc-max is missing or zero. Skipping state-of-charge schedule." - ) - capacity = f"{soc_max} MWh" # all flex model fields are in MWh by now - soc_schedule[state_of_charge_sensor] = convert_units( - integrate_time_series( - series=ems_schedule[d], - initial_stock=soc_at_start[d], - stock_delta=device_constraints[d]["stock delta"] - * resolution - / timedelta(hours=1), - up_efficiency=device_constraints[d]["derivative up efficiency"], - down_efficiency=device_constraints[d]["derivative down efficiency"], - storage_efficiency=device_constraints[d]["efficiency"] - .astype(float) - .fillna(1), - ), - from_unit="MWh", - to_unit=soc_unit, - capacity=capacity, + # Skip devices without a known initial SoC (required for integration) + if soc_at_start[d] is None: + continue + + soc_mwh = integrate_time_series( + series=ems_schedule[d], + initial_stock=soc_at_start[d], + stock_delta=device_constraints[d]["stock delta"] + * resolution + / timedelta(hours=1), + up_efficiency=device_constraints[d]["derivative up efficiency"], + down_efficiency=device_constraints[d]["derivative down efficiency"], + storage_efficiency=device_constraints[d]["efficiency"] + .astype(float) + .fillna(1), ) - return soc_schedule + soc_schedule_mwh[d] = soc_mwh + + if has_soc_sensor: + soc_unit = state_of_charge_sensor.unit + capacity = None + if soc_unit == "%": + soc_max = flex_model_d.get("soc_max") + if isinstance(soc_max, (Sensor, SensorReference)): + raise ValueError( + f"Cannot convert state-of-charge schedule to '%' unit for sensor {state_of_charge_sensor.id}: " + "soc-max as a sensor reference is not supported for '%' unit conversion. " + "Skipping state-of-charge schedule." + ) + if not soc_max: + raise ValueError( + f"Cannot convert state-of-charge schedule to '%' unit for sensor {state_of_charge_sensor.id}: " + "soc-max is missing or zero. Skipping state-of-charge schedule." + ) + capacity = ( + f"{soc_max} MWh" # all flex model fields are in MWh by now + ) + soc_schedule[state_of_charge_sensor] = convert_units( + soc_mwh, + from_unit="MWh", + to_unit=soc_unit, + capacity=capacity, + ) + return soc_schedule, soc_schedule_mwh + + def _compute_unresolved_targets( + self, + flex_model: list[dict], + soc_schedule_mwh: dict, + start: datetime, + end: datetime, + resolution: timedelta, + most_relevant_only: bool = False, + ) -> tuple[list, list]: + """Compute unmet and met SoC minima/maxima targets per device. + + For each device that has ``soc-minima`` or ``soc-maxima`` constraints in the flex model, + compares the computed MWh SoC schedule against those constraints. + Devices without a ``state_of_charge`` Sensor are included + as long as a device key can be determined from the power sensor. + + The result includes asset ID for each constraint. + Devices for which an asset ID cannot be determined are skipped. + + Constraints are evaluated over the window ``(start + resolution, end)`` + (i.e. the first scheduled slot through the end of the schedule). + The ``start`` slot itself is the initial condition (``soc_at_start``), + not a scheduled value, so it is excluded. + + Note: ``soc-targets`` are modelled as hard constraints and are not checked here, + as by definition the scheduler will not allow any deviation from them. + + :param flex_model: The deserialized flex model (list of per-device dicts). + :param soc_schedule_mwh: MWh SoC schedule keyed by device index ``d``. + :param start: Start of the schedule. + :param end: End of the schedule. + :param resolution: Schedule resolution. + :param most_relevant_only: If False (the default), report every violated/met slot. + If True, report only the single most relevant slot + (the first violation, or the tightest margin). + Either way, the result holds a list. + :returns: A tuple ``(unresolved, resolved)``. + + ``unresolved`` is a list of dicts, each with ``"asset"`` field and constraint info. + Each constraint entry is a list of dicts + ``{"datetime": , "violation": " kWh"}`` + (one per violated slot, or just the first if ``most_relevant_only`` is True), + where ``violation`` is always positive. + + ``resolved`` is also a list of dicts with ``"asset"`` field and constraint info. + Each constraint entry is a list of dicts + ``{"datetime": , "margin": " kWh"}`` + (one per met slot, or just the slot with the tightest/smallest positive + margin if ``most_relevant_only`` is True). + """ + # Use the configured rounding precision, or the scheduler's default of 6. + precision = self.round_to_decimals if self.round_to_decimals is not None else 6 + + unresolved: list = [] + resolved: list = [] + + for d, flex_model_d in enumerate(flex_model): + soc_mwh = soc_schedule_mwh.get(d) + if soc_mwh is None: + continue + + # Determine device key: prefer asset ID, fall back to power sensor ID. + # Devices without a state-of-charge sensor are included as long as a + # key can be derived from the power sensor's generic asset (or the + # power sensor itself). + power_sensor = flex_model_d.get("sensor") + if ( + power_sensor is not None + and hasattr(power_sensor, "generic_asset") + and power_sensor.generic_asset is not None + ): + asset_id = power_sensor.generic_asset.id + else: + continue + + device_violations: dict = {} + device_resolved: dict = {} + + # Check soc_minima (first time slot where scheduled SoC < minima) + soc_minima_d = flex_model_d.get("soc_minima") + if soc_minima_d is not None: + soc_minima_series = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_minima_d, + unit="MWh", + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=self.belief_time, + as_instantaneous_events=True, + resolve_overlaps="max", + ) + defined_minima = soc_minima_series.dropna() + if len(defined_minima) > 0: + aligned_soc = soc_mwh.reindex(defined_minima.index) + shortages = defined_minima - aligned_soc + violations = shortages[shortages > 0] + if not violations.empty: + violation_times = ( + [violations.index[0]] + if most_relevant_only + else violations.index + ) + device_violations["soc-minima"] = [ + { + "datetime": t.tz_convert("UTC").isoformat(), + "violation": f"{round(float(violations[t]) * 1000, precision)} kWh", + } + for t in violation_times + ] + else: + # All minima met — margins are the headroom above the minimum. + # violations.empty guarantees shortages <= 0, so margins (soc - minima) >= 0. + margins = aligned_soc - defined_minima + margin_times = ( + [margins.idxmin()] if most_relevant_only else margins.index + ) + device_resolved["soc-minima"] = [ + { + "datetime": t.tz_convert("UTC").isoformat(), + "margin": f"{round(float(margins[t]) * 1000, precision)} kWh", + } + for t in margin_times + ] + + # Check soc_maxima (first time slot where scheduled SoC > maxima) + soc_maxima_d = flex_model_d.get("soc_maxima") + if soc_maxima_d is not None: + soc_maxima_series = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_maxima_d, + unit="MWh", + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=self.belief_time, + as_instantaneous_events=True, + resolve_overlaps="min", + ) + defined_maxima = soc_maxima_series.dropna() + if len(defined_maxima) > 0: + aligned_soc = soc_mwh.reindex(defined_maxima.index) + excesses = aligned_soc - defined_maxima + violations = excesses[excesses > 0] + if not violations.empty: + violation_times = ( + [violations.index[0]] + if most_relevant_only + else violations.index + ) + device_violations["soc-maxima"] = [ + { + "datetime": t.tz_convert("UTC").isoformat(), + "violation": f"{round(float(violations[t]) * 1000, precision)} kWh", + } + for t in violation_times + ] + else: + # All maxima met — margins are the headroom below the maximum. + # violations.empty guarantees excesses <= 0, so margins (maxima - soc) >= 0. + margins = defined_maxima - aligned_soc + margin_times = ( + [margins.idxmin()] if most_relevant_only else margins.index + ) + device_resolved["soc-maxima"] = [ + { + "datetime": t.tz_convert("UTC").isoformat(), + "margin": f"{round(float(margins[t]) * 1000, precision)} kWh", + } + for t in margin_times + ] + + if device_violations: + violation_entry = {"asset": asset_id} + violation_entry.update(device_violations) + unresolved.append(violation_entry) + if device_resolved: + resolved_entry = {"asset": asset_id} + resolved_entry.update(device_resolved) + resolved.append(resolved_entry) + + return unresolved, resolved @staticmethod def _build_consumption_production_schedules( @@ -1870,7 +2077,7 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: flex_model["sensor"] = sensors[0] flex_model = [flex_model] - soc_schedule = self._build_soc_schedule( + soc_schedule, soc_schedule_mwh = self._build_soc_schedule( flex_model, ems_schedule, soc_at_start, device_constraints, resolution ) @@ -1911,8 +2118,17 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: ) for sensor in consumption_production_schedule.keys() } + # Round the MWh SoC schedule to the same precision so that violation + # detection does not flag floating-point epsilon differences. + soc_schedule_mwh = { + d: series.round(self.round_to_decimals) + for d, series in soc_schedule_mwh.items() + } if self.return_multiple: + unresolved, resolved = self._compute_unresolved_targets( + flex_model, soc_schedule_mwh, start, end, resolution + ) storage_schedules = [ { "name": "storage_schedule", @@ -1964,11 +2180,21 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: } for sensor, data in consumption_production_schedule.items() ] + scheduling_result = [ + { + "name": SCHEDULING_RESULT_KEY, + "data": SchedulingJobResult( + unresolved=unresolved, + resolved=resolved, + ), + } + ] return ( storage_schedules + commitment_costs + soc_schedules + consumption_production_schedules + + scheduling_result ) else: return storage_schedule[sensors[0]] @@ -1977,8 +2203,8 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: def create_constraint_violations_message(constraint_violations: list) -> str: """Create a human-readable message with the constraint_violations. - :param constraint_violations: list with the constraint violations - :return: human-readable message + :param constraint_violations: List with the constraint violations. + :returns: Human-readable message. """ message = "" @@ -2336,7 +2562,7 @@ def get_pattern_match_word(word: str) -> str: - word boundary - arithmetic operations - :return: regex expression + :returns: regex expression """ regex = r"(^|\s|$|\b|\+|\-|\*|\/\|\\)" @@ -2347,9 +2573,9 @@ def get_pattern_match_word(word: str) -> str: def sanitize_expression(expression: str, columns: list) -> tuple[str, list]: """Wrap column in commas to accept arbitrary column names (e.g. with spaces). - :param expression: expression to sanitize - :param columns: list with the name of the columns of the input data for the expression. - :return: sanitized expression and columns (variables) used in the expression + :param expression: Expression to sanitize. + :param columns: List with the name of the columns of the input data for the expression. + :returns: Sanitized expression and columns (variables) used in the expression. """ _expression = copy.copy(expression) @@ -2382,7 +2608,7 @@ def validate_constraint( No need to use the syntax `column` to reference column, just use the column name. :param round_to_decimals: Number of decimals to round off to before validating constraints. - :return: List of constraint violations, specifying their time, constraint and violation. + :returns: List of constraint violations, specifying their time, constraint and violation. """ constraint_expression = f"{lhs_expression} {inequality} {rhs_expression}" diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index ed8c09be82..58001b898d 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta +from unittest import mock import pytz import pytest @@ -15,6 +16,7 @@ get_sensors_from_db, series_to_ts_specs, ) +from flexmeasures.data.services.scheduling_result import SchedulingJobResult def test_battery_solver_multi_commitment(add_battery_assets, db): @@ -290,6 +292,644 @@ def test_battery_relaxation(add_battery_assets, db): ) # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours +def test_unresolved_targets_soc_minima(add_battery_assets, db): + """Test that unresolved soc-minima targets are reported in the scheduling result. + + A battery starts at 0.4 MWh with a very limited charging capacity (0.01 MW). + With 100% efficiency and 24 hours, it can gain at most 0.01 * 24 = 0.24 MWh, + reaching a max SoC of ~0.64 MWh. No roundtrip or storage efficiency is set, + so the default (100%) applies. + A soc-minima of 0.9 MWh is set as a soft constraint (via a breach price). + The scheduler will charge at full capacity but still fail to reach the target, + so the scheduling result should report an unresolved soc-minima. + """ + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + soc_sensor = Sensor( + name="state-of-charge-minima-test", + generic_asset=battery.generic_asset, + unit="MWh", + event_resolution=timedelta(0), + ) + db.session.add(soc_sensor) + db.session.flush() + + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1)) + end = tz.localize(datetime(2015, 1, 2)) + resolution = timedelta(minutes=15) + soc_at_start = 0.4 + index = initialize_index(start=start, end=end, resolution=resolution) + consumption_prices = pd.Series(100, index=index) + + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": f"{soc_at_start} MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.01 MVA", # very limited: max gain 0.24 MWh over 24 h + "soc-minima": [ + { + "datetime": "2015-01-02T00:00:00+01:00", + "value": "0.9 MWh", # unreachable + } + ], + "state-of-charge": {"sensor": soc_sensor.id}, + "prefer-charging-sooner": False, + }, + flex_context={ + "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "production-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "site-power-capacity": "2 MW", + "soc-minima-breach-price": "1 EUR/kWh", # soft constraint + }, + return_multiple=True, + ) + results = scheduler.compute() + + # The scheduling_result entry should be present + scheduling_result_entry = next( + (r for r in results if r.get("name") == "scheduling_result"), None + ) + assert scheduling_result_entry is not None + + scheduling_result = scheduling_result_entry["data"] + assert isinstance(scheduling_result, SchedulingJobResult) + + asset_id = battery.generic_asset.id + unresolved = scheduling_result.unresolved + entry = next((e for e in unresolved if e["asset"] == asset_id), None) + assert ( + entry is not None + ), "Expected an unresolved soc-minima since the target is unreachable" + assert "soc-minima" in entry + # Only a single soc-minima datetime was defined in the flex model, so the + # violation list holds exactly one entry. + assert len(entry["soc-minima"]) == 1 + # The scheduled SoC should be below the 0.9 MWh target (violation == 260.0 kWh shortage) + assert entry["soc-minima"][0]["violation"] == "260.0 kWh" + # The constraint is at 2015-01-02T00:00:00+01:00 = 2015-01-01T23:00:00+00:00 (UTC) + assert entry["soc-minima"][0]["datetime"] == "2015-01-01T23:00:00+00:00" + + # No soc-maxima was set, so it should not appear + assert "soc-maxima" not in entry + + # No soc-maxima constraint defined, so resolved should be empty + assert scheduling_result.resolved == [] + + +def test_unresolved_targets_none_when_met(add_battery_assets, db): + """Test that no unresolved targets are reported when constraints are fully met. + + A battery starts at 0.4 MWh and has a soc-minima of 0.5 MWh at end of schedule. + With enough capacity, the scheduler can easily charge to 0.5 MWh, so the + scheduling result should have no unresolved soc-minima. + """ + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + soc_sensor = Sensor( + name="state-of-charge-none-when-met-test", + generic_asset=battery.generic_asset, + unit="MWh", + event_resolution=timedelta(0), + ) + db.session.add(soc_sensor) + db.session.flush() + + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1)) + end = tz.localize(datetime(2015, 1, 2)) + resolution = timedelta(minutes=15) + soc_at_start = 0.4 + index = initialize_index(start=start, end=end, resolution=resolution) + consumption_prices = pd.Series(100, index=index) + + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": f"{soc_at_start} MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "2 MVA", # plenty of capacity to reach 0.5 MWh + "soc-minima": [ + { + "datetime": "2015-01-02T00:00:00+01:00", + "value": "0.5 MWh", # easily reachable + } + ], + "state-of-charge": {"sensor": soc_sensor.id}, + "prefer-charging-sooner": False, + }, + flex_context={ + "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "production-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "site-power-capacity": "2 MW", + "soc-minima-breach-price": "1 EUR/kWh", # soft constraint + }, + return_multiple=True, + ) + results = scheduler.compute() + + scheduling_result_entry = next( + (r for r in results if r.get("name") == "scheduling_result"), None + ) + assert scheduling_result_entry is not None + scheduling_result = scheduling_result_entry["data"] + asset_id = battery.generic_asset.id + unresolved = scheduling_result.unresolved + # The minima target is met, so no unresolved targets expected + assert unresolved == [] + + # The soc-minima was met, so resolved should report it + entry = next( + (e for e in scheduling_result.resolved if e["asset"] == asset_id), None + ) + assert entry is not None + assert "soc-minima" in entry + # Only a single soc-minima datetime was defined in the flex model, so the + # margin list holds exactly one entry. + assert len(entry["soc-minima"]) == 1 + margin_str = entry["soc-minima"][0]["margin"] + # Margin should be a non-negative kWh string + assert margin_str.endswith(" kWh") + assert float(margin_str.replace(" kWh", "")) >= 0 + + +def test_unresolved_targets_soc_maxima(add_battery_assets, db): + """Test that unresolved soc-maxima targets are reported in the scheduling result. + + A battery starts at 0.9 MWh with a very limited discharge capacity (0.01 MW). + With 100% efficiency and 24 hours, it can discharge at most 0.01 * 24 = 0.24 MWh, + reaching a min SoC of ~0.66 MWh. No roundtrip or storage efficiency is set, + so the default (100%) applies. + A soc-maxima of 0.5 MWh is set as a soft constraint (via a breach price). + The scheduler will discharge at full capacity but still remain above the target, + so the scheduling result should report an unresolved soc-maxima. + """ + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + soc_sensor = Sensor( + name="state-of-charge-maxima-test", + generic_asset=battery.generic_asset, + unit="MWh", + event_resolution=timedelta(0), + ) + db.session.add(soc_sensor) + db.session.flush() + + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1)) + end = tz.localize(datetime(2015, 1, 2)) + resolution = timedelta(minutes=15) + soc_at_start = 0.9 + index = initialize_index(start=start, end=end, resolution=resolution) + consumption_prices = pd.Series(100, index=index) + + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": f"{soc_at_start} MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.01 MVA", # very limited: max discharge 0.24 MWh over 24 h + "soc-maxima": [ + { + "datetime": "2015-01-02T00:00:00+01:00", + "value": "0.5 MWh", # unreachably low + } + ], + "state-of-charge": {"sensor": soc_sensor.id}, + "prefer-charging-sooner": False, + }, + flex_context={ + "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "production-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "site-power-capacity": "2 MW", + "soc-maxima-breach-price": "1 EUR/kWh", # soft constraint + }, + return_multiple=True, + ) + results = scheduler.compute() + + scheduling_result_entry = next( + (r for r in results if r.get("name") == "scheduling_result"), None + ) + assert scheduling_result_entry is not None + + asset_id = battery.generic_asset.id + unresolved = scheduling_result_entry["data"].unresolved + entry = next((e for e in unresolved if e["asset"] == asset_id), None) + assert ( + entry is not None + ), "Expected an unresolved soc-maxima since the target is unreachable" + assert "soc-maxima" in entry + # Only a single soc-maxima datetime was defined in the flex model, so the + # violation list holds exactly one entry. + assert len(entry["soc-maxima"]) == 1 + # The scheduled SoC should be above the 0.5 MWh target (violation == 160.0 kWh excess) + assert entry["soc-maxima"][0]["violation"] == "160.0 kWh" + # The constraint is at 2015-01-02T00:00:00+01:00 = 2015-01-01T23:00:00+00:00 (UTC) + assert entry["soc-maxima"][0]["datetime"] == "2015-01-01T23:00:00+00:00" + + # No soc-minima was set, so it should not appear + assert "soc-minima" not in entry + + # No soc-minima constraint defined, so resolved should be empty + assert scheduling_result_entry["data"].resolved == [] + + +def test_unresolved_targets_no_soc_sensor(add_battery_assets, db): + """Regression: unresolved/resolved reporting works without a state_of_charge sensor. + + A battery has ``soc-minima`` constraints but no ``state-of-charge`` sensor + configured in the flex model. The production code must still produce + unresolved/resolved entries keyed by the asset ID (not the SoC sensor ID). + """ + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1)) + end = tz.localize(datetime(2015, 1, 2)) + resolution = timedelta(minutes=15) + soc_at_start = 0.4 + index = initialize_index(start=start, end=end, resolution=resolution) + consumption_prices = pd.Series(100, index=index) + + # No "state-of-charge" key in flex_model — intentionally omitted. + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": f"{soc_at_start} MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.01 MVA", # very limited: max gain 0.24 MWh over 24 h + "soc-minima": [ + { + "datetime": "2015-01-02T00:00:00+01:00", + "value": "0.9 MWh", # unreachable given the limited capacity + } + ], + "prefer-charging-sooner": False, + }, + flex_context={ + "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "production-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "site-power-capacity": "2 MW", + "soc-minima-breach-price": "1 EUR/kWh", # soft constraint + }, + return_multiple=True, + ) + results = scheduler.compute() + + scheduling_result_entry = next( + (r for r in results if r.get("name") == "scheduling_result"), None + ) + assert scheduling_result_entry is not None, "scheduling_result entry missing" + + scheduling_result = scheduling_result_entry["data"] + assert isinstance(scheduling_result, SchedulingJobResult) + + # Result must be keyed by the asset ID, not by a SoC sensor ID. + asset_id = battery.generic_asset.id + + unresolved = scheduling_result.unresolved + entry = next((e for e in unresolved if e["asset"] == asset_id), None) + assert entry is not None, ( + f"Expected an unresolved entry for asset ID {asset_id!r}; " + f"got: {unresolved!r}" + ) + assert "soc-minima" in entry + # Only a single soc-minima datetime was defined in the flex model, so the + # violation list holds exactly one entry. + assert len(entry["soc-minima"]) == 1 + assert entry["soc-minima"][0]["violation"] == "260.0 kWh" + assert entry["soc-minima"][0]["datetime"] == "2015-01-01T23:00:00+00:00" + + # No soc-maxima constraint was set. + assert "soc-maxima" not in entry + + # No soc-maxima constraint defined, so resolved should be empty. + assert scheduling_result.resolved == [] + + +def test_unresolved_targets_most_relevant_only_flag_soc_minima_violations( + add_battery_assets, db +): + """Test the ``most_relevant_only`` flag of ``_compute_unresolved_targets`` for unresolved soc-minima. + + A battery starts at 0.4 MWh with a very limited charging capacity (0.01 MW), + so it can gain at most 0.01 * 24 = 0.24 MWh over the 24-hour schedule, + reaching a max SoC of ~0.64 MWh. Three soc-minima checkpoints of 0.9 MWh + are set at different times, all of which are unreachable given this + physical limit, regardless of how the scheduler distributes charging. + + With the default ``most_relevant_only=False``, ``_compute_unresolved_targets`` + reports every violated slot (all three checkpoints, chronologically ordered). + With ``most_relevant_only=True``, it reports only the first (earliest) violation. + """ + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + soc_sensor = Sensor( + name="state-of-charge-all-flag-minima-test", + generic_asset=battery.generic_asset, + unit="MWh", + event_resolution=timedelta(0), + ) + db.session.add(soc_sensor) + db.session.flush() + + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1)) + end = tz.localize(datetime(2015, 1, 2)) + resolution = timedelta(minutes=15) + soc_at_start = 0.4 + index = initialize_index(start=start, end=end, resolution=resolution) + consumption_prices = pd.Series(100, index=index) + + # All three checkpoints require 0.9 MWh, which is unreachable at any point + # in the schedule given the 0.01 MW charging limit (max reachable ~0.64 MWh). + violation_datetimes = [ + "2015-01-01T06:00:00+01:00", + "2015-01-01T12:00:00+01:00", + "2015-01-02T00:00:00+01:00", + ] + + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": f"{soc_at_start} MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.01 MVA", + "soc-minima": [ + {"datetime": dt, "value": "0.9 MWh"} for dt in violation_datetimes + ], + "state-of-charge": {"sensor": soc_sensor.id}, + "prefer-charging-sooner": False, + }, + flex_context={ + "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "production-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "site-power-capacity": "2 MW", + "soc-minima-breach-price": "1 EUR/kWh", # soft constraint + }, + return_multiple=True, + ) + + # Intercept the private helper to also capture what it would return with + # most_relevant_only=True, without affecting the (default + # most_relevant_only=False) result used by compute(). + captured: dict = {} + original = StorageScheduler._compute_unresolved_targets + + def spy( + self, + flex_model, + soc_schedule_mwh, + start, + end, + resolution, + most_relevant_only=False, + ): + captured["most_relevant_only"] = original( + self, + flex_model, + soc_schedule_mwh, + start, + end, + resolution, + most_relevant_only=True, + ) + return original( + self, + flex_model, + soc_schedule_mwh, + start, + end, + resolution, + most_relevant_only=most_relevant_only, + ) + + with mock.patch.object(StorageScheduler, "_compute_unresolved_targets", spy): + results = scheduler.compute() + + scheduling_result_entry = next( + (r for r in results if r.get("name") == "scheduling_result"), None + ) + assert scheduling_result_entry is not None + scheduling_result = scheduling_result_entry["data"] + asset_id = battery.generic_asset.id + + # --- most_relevant_only=False (the default used by compute()) --- + entry = next( + (e for e in scheduling_result.unresolved if e["asset"] == asset_id), None + ) + assert entry is not None + assert "soc-minima" in entry + violations = entry["soc-minima"] + assert len(violations) == len(violation_datetimes), ( + f"Expected all {len(violation_datetimes)} violated slots to be reported, " + f"got: {violations!r}" + ) + expected_utc_datetimes = [ + pd.Timestamp(dt).tz_convert("UTC").isoformat() for dt in violation_datetimes + ] + # Entries must be chronologically ordered and match the checkpoint datetimes. + assert [v["datetime"] for v in violations] == expected_utc_datetimes + for v in violations: + assert v["violation"].endswith(" kWh") + assert float(v["violation"].replace(" kWh", "")) > 0 + + # --- most_relevant_only=True --- + unresolved_most_relevant_only, _resolved_most_relevant_only = captured[ + "most_relevant_only" + ] + entry_most_relevant_only = next( + (e for e in unresolved_most_relevant_only if e["asset"] == asset_id), None + ) + assert entry_most_relevant_only is not None + # Only the first (earliest) violation should be reported. + assert len(entry_most_relevant_only["soc-minima"]) == 1 + assert entry_most_relevant_only["soc-minima"][0] == violations[0] + + +def test_unresolved_targets_most_relevant_only_flag_soc_minima_resolved_margins( + add_battery_assets, db +): + """Test the ``most_relevant_only`` flag of ``_compute_unresolved_targets`` for resolved (met) soc-minima. + + A battery starts at 0.4 MWh with plenty of charging capacity, a positive + consumption price, and a negative production price (so that neither charging + nor discharging is ever done without reason). Two soc-minima checkpoints are + set: an earlier, tighter one (0.5 MWh) and a later, much slacker one (0.1 MWh). + Both are met, but with different margins: the battery charges up to 0.5 MWh as + soon as possible (``prefer-charging-sooner`` defaults to True) to satisfy the + tighter, earlier checkpoint, and then has no incentive to move further, so it + stays there — leaving zero margin at the tighter checkpoint and a much larger + margin at the slacker, later checkpoint. + + With the default ``most_relevant_only=False``, both met slots are reported + (chronologically ordered). With ``most_relevant_only=True``, only the slot + with the tightest (smallest) margin is reported. + """ + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + soc_sensor = Sensor( + name="state-of-charge-all-flag-margins-test", + generic_asset=battery.generic_asset, + unit="MWh", + event_resolution=timedelta(0), + ) + db.session.add(soc_sensor) + db.session.flush() + + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1)) + end = tz.localize(datetime(2015, 1, 2)) + resolution = timedelta(minutes=15) + soc_at_start = 0.4 + index = initialize_index(start=start, end=end, resolution=resolution) + consumption_prices = pd.Series(100, index=index) + # A (small) negative production price means discharging (selling energy) + # incurs a cost rather than earning revenue, so the battery has no incentive + # to move away from a checkpoint once it has been satisfied. + production_prices = pd.Series(-1, index=index) + + tight_datetime = "2015-01-01T06:00:00+01:00" + slack_datetime = "2015-01-02T00:00:00+01:00" + + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": f"{soc_at_start} MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "2 MVA", + "soc-minima": [ + {"datetime": tight_datetime, "value": "0.5 MWh"}, + {"datetime": slack_datetime, "value": "0.1 MWh"}, + ], + "state-of-charge": {"sensor": soc_sensor.id}, + }, + flex_context={ + "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"), + "production-price": series_to_ts_specs(production_prices, unit="EUR/MWh"), + "site-power-capacity": "2 MW", + "soc-minima-breach-price": "1 EUR/kWh", # soft constraint + }, + return_multiple=True, + ) + + # Intercept the private helper to also capture what it would return with + # most_relevant_only=True, without affecting the (default + # most_relevant_only=False) result used by compute(). + captured: dict = {} + original = StorageScheduler._compute_unresolved_targets + + def spy( + self, + flex_model, + soc_schedule_mwh, + start, + end, + resolution, + most_relevant_only=False, + ): + captured["most_relevant_only"] = original( + self, + flex_model, + soc_schedule_mwh, + start, + end, + resolution, + most_relevant_only=True, + ) + return original( + self, + flex_model, + soc_schedule_mwh, + start, + end, + resolution, + most_relevant_only=most_relevant_only, + ) + + with mock.patch.object(StorageScheduler, "_compute_unresolved_targets", spy): + results = scheduler.compute() + + scheduling_result_entry = next( + (r for r in results if r.get("name") == "scheduling_result"), None + ) + assert scheduling_result_entry is not None + scheduling_result = scheduling_result_entry["data"] + asset_id = battery.generic_asset.id + + # No violations expected: both checkpoints are met. + assert scheduling_result.unresolved == [] + + # --- most_relevant_only=False (the default used by compute()) --- + entry = next( + (e for e in scheduling_result.resolved if e["asset"] == asset_id), None + ) + assert entry is not None + assert "soc-minima" in entry + margins = entry["soc-minima"] + assert ( + len(margins) == 2 + ), f"Expected both met slots to be reported, got: {margins!r}" + expected_utc_datetimes = [ + pd.Timestamp(dt).tz_convert("UTC").isoformat() + for dt in (tight_datetime, slack_datetime) + ] + assert [m["datetime"] for m in margins] == expected_utc_datetimes + margin_values = [float(m["margin"].replace(" kWh", "")) for m in margins] + assert all(v >= 0 for v in margin_values) + # The tighter (earlier, higher-value) checkpoint should have the smaller margin. + tight_margin, slack_margin = margin_values + assert tight_margin < slack_margin + + # --- most_relevant_only=True --- + _unresolved_most_relevant_only, resolved_most_relevant_only = captured[ + "most_relevant_only" + ] + entry_most_relevant_only = next( + (e for e in resolved_most_relevant_only if e["asset"] == asset_id), None + ) + assert entry_most_relevant_only is not None + # Only the tightest (smallest) margin slot should be reported. + assert len(entry_most_relevant_only["soc-minima"]) == 1 + expected_tightest = min( + margins, key=lambda m: float(m["margin"].replace(" kWh", "")) + ) + assert entry_most_relevant_only["soc-minima"][0] == expected_tightest + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 7d3c06fe4b..13d128240e 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -20,6 +20,12 @@ def serialize_variable_quantity(value): return VariableQuantityDumpSchema().dump({"value": value})["value"] +def assert_quantity_almost_equal(actual: ur.Quantity, expected: ur.Quantity): + """Magnitude equality with a tolerance, since it can differ slightly by pint/numpy version.""" + assert actual.units == expected.units + assert actual.magnitude == pytest.approx(expected.magnitude) + + @pytest.mark.parametrize( "src_quantity, dst_unit, fails, exp_dst_quantity", [ @@ -93,10 +99,12 @@ def test_quantity_or_sensor_deserialize( try: dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): - assert dst_quantity == ur.Quantity(exp_dst_quantity) + assert_quantity_almost_equal(dst_quantity, ur.Quantity(exp_dst_quantity)) assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): - assert dst_quantity[0]["value"] == ur.Quantity(exp_dst_quantity) + assert_quantity_almost_equal( + dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) + ) assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 5c7c7886fd..e21f1b5317 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -27,7 +27,10 @@ from flexmeasures.data import db from flexmeasures.data.models.planning import Scheduler, SchedulerOutputType -from flexmeasures.data.models.planning.storage import StorageScheduler +from flexmeasures.data.models.planning.storage import ( + StorageScheduler, + SCHEDULING_RESULT_KEY, +) from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException from flexmeasures.data.models.planning.process import ProcessScheduler from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -701,9 +704,12 @@ def make_schedule( # noqa: C901 scheduler_specs: dict | None = None, dry_run: bool = False, **scheduler_kwargs: dict, -) -> bool: +) -> dict: """ - This function computes a schedule. It returns True if it ran successfully. + This function computes a schedule. It returns a dict, empty on schedulers + that don't (yet) produce further analysis. If the scheduler produced soft + state-of-charge constraint analysis (see ``SchedulingJobResult``), the dict + instead holds that analysis under ``unresolved`` and ``resolved`` keys. It can be queued as a job (see create_scheduling_job). In that case, it will probably run on a different FlexMeasures node than where the job is created. @@ -799,7 +805,11 @@ def make_schedule( # noqa: C901 rq_job.save_meta() # Save any result that specifies a sensor to save it to + scheduling_result_dict: dict = {} for result in consumption_schedule: + if result.get("name") == SCHEDULING_RESULT_KEY: + scheduling_result_dict = result["data"].to_dict() + continue if "sensor" not in result: continue @@ -842,7 +852,7 @@ def make_schedule( # noqa: C901 scheduler.persist_flex_model() db.session.commit() - return True + return scheduling_result_dict def find_scheduler_class(asset_or_sensor: Asset | Sensor) -> type: diff --git a/flexmeasures/data/services/scheduling_result.py b/flexmeasures/data/services/scheduling_result.py new file mode 100644 index 0000000000..418b499873 --- /dev/null +++ b/flexmeasures/data/services/scheduling_result.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class SchedulingJobResult: + """Constraint analysis results from a scheduling job. + + Holds soft state-of-charge constraint analysis (unmet and satisfied targets) produced by the scheduler when optimizing storage devices. + Results are available exclusively via ``GET /api/v3_0/jobs/`` in the ``result`` field. + + The sensor schedule endpoint (``GET /api/v3_0/sensors//schedules/``) returns power values only and does not include constraint analysis. + + **Structure:** + Results contain two top-level fields: + - ``unresolved``: List of soft constraints that the scheduler could not satisfy + - Each entry is a dict with ``"asset"`` field (asset ID) and constraint-type keys (``"soc-minima"``, ``"soc-maxima"``) + - Each constraint-type key holds a list of dicts, one per violated slot (chronologically ordered): ``{"datetime": ISO 8601 UTC, "violation": "X kWh"}`` + - ``resolved``: List of soft constraints that were satisfied with available headroom + - Each entry is a dict with ``"asset"`` field and constraint-type keys + - Each constraint-type key holds a list of dicts, one per met slot (chronologically ordered): ``{"datetime": ISO 8601 UTC, "margin": "X kWh"}`` + + **Important:** ``soc-targets`` (hard constraints) are never included since they are strictly enforced by the scheduler. + Only hard constraint failures cause job failure. + + Example:: + + { + "unresolved": [ + { + "asset": 42, + "soc-minima": [ + {"datetime": "2024-01-01T10:00:00+00:00", "violation": "260.0 kWh"}, + {"datetime": "2024-01-01T10:15:00+00:00", "violation": "180.0 kWh"}, + ], + } + ], + "resolved": [ + { + "asset": 42, + "soc-maxima": [ + {"datetime": "2024-01-01T12:00:00+00:00", "margin": "40.0 kWh"}, + ], + } + ] + } + + For usage examples and interpretation guidance, see ``scheduling_constraint_results`` in the scheduling documentation. + """ + + unresolved: list = field(default_factory=list) + resolved: list = field(default_factory=list) + + def to_dict(self) -> dict: + """Serialize to a JSON-compatible dict.""" + return { + "unresolved": self.unresolved, + "resolved": self.resolved, + } + + @classmethod + def from_dict(cls, d: dict) -> "SchedulingJobResult": + """Deserialize from a dict.""" + return cls( + unresolved=d.get("unresolved", {}), + resolved=d.get("resolved", {}), + ) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index b352dc217f..1e824690c4 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -693,7 +693,7 @@ "/api/v3_0/sensors/{id}/schedules/{uuid}": { "get": { "summary": "Get schedule for one device", - "description": "Get a schedule from FlexMeasures.\n\nOptional fields:\n\n- \"duration\" (6 hours by default; can be increased to plan further into the future)\n- \"unit\" (by default, the unit of the schedule is the sensor's unit; a compatible unit can be requested)\n- \"sign-convention\" (controls how power values are signed in the response; see below)\n\nSign convention\n\nBy default (sign-convention: consumption-positive), the endpoint always returns schedules where\nconsumption is positive and production is negative, regardless of how the values are stored in the database.\nThis is the most common convention and matches the perspective of a consumer.\n\nSet sign-convention: production-positive to flip the sign so that production is returned as\npositive and consumption as negative. This matches the perspective of a producer.\n\nSet sign-convention: wysiwyg (what-you-see-is-what-you-get) to return the values with the same sign\nas database values and what is seen in UI charts. The values will indicate exactly what is stored,\nwhich is itself determined by the sensor's consumption_is_positive attribute (if set)\nor by the scheduler's default storage convention (production positive in the database).\n", + "description": "Get a schedule from FlexMeasures.\n\nOptional fields:\n\n- \"duration\" (6 hours by default; can be increased to plan further into the future)\n- \"unit\" (by default, the unit of the schedule is the sensor's unit; a compatible unit can be requested)\n- \"sign-convention\" (controls how power values are signed in the response; see below)\n\nSign convention\n\nBy default (sign-convention: consumption-positive), the endpoint always returns schedules where\nconsumption is positive and production is negative, regardless of how the values are stored in the database.\nThis is the most common convention and matches the perspective of a consumer.\n\nSet sign-convention: production-positive to flip the sign so that production is returned as\npositive and consumption as negative. This matches the perspective of a producer.\n\nSet sign-convention: wysiwyg (what-you-see-is-what-you-get) to return the values with the same sign\nas database values and what is seen in UI charts. The values will indicate exactly what is stored,\nwhich is itself determined by the sensor's consumption_is_positive attribute (if set)\nor by the scheduler's default storage convention (production positive in the database).\n\nConstraint analysis\n\nFor detailed constraint analysis (unmet and resolved constraints), use the\n[GET /api/v3_0/jobs/](#/Jobs/get_api_v3_0_jobs__uuid_) endpoint.\n", "security": [ { "ApiKeyAuth": [] @@ -4215,7 +4215,7 @@ "/api/v3_0/jobs/{uuid}": { "get": { "summary": "Get the status of a background job", - "description": "Look up a background job by its UUID and see whether it is\nqueued, running, finished, or failed.\n\nThe response includes a status message plus job metadata such\nas the queue name, function name, timestamps, and the job\nresult when available.\n\nFailed jobs also include traceback information when the worker\nstored it with the job result.\n", + "description": "Look up a background job by its UUID and see whether it is\nqueued, running, finished, or failed.\n\nThe response includes a status message plus job metadata such\nas the queue name, function name, timestamps, and the job\nresult when available.\n\nFailed jobs also include traceback information when the worker\nstored it with the job result.\n\nFor a finished scheduling job, result is an object. For a\nStorageScheduler job it holds soft state-of-charge constraint\nanalysis: unresolved lists constraints the scheduler could not\nsatisfy, and resolved lists constraints that were satisfied\nwith some margin. Each device entry's soc-minima/soc-maxima\nvalue is a list, holding one entry per violated slot (for\nunresolved) or per met slot with its margin (for resolved),\nordered chronologically. Both arrays are empty when the flex model\ndefines no soc-minima/soc-maxima, or when a scheduler other\nthan StorageScheduler was used. This is the only place\nconstraint analysis is available \u2014 the sensor schedule endpoint\n(GET /api/v3_0/sensors//schedules/) returns power\nvalues only.\n", "security": [ { "ApiKeyAuth": [] @@ -4260,7 +4260,7 @@ "description": "Human-readable description of the job status." }, "result": { - "description": "Return value of the job function, or null when not yet available.", + "description": "Return value of the job function, or null when not yet available. For a finished scheduling job, this is an object; a StorageScheduler job populates it with unresolved/resolved soft state-of-charge constraint analysis (empty arrays when the flex model defines no soc-minima/soc-maxima, or when a scheduler other than StorageScheduler was used).\n", "nullable": true }, "func_name": { @@ -4316,7 +4316,24 @@ "value": { "status": "FINISHED", "message": "Scheduling job has finished.", - "result": null, + "result": { + "unresolved": [ + { + "asset": 42, + "soc-minima": [ + { + "datetime": "2024-01-01T10:00:00+00:00", + "violation": "260.0 kWh" + }, + { + "datetime": "2024-01-01T10:15:00+00:00", + "violation": "180.0 kWh" + } + ] + } + ], + "resolved": [] + }, "func_name": "flexmeasures.data.services.scheduling.create_schedule", "origin": "scheduling", "enqueued_at": "2026-04-28T10:00:00+00:00",