docs(triage): CumulusCI v5 issue triage evidence pack + xfail regression tests

cumulusci/tests/triage/README.md

@@ -0,0 +1,50 @@
+# `cumulusci/tests/triage/`
+
+Regression-repro tests for open issues. Each test file targets one
+issue.
+
+## Conventions
+
+-   File naming: `test_issue_<NNNN>.py` where `<NNNN>` is the GitHub
+    issue number.
+-   Every test in this directory uses `@pytest.mark.xfail(strict=False)`
+    by default. The xfail marker captures the expected failure mode and
+    is removed by the corresponding fix-PR.
+-   `strict=False` is intentional: if a bug resolves independently (a
+    different PR lands, an upstream dependency is fixed, etc.), the
+    test will `XPASS` rather than fail CI. A harvest pass periodically
+    converts `XPASS` issues to `NOT-REPRODUCED-on-dev` and either
+    rewrites the test or drops it.
+-   Tests MUST be fast (`< 2s`), import-only or mocked. No live
+    Salesforce org, no real network, no scratch-org creation. Use
+    `unittest.mock` / fixtures liberally.
+
+## Lifecycle
+
+1.  Triage subagent verifies a bug reproduces on `origin/dev`.
+2.  Subagent writes `test_issue_<NNNN>.py` with `@pytest.mark.xfail` +
+    a code-level assertion that captures the bug.
+3.  Test is committed to this directory via the triage umbrella PR.
+4.  When the bug is fixed:
+    -   Fix-PR removes the `@pytest.mark.xfail` marker.
+    -   Fix-PR confirms the test now passes.
+    -   Fix-PR moves the test out of this directory to its natural
+        home (e.g. `cumulusci/tasks/<module>/tests/`) - or leaves it
+        here with the marker removed, whichever is cleaner.
+
+## See also
+
+-   `docs/triage/v5/repro-results.md` - narrative evidence per issue.
+-   `docs/triage/v5/fix-sketches/issue_<NNNN>.md` - proposed fix
+    approach per issue.
+-   `docs/triage/v5/proposals.md` - pass-1 classification matrix.
+
+## Running
+
+```bash
+uv run pytest cumulusci/tests/triage/ -v
+```
+
+Expected outcome: every test reports `XFAIL`. If any reports `XPASS`,
+that is signal that a bug resolved independently - see the harvest
+xpass list in `docs/triage/v5/` (if present).

cumulusci/tests/triage/test_issue_1348.py

@@ -0,0 +1,46 @@
+"""Regression repro for #1348.
+
+See docs/triage/v5/repro-results.md for full narrative.
+Verdict: REPRODUCED-on-v4.10.0 (R1+R2 + Task 4 recovery).
+This xfail marker will be removed by the corresponding fix-PR.
+
+Root cause: CumulusCI is GitHub-only. There are no `gitlab` or
+`bitbucket` references in the `cumulusci/` package, and the
+`ci_feature` flow still hardcodes GitHub-specific tasks
+(`github_parent_pr_notes`, `github_automerge_feature`). The
+2017 feature ask is to add a VCS abstraction so projects on
+other providers can use CumulusCI.
+
+This test loads the universal cumulusci.yml and asserts that
+`ci_feature` does NOT step into any GitHub-specific task -
+which it does today, so this fails -> XFAIL.
+"""
+
+import pytest
+
+from cumulusci.utils.yaml.cumulusci_yml import cci_safe_load
+from pathlib import Path
+
+import cumulusci
+
+
+@pytest.mark.xfail(
+    reason="repro for #1348 - see docs/triage/v5/repro-results.md",
+    strict=False,
+)
+def test_issue_1348():
+    universal_yaml = Path(cumulusci.__file__).resolve().parent / "cumulusci.yml"
+    config = cci_safe_load(universal_yaml.open(), str(universal_yaml))
+    flows = config.get("flows", {})
+    ci_feature = flows.get("ci_feature") or {}
+    steps = ci_feature.get("steps", {})
+    step_text = " ".join(
+        str(step.get("task") or step.get("flow") or "") for step in steps.values()
+    )
+    has_github_specific = (
+        "github_parent_pr_notes" in step_text or "github_automerge_feature" in step_text
+    )
+    assert not has_github_specific, (
+        "ci_feature still references GitHub-specific tasks; no VCS "
+        "abstraction exposed for GitLab/Bitbucket users (see #1348)."
+    )

cumulusci/tests/triage/test_issue_1432.py

@@ -0,0 +1,50 @@
+"""Regression repro for #1432.
+
+See docs/triage/v5/repro-results.md for full narrative.
+Verdict: REPRODUCED-on-v4.10.0 (no_reverify_needed).
+This xfail marker will be removed by the corresponding fix-PR.
+
+Root cause: ``BaseTask._validate_options`` (cumulusci/core/tasks.py
+around lines 187-197) only checks for *missing* required options when
+the task uses the legacy ``task_options`` dict. Unknown keys are
+silently accepted - passing ``-o colour red`` to a task that declares
+``color`` results in no error.
+
+The new-style Pydantic ``Options`` class path *does* reject extras
+(``"extra options"`` message in the same file), so the bug is partially
+mitigated for tasks that opt in. Legacy ``task_options`` dict tasks
+remain affected.
+
+The fix is to also reject unknown keys when validating the legacy
+dict-style options. This test asserts the source of
+``BaseTask._validate_options`` checks for unknown keys; on dev it fails
+because only ``required`` is checked.
+"""
+
+import inspect
+
+import pytest
+
+from cumulusci.core.tasks import BaseTask
+
+
+@pytest.mark.xfail(
+    reason="repro for #1432 - see docs/triage/v5/repro-results.md", strict=False
+)
+def test_issue_1432():
+    src = inspect.getsource(BaseTask._validate_options)
+    has_unknown_check = any(
+        token in src
+        for token in (
+            "not in self.task_options",
+            "not in task_options",
+            "unknown",
+            "unexpected",
+            "extra option",
+        )
+    )
+    assert has_unknown_check, (
+        "BaseTask._validate_options still only checks for missing required "
+        "options; unknown task_options keys are silently accepted for legacy "
+        "task_options-dict tasks (see #1432)"
+    )

cumulusci/tests/triage/test_issue_1769.py

@@ -0,0 +1,47 @@
+"""Regression repro for #1769.
+
+See docs/triage/v5/repro-results.md for full narrative.
+Verdict: REPRODUCED-on-dev (R1+R2 + Task 4 recovery).
+This xfail marker will be removed by the corresponding fix-PR.
+
+Root cause: cumulusci/tasks/bulkdata/tests/test_load.py still uses the
+2020-vintage pattern `lookups["Id"] = MappingLookup(name="Id",
+table="accounts", key_field="sf_id")` (line 739 and several siblings:
+~754, 773, 801, 1119, 1187, 1255) to describe an after-step's
+UPDATE-on-Id dependency. davidmreed acknowledged in 2020 that this
+is a "horrible hack" he intended to clean up; six years later it
+remains in the test file.
+
+A real fix would remove `Id` from the `lookups` dict (or stop using
+`MappingLookup` to express the self-update relationship) and have
+the after-step logic synthesize that relationship internally.
+
+This test asserts that the offending pattern is absent from test_load.py.
+On dev the pattern is present, so the assertion fails -> XFAIL.
+"""
+
+from pathlib import Path
+
+import pytest
+
+import cumulusci
+
+
+@pytest.mark.xfail(
+    reason="repro for #1769 - see docs/triage/v5/repro-results.md", strict=False
+)
+def test_issue_1769():
+    test_load_path = (
+        Path(cumulusci.__file__).parent
+        / "tasks"
+        / "bulkdata"
+        / "tests"
+        / "test_load.py"
+    )
+    text = test_load_path.read_text()
+    bad_pattern = 'lookups["Id"] = MappingLookup('
+    assert bad_pattern not in text, (
+        f"test_load.py still contains the 2020-vintage smell {bad_pattern!r}; "
+        "expected the after-step Id-update relationship to be expressed "
+        "without injecting a MappingLookup keyed on 'Id'."
+    )

cumulusci/tests/triage/test_issue_2013.py

@@ -0,0 +1,51 @@
+"""Regression repro for #2013.
+
+See docs/triage/v5/repro-results.md for full narrative.
+Verdict: REPRODUCED-on-dev (R1+R2 + Task 4 recovery).
+This xfail marker will be removed by the corresponding fix-PR.
+
+Root cause: cumulusci.tasks.bulkdata.utils.create_table_if_needed builds
+a SQLAlchemy `Table(tablename, metadata, *fields)` before calling
+`inspector.has_table()`. When two mapping steps share the same
+sf_object name, the SQLAlchemy `Table()` constructor raises
+`InvalidRequestError: Table 'X' is already defined ...` BEFORE the
+intended `BulkDataException("Table already exists: ...")` is raised.
+
+The fix is to either (a) wrap the `Table()` call in a try/except and
+re-raise as BulkDataException, or (b) validate uniqueness at
+mapping-parse time. Either way, when the table already exists in the
+metadata, callers should observe a CumulusCI-typed BulkDataException,
+not a SQLAlchemy-typed InvalidRequestError.
+"""
+
+from sqlalchemy import Column, MetaData, String, create_engine
+
+import pytest
+
+from cumulusci.core.exceptions import BulkDataException
+from cumulusci.tasks.bulkdata.utils import create_table_if_needed
+
+
+@pytest.mark.xfail(
+    reason="repro for #2013 - see docs/triage/v5/repro-results.md", strict=False
+)
+def test_issue_2013():
+    engine = create_engine("sqlite:///:memory:")
+    metadata = MetaData(bind=engine)
+
+    create_table_if_needed(
+        "Account", metadata, [Column("sf_id", String(255), primary_key=True)]
+    )
+
+    raised = None
+    try:
+        create_table_if_needed(
+            "Account", metadata, [Column("sf_id", String(255), primary_key=True)]
+        )
+    except BaseException as e:
+        raised = e
+
+    assert isinstance(raised, BulkDataException), (
+        "Expected BulkDataException for duplicate table; "
+        f"got {type(raised).__name__}: {raised}"
+    )

cumulusci/tests/triage/test_issue_2140.py

@@ -0,0 +1,46 @@
+"""Regression repro for #2140.
+
+See docs/triage/v5/repro-results.md for full narrative.
+Verdict: REPRODUCED-on-v4.10.0 (R1+R2 + Task 4 recovery).
+This xfail marker will be removed by the corresponding fix-PR.
+
+Root cause: `CliRuntime.get_org()` in cumulusci/cli/runtime.py
+hard-errors with `click.UsageError("No org specified and no
+default org set.")` when the requested org does not exist in
+the keychain (or `OrgNotFound` from `keychain.get_org()`
+bubbles up to `cli/org.py` as a plain ClickException). The 2020
+ask is to instead surface an interactive prompt that lets the
+user pick from configured scratch org configs.
+
+This test asserts that `get_org` source contains a prompt
+mechanism (click.prompt / click.Choice / scratch-config-listing
+helper). On dev it doesn't, so the assertion fails -> XFAIL.
+"""
+
+import inspect
+
+import pytest
+
+from cumulusci.cli.runtime import CliRuntime
+
+
+@pytest.mark.xfail(
+    reason="repro for #2140 - see docs/triage/v5/repro-results.md",
+    strict=False,
+)
+def test_issue_2140():
+    src = inspect.getsource(CliRuntime.get_org)
+    has_prompt = any(
+        token in src
+        for token in (
+            "click.prompt",
+            "click.confirm",
+            "click.Choice",
+            "scratch_configs",
+            "scratch configs",
+        )
+    )
+    assert has_prompt, (
+        "CliRuntime.get_org still hard-errors when no org is found; "
+        "no interactive scratch-config picker (see #2140)."
+    )

cumulusci/tests/triage/test_issue_2153.py

@@ -0,0 +1,48 @@
+"""Regression repro for #2153.
+
+See docs/triage/v5/repro-results.md for full narrative.
+Verdict: REPRODUCED-on-dev (R1+R2 + Task 4 recovery).
+This xfail marker will be removed by the corresponding fix-PR.
+
+Root cause: `MergeBranch._create_conflict_pull_request`
+(cumulusci/tasks/github/merge.py:264-288) only calls
+`self.repo.create_pull(...)` to open the auto-generated
+"Merge <source> into <branch>" PR. The original 2020 ask is to
+also drop a comment on the source/child PR which tags the branch
+subscribers so they get notified about the conflict.
+
+A repo-wide search for `create_comment`/`issue_comment` returns
+only test-fixture hits - production GitHub task code never
+opens a PR/issue comment as part of this conflict path.
+
+This test asserts that the merge task surface mentions a
+comment-posting call. On dev it doesn't, so the assertion fails
+-> XFAIL.
+"""
+
+import inspect
+
+import pytest
+
+from cumulusci.tasks.github.merge import MergeBranch
+
+
+@pytest.mark.xfail(
+    reason="repro for #2153 - see docs/triage/v5/repro-results.md",
+    strict=False,
+)
+def test_issue_2153():
+    src = inspect.getsource(MergeBranch)
+    posts_comment = any(
+        token in src
+        for token in (
+            "create_comment",
+            "issue_comment",
+            "post_comment",
+        )
+    )
+    assert posts_comment, (
+        "MergeBranch still only opens an auto-merge PR on conflict; "
+        "no comment-on-original-PR path found to notify branch "
+        "subscribers (see #2153)."
+    )

cumulusci/tests/triage/test_issue_2325.py

@@ -0,0 +1,53 @@
+"""Regression repro for #2325.
+
+See docs/triage/v5/repro-results.md for full narrative.
+Verdict: REPRODUCED-on-v4.10.0 (no_reverify_needed).
+This xfail marker will be removed by the corresponding fix-PR.
+
+Root cause: CumulusCI ships ``disable_tdtm_trigger_handlers`` /
+``restore_tdtm_trigger_handlers`` (cumulusci.yml:738-747) for triggers
+and ``set_duplicate_rule_status`` for DuplicateRule, but offers no
+analogous task for toggling Salesforce ValidationRules around a data
+load. The user requested an OOTB ``set_validation_rule_status`` /
+``disable_validation_rules`` task; ``cci task list | grep -i validation``
+returns only the duplicate-rule task on v4.10.0.
+
+The fix is to add a ``MetadataSingleEntityTransformTask`` subclass for
+``entity = "ValidationRule"`` and wire it into ``cumulusci.yml`` with
+both disable/restore (or set-status) flavours, mirroring the existing
+TDTM/DuplicateRule pattern.
+
+This test asserts ``cumulusci.yml`` declares at least one validation-rule
+toggle task; on dev it fails because no such task exists.
+"""
+
+from pathlib import Path
+
+import pytest
+import yaml
+
+import cumulusci
+
+
+@pytest.mark.xfail(
+    reason="repro for #2325 - see docs/triage/v5/repro-results.md", strict=False
+)
+def test_issue_2325():
+    cci_root = Path(cumulusci.__file__).parent
+    with open(cci_root / "cumulusci.yml") as f:
+        data = yaml.safe_load(f)
+
+    task_names = set(data.get("tasks", {}).keys())
+    candidates = {
+        "set_validation_rule_status",
+        "disable_validation_rules",
+        "restore_validation_rules",
+        "activate_validation_rules",
+        "deactivate_validation_rules",
+    }
+    intersection = task_names & candidates
+    assert intersection, (
+        "cumulusci.yml still ships no ValidationRule toggle task. Expected one "
+        f"of {sorted(candidates)} to mirror the disable_tdtm_trigger_handlers / "
+        "set_duplicate_rule_status pattern (see #2325)"
+    )