From 75ee31e223d3f6f832fd6d39e4fca106b6cd0bfe Mon Sep 17 00:00:00 2001 From: James Kent Date: Fri, 16 Oct 2026 08:34:12 -0500 Subject: [PATCH] Add an optional stable_id to every extraction entity A store mints stable_id, a lowercase UUID, the first time it sees an entity, and keeps it across re-runs, so a later extraction of the same paper matches the same entity. A local_id is unique only within one record. Extractors never set it. It is declared in extraction-deviations.yaml's required_additions, so pondie's generator keeps it. Storage's `id` is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../run/records/x7Ex4mpleA01.extraction.json | 1 + extraction-readme.md | 8 + .../jsonschema/extraction-record.schema.json | 192 +++++++++++ .../linkml/extraction-deviations.yaml | 126 +++++++ .../acquisition.yaml | 12 + .../analysis.yaml | 48 +++ .../assessment.yaml | 6 + .../neuroimaging-study-extraction/design.yaml | 12 + .../neuroimaging-study-extraction/group.yaml | 6 + .../neuroimaging-study-extraction/region.yaml | 6 + .../neuroimaging-study-extraction/study.yaml | 6 + .../neuroimaging-study-extraction/task.yaml | 12 + src/study_schema/models/extraction.py | 322 ++++++++++++++++++ tests/test_stable_id.py | 72 ++++ 14 files changed, 829 insertions(+) create mode 100644 tests/test_stable_id.py diff --git a/examples/run/records/x7Ex4mpleA01.extraction.json b/examples/run/records/x7Ex4mpleA01.extraction.json index 782e032..83a5e31 100644 --- a/examples/run/records/x7Ex4mpleA01.extraction.json +++ b/examples/run/records/x7Ex4mpleA01.extraction.json @@ -7,6 +7,7 @@ "repaired_by": null, "paper_sections": null }, + "stable_id": null, "local_id": "x7Ex4mpleA01", "description": null, "hypothesis": null, diff --git a/extraction-readme.md b/extraction-readme.md index f10d865..cb5fa4a 100644 --- a/extraction-readme.md +++ b/extraction-readme.md @@ -244,6 +244,14 @@ is the `FactorLevel.level` label pointing at it — `baseline`, `T0`, `week 12`, paper writes. What is normalized is placement (`relation_to_intervention`) and sequence (`order`), which carry every query an occasion vocabulary would have served. +### An entity's stable_id is minted by the store + +Every entity may carry a `stable_id`, a UUID that the store mints the first time the entity +appears and keeps across re-runs, so a later extraction matches the entity it describes. An +extractor never sets it and leaves it null. A `local_id` is unique only within one record, so +it cannot tell the same entity from a new one after a rename or a pipeline re-run; the +`stable_id` can. + ### Cells carry direction; weights do not Direction is recorded on the cell, never as a numeric contrast weight. One `Cell` is one level diff --git a/src/study_schema/jsonschema/extraction-record.schema.json b/src/study_schema/jsonschema/extraction-record.schema.json index d87a025..d31d305 100644 --- a/src/study_schema/jsonschema/extraction-record.schema.json +++ b/src/study_schema/jsonschema/extraction-record.schema.json @@ -47,6 +47,14 @@ "modality": { "$ref": "#/$defs/ExtractedModality", "description": "Imaging or recording modality. A value in the Modality vocabulary also names the class holding this acquisition's modality-specific parameters, through that value's `instantiates`.\nOpen vocabulary: write the source's own wording in the field when nothing here fits, because the value is required and a closed range would discard the whole acquisition over one unrecognised word. Only a vocabulary value carries an `instantiates`, so an acquisition outside it resolves to the base class until normalization maps the wording back on -- which is why `acquisition_type` is derived rather than extracted, and why an unmapped modality is worth reporting." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -256,6 +264,14 @@ "$ref": "#/$defs/ExtractedSpatialScope", "description": "Spatial domain of the analysis modelled. The spatial_scope refers to what voxels were modelled, not which voxels/brain regions had a correction applied." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "tables": { "description": "The source tables reporting results for this analysis.", "items": { @@ -409,6 +425,14 @@ "name": { "$ref": "#/$defs/ExtractedString", "description": "Short arm label as the source states it, for example methylphenidate, sham tDCS, waitlist." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -467,6 +491,14 @@ "name": { "$ref": "#/$defs/ExtractedString", "description": "Name of the assessment, such as Digit Span, MMSE, or BDI." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -656,6 +688,14 @@ "$ref": "#/$defs/ExtractedString", "description": "Short condition label that distinguishes the condition from other task periods or modeled effects." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "stimulus_content": { "anyOf": [ { @@ -981,6 +1021,14 @@ } ], "description": "What these coordinates are for." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -1134,6 +1182,14 @@ } ], "description": "Model or product name of the device, as reported, such as Prisma, Signa, Achieva, or actiCHamp." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -1255,6 +1311,14 @@ } ], "description": "Sampling frequency of the recording in hertz." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -4490,6 +4554,14 @@ ], "description": "Reported source-detector separations in millimeters." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "wavelengths_nm": { "anyOf": [ { @@ -5015,6 +5087,14 @@ "species": { "$ref": "#/$defs/ExtractedSpecies", "description": "The species of this cohort." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -5176,6 +5256,14 @@ ], "description": "Reported search volume, mask size, or number of vertices or voxels used for inference." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "tfce_parameters": { "anyOf": [ { @@ -5493,6 +5581,14 @@ } ], "description": "Time between the start of successive acquired volumes, in seconds." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -5567,6 +5663,14 @@ ], "description": "Named metric when type is intentionally broad, such as fractional anisotropy, ALFF, fALFF, or a specific tracer binding potential." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "type": { "$ref": "#/$defs/ExtractedMeasureType", "description": "Normalized quantity, for cross-study queries." @@ -5762,6 +5866,14 @@ ], "description": "The element this model was fitted over. Distinct from the Analysis's `spatial_scope`, which gives the extent rather than the element: a whole-brain model can be voxelwise or vertexwise, and a model over an ROI can average the region or test its parcels. Record it rather than inferring it from `coordinate_space`.\nA model fitted over a different element is a different model, so a voxelwise GLM and an ROI-mean ANOVA of the same design take two records: no contrast of one can be read off the other's fit." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "stage": { "$ref": "#/$defs/ExtractedString", "description": "Stage of analysis, in the source's terms: run, session, subject and group are the usual words, and any other the source uses is storable as written. Deliberately left open -- no vocabulary fits every design, and nothing derives anything from it.\nA label, not an ordering. Only `inputs_from` records which stage fed another: two records labelled `group` say nothing about their relationship, and these strings never imply a chain." @@ -5869,6 +5981,14 @@ ], "description": "The source's own definition of the term, and the origin of a continuous term that no Assessment supplies, such as age, dose, a motion parameter, cardiac phase, or scanner drift.\n**Required for a derived column, whatever its `assessment` says.** A change score names its instrument in `assessment` and its derivation here, in the source's own words and including the occasions it spans -- \"percent reduction in BDI, (post-pre)/pre, from baseline to post-treatment\". Nothing else can carry those occasions: `FactorLevel.timepoints` names the occasions a factor *compares*, and a derived column has no levels to hang them on, so where a study has several post-intervention occasions this is the only thing separating a change to the endpoint from a change to a later follow-up.\nNot the shape of the fitted relationship, which is `functional_form`: a percent change entered linearly is a linear term computed from a difference, so recording the construction as the form would lose whether the fit was linear or quadratic. A transform *of the measure* the vocabulary cannot name still goes in functional_form." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "type": { "$ref": "#/$defs/ExtractedTermType", "description": "Whether the term is categorical or continuous. Categorical when the column distinguishes named cells of the design -- occasions, cohorts, conditions, arms, regions -- whether or not the paper names the axis; `levels` is then required. Continuous when the column carries a measured value and the model fits a slope.\nSmall as it is, this field settles four others: whether `levels` exists, whether a `Cell` here takes a `level`, whether the term can be crossed, and which branch of the derived kind fires. A term recorded continuous is one no contrast can be built across.\n**A term whose name states a comparison -- `pre > post`, `A versus B` -- is almost always a categorical factor written down from its contrast's side.** The design matrix distinguished two occasions, cohorts or conditions; the paper labelled the difference and left the axis unnamed. Record the factor with a level per side and let the cells carry the direction (`representing-models.md` §5.6). Collapsed into one continuous column, nothing says what was compared and the effect reads as a slope.\nThe exception is a column genuinely holding one number per participant -- a difference score, a percent change entered as a covariate. That is continuous, and `source_definition` records the subtraction it came from." @@ -6052,6 +6172,14 @@ } ], "description": "Why this is not an MRI, EEG, FNIRS, or PET record." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -6166,6 +6294,14 @@ ], "description": "Reported reconstructed spatial resolution, such as FWHM dimensions, in millimeters." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "tracer_name": { "anyOf": [ { @@ -6363,6 +6499,14 @@ ], "description": "Software packages or methods used, including version and revision identifiers." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "steps": { "anyOf": [ { @@ -6439,6 +6583,14 @@ } ], "description": "What kind of thing the region is, which `definition_method` does not settle: an atlas parcel and an anatomical structure can both come from an atlas, and a network is neither." + }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] } }, "required": [ @@ -6897,6 +7049,14 @@ ], "description": "Whether the paper reports inferential analyses of its own. An empty `analyses` list is otherwise ambiguous: a methods or data paper that tests nothing and a paper the extractor read badly look the same. With this set, `no_analyses_reported` asserts the emptiness, with the sentences that show it as evidence.\nSays nothing about whether the analyses found anything; that is each `Analysis.outcome`, summarised in `outcome_summary`." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "study_type": { "description": "PubMed publication types for this record, verbatim, copied from the parsed paper's `Bibliography.publication_types`, not read off the paper: it is what the indexers said about the article, which is what a publication-type exclusion asks about.", "items": { @@ -7070,6 +7230,14 @@ ], "description": "What this table is for. `reported_effect` when its rows are the foci of a tested effect, which is the common case and which is derived from the analyses that name the table; otherwise what the rows are instead -- region or ROI definitions, atlas parcels, connectivity seeds, the peaks of a decomposition's components, coordinates cited from other papers, a stimulus list, demographics, descriptive statistics with no test.\nA coordinate table is not an analysis merely by having coordinates in it, and this is the only field that says which it is. Absence means nothing decided: a table deliberately encoded as a non-analysis and one no pass considered are otherwise the same silence, and the coordinates of the first belong on the entity they locate (a Region's description) rather than on a contrast that never produced them.\nA per-table answer, so it cannot be right for a table whose row groups serve different purposes -- results beside the ROIs they were tested in. Each row group states its own role in a `CoordinateSet`." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "table_number": { "anyOf": [ { @@ -7242,6 +7410,14 @@ ], "description": "How participants responded. Multivalued because a paradigm often collects more than one -- a button press for accuracy and a spoken report afterwards." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "stimuli": { "anyOf": [ { @@ -7359,6 +7535,14 @@ "$ref": "#/$defs/ExtractedTimepointRelation", "description": "Where this occasion sits relative to the intervention. Required, so an occasion the source does not place still carries the slot, as `extraction_status: not_reported`." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "time_from_intervention": { "anyOf": [ { @@ -7626,6 +7810,14 @@ ], "description": "Whether the paper reports inferential analyses of its own. An empty `analyses` list is otherwise ambiguous: a methods or data paper that tests nothing and a paper the extractor read badly look the same. With this set, `no_analyses_reported` asserts the emptiness, with the sentences that show it as evidence.\nSays nothing about whether the analyses found anything; that is each `Analysis.outcome`, summarised in `outcome_summary`." }, + "stable_id": { + "description": "UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": [ + "string", + "null" + ] + }, "study_type": { "description": "PubMed publication types for this record, verbatim, copied from the parsed paper's `Bibliography.publication_types`, not read off the paper: it is what the indexers said about the article, which is what a publication-type exclusion asks about.", "items": { diff --git a/src/study_schema/linkml/extraction-deviations.yaml b/src/study_schema/linkml/extraction-deviations.yaml index 2a86d14..2041f48 100644 --- a/src/study_schema/linkml/extraction-deviations.yaml +++ b/src/study_schema/linkml/extraction-deviations.yaml @@ -150,12 +150,138 @@ required_additions: slots: + # The store's key for an entity across re-runs; storage's `id` stays the store's own and is never extracted. study.Study: extraction_metadata: range: ExtractionMetadata required: true inlined: true description: Provenance for this extraction record. + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + acquisition.Acquisition: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + acquisition.Device: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.Analysis: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.Measure: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.Preprocessing: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.ModelEstimation: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.ModelTerm: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.InferenceSettings: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.Table: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + analysis.CoordinateSet: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + assessment.Assessment: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + design.Arm: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + design.Timepoint: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + group.Group: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + region.Region: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + task.Task: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. + task.Condition: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. # Nothing here yet. The projection is the baseline; add entries as data shows a different # shape extracts better. Candidates carried over from the hand-written schema that was diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/acquisition.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/acquisition.yaml index b873239..a66311a 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/acquisition.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/acquisition.yaml @@ -96,6 +96,12 @@ classes: lists together. Everything modality-specific lives in the class the Modality value instantiates -- MRI, EEG, FNIRS, PET, or OtherModality. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -281,6 +287,12 @@ classes: The scanner, amplifier, or recording system an acquisition was made on, split into the two parts a source reliably reports. Either part can be missing: papers name a model with no manufacturer as often as the reverse. One record is one physical machine, referenced by every acquisition made on it: a study that collected functional, structural and diffusion scans in one session has three Acquisitions and one Device. Two records mean the study says it used two machines -- a second site, or a rescan on different hardware -- which is a claim about the data and not a way of recording that two sequences differed. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/analysis.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/analysis.yaml index 8ae8262..a593987 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/analysis.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/analysis.yaml @@ -32,6 +32,12 @@ classes: A distinct reported inferential map or tested effect, linking its source acquisitions, sample, paradigm context, tested effect, preprocessing, model estimation, inference settings, and source result tables. Create one Analysis for each distinct tested effect. Separate Analyses are required when normalized direction, participant-group comparison, method, the pattern of `effect.cells`, seed or connectivity model, decoded variable, component identity, or spatial_scope differ, and when an omnibus effect and its directional post-hoc contrast are both reported. ROI definitions, acquisition summaries, descriptive sample tables, and masks without inferential effects are not Analyses. Nor is a map reported with no inferential test at all -- an ICA or PCA component map presented descriptively, a connectivity matrix given without a test -- there being nothing for such a map to be the effect of. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -270,6 +276,12 @@ classes: One record may be referenced by several Analyses, so every field must hold for all of them; an analysis of a different quantity takes its own record. The test is mechanical: two analyses share a record when `family`, `type`, `specific_metric` and `unit` all match. A study measures one or two quantities and tests them many ways, so sharing is the common case -- eight analyses of one connectivity measure are one record, not eight. Not a property of the thresholding scheme: one correction scheme routinely covers several measures, and one measure is often tested under several schemes. See storage-schema-design-notes.md for the reasoning behind this shape. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -377,6 +389,12 @@ classes: An ordered preprocessing pipeline applied to acquired data before or as part of statistical modeling. This can be expanded with COBIDAS. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -415,6 +433,12 @@ classes: The test for whether two analyses share a record is mechanical: list the columns each was fitted with. A different dependent variable, a different covariate set, or a different subset of participants means a different design matrix and therefore a separate record, even when family, estimator and software are identical. Two BPM regressions of the same measure on the same regressor, one over the patients and one over patients and controls pooled, are two records. Sharing one record across matrices corrupts the derived adjustment set. The adjustment set is the model's terms minus the terms named by its cells. If several matrices share one record, they claim adjustments that their original models never made, and the record cannot expose the error. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -498,6 +522,12 @@ classes: A term is something the measure was regressed *on*. The quantity being modelled is `Analysis.measure` and never a term of the model that models it. A table laid out with one row-block per measured parameter -- four diffusion parameters, three frequency bands -- is reporting one analysis per parameter over one measure each, not one analysis over a factor whose levels are the parameters. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -636,6 +666,12 @@ classes: One record may be referenced by several Analyses, so every field must hold for all of them; an analysis thresholded or corrected differently takes its own record. The test is mechanical: two analyses share a record when the threshold, its type, the correction method, the scope corrected over and the alpha are all the same. A paper that states one correction scheme in Methods and applies it throughout has one record. Inference attaches to the analysis, not to the fit: one model estimated once is often thresholded several ways -- an ROI test Bonferroni-corrected over four regions and a searchlight over the same fit corrected by FDR. See storage-schema-design-notes.md for the reasoning behind this shape. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -705,6 +741,12 @@ classes: cell structure and point-level provenance may remain in the repository's coordinate-table representation and be linked with source_path. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -740,6 +782,12 @@ classes: A group of coordinates reported together -- a row group of a table, or a run of sentences in the text -- and what it is for. The coordinates stay in the parse, as for tables; this records their role and links them to what they belong to: the analysis a result reports, the regions an anchor locates. A set has one role: its dominant one. The groups come from the table parse, which splits a table by what was tested rather than by what each row is for, so a set can mix roles -- a connectivity results table that lists its own seed as a row. Such a set keeps the role of most of its points, and the odd point takes that role with it. This is a known error, about 1.3% of sets, accepted rather than splitting the parse's groups. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/assessment.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/assessment.yaml index 37c4caa..46a2a40 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/assessment.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/assessment.yaml @@ -24,6 +24,12 @@ classes: procedure that produces participant-level measurements independently of the neuroimaging acquisition or experimental conditions. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/design.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/design.yaml index 8a3ab2e..1107649 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/design.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/design.yaml @@ -190,6 +190,12 @@ classes: description: One thing participants were assigned to receive, or to receive in place of it. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -219,6 +225,12 @@ classes: One occasion on which data were collected, and where it sits relative to the intervention. Declared once on `StudyDesign.timepoints` and referenced by identifier; `FactorLevel.timepoints` is the only slot that references one, so it is the whole of how an occasion reaches a model. A study that *compared* occasions did so with a `ModelTerm` whose levels name them, paradigm or no paradigm. An occasion no level references was collected but never modelled: legitimate where the record only describes when follow-up happened, a defect where the analyses report change over time, which `validate_record.py` flags. Names stay in the source's wording; what is normalized is where an occasion sits and in what order. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/group.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/group.yaml index eb55773..846c737 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/group.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/group.yaml @@ -46,6 +46,12 @@ classes: Group: description: An aggregate participant group and its reported demographics. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/region.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/region.yaml index f27b611..901b020 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/region.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/region.yaml @@ -64,6 +64,12 @@ classes: a model term carries. Declared once and referenced from each, because the same seed written separately in each place is three spellings nothing can join. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/study.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/study.yaml index 52a7926..9b028db 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/study.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/study.yaml @@ -54,6 +54,12 @@ classes: required: true inlined: true description: Provenance for this extraction record. + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/linkml/neuroimaging-study-extraction/task.yaml b/src/study_schema/linkml/neuroimaging-study-extraction/task.yaml index 99ee4dd..0d0153c 100644 --- a/src/study_schema/linkml/neuroimaging-study-extraction/task.yaml +++ b/src/study_schema/linkml/neuroimaging-study-extraction/task.yaml @@ -157,6 +157,12 @@ classes: a structural comparison across occasions is carried by the Effect's timepoint terms and linked through Analysis.acquisitions. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string @@ -261,6 +267,12 @@ classes: the task was run on and the arm it was run under are Timepoints and Arms, and enter an analysis as levels of their own model terms, compared by `Cell`s on the Effect. attributes: + stable_id: + range: string + pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + description: >- + UUID minted by the store the first time this entity appears, never by an extractor, + and kept across re-runs so a later extraction matches the same entity. local_id: identifier: true range: string diff --git a/src/study_schema/models/extraction.py b/src/study_schema/models/extraction.py index 57e71da..fa7214d 100644 --- a/src/study_schema/models/extraction.py +++ b/src/study_schema/models/extraction.py @@ -1540,24 +1540,52 @@ class Arm(ConfiguredBaseModel): """ One thing participants were assigned to receive, or to receive in place of it. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) name: ExtractedString = Field(default=..., description="""Short arm label as the source states it, for example methylphenidate, sham tDCS, waitlist.""") description: Optional[ExtractedString] = Field(default=None, description="""Complete definition of what this arm received.""") arm_kind: ExtractedArmKind = Field(default=..., description="""What class of thing this arm received. Required, so an arm whose kind the source leaves unclear still carries the slot, as `extraction_status: not_reported`.""") agent: Optional[ExtractedString] = Field(default=None, description="""The drug, agent, or stimulation target this arm received, as the source names it. The name only; dose, unit, route, and schedule are not recorded.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class Timepoint(ConfiguredBaseModel): """ One occasion on which data were collected, and where it sits relative to the intervention. Declared once on `StudyDesign.timepoints` and referenced by identifier; `FactorLevel.timepoints` is the only slot that references one, so it is the whole of how an occasion reaches a model. A study that *compared* occasions did so with a `ModelTerm` whose levels name them, paradigm or no paradigm. An occasion no level references was collected but never modelled: legitimate where the record only describes when follow-up happened, a defect where the analyses report change over time, which `validate_record.py` flags. Names stay in the source's wording; what is normalized is where an occasion sits and in what order. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) name: ExtractedString = Field(default=..., description="""Short occasion label as the source states it, for example baseline, post-treatment, six-month follow-up.""") relation_to_intervention: ExtractedTimepointRelation = Field(default=..., description="""Where this occasion sits relative to the intervention. Required, so an occasion the source does not place still carries the slot, as `extraction_status: not_reported`.""") order: Optional[ExtractedInteger] = Field(default=None, description="""Position of this occasion in the study's sequence, counting from 1. Carries the ordering that `relation_to_intervention` cannot when several occasions share a relation -- the earliest `post_intervention` occasion is the study's endpoint, later ones are follow-ups.""") time_from_intervention: Optional[ExtractedString] = Field(default=None, description="""Elapsed time between the intervention and this occasion, in the source's own words and units, for example 24 h or six weeks.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class ExtractedAllocation(ExtractedValue): """ @@ -1618,6 +1646,7 @@ class Group(ConfiguredBaseModel): """ An aggregate participant group and its reported demographics. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) name: ExtractedString = Field(default=..., description="""Study-local group label, for example healthy controls or patients with Parkinson disease.""") description: Optional[ExtractedString] = Field(default=None, description="""Narrative definition of the group and its role in the study.""") @@ -1666,6 +1695,19 @@ class Group(ConfiguredBaseModel): age_unit_normalized: Optional[ExtractedString] = Field(default=None, description="""`age_unit` folded onto years / months / weeks / gestational_weeks / UNKNOWN by `pondie.normalization.age_unit`, never extracted. The source's own wording stays in `age_unit`.""") other_characteristics: Optional[ExtractedStringList] = Field(default=None, description="""Cohort traits moved out of `population_characteristics` because every plausible cohort could carry them. Written by `pondie.normalization.population_characteristics`, never extracted.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class CategoryDistribution(ConfiguredBaseModel): """ @@ -1705,6 +1747,7 @@ class Task(ConfiguredBaseModel): """ A meaningful participant paradigm, such as a task, resting-state protocol, stimulation, or passive viewing, performed in association with an acquisition. A Task record is required wherever the study models conditions, a resting-state protocol included. Acquisitions with no paradigm do not require one, even in a longitudinal or interventional study: a structural comparison across occasions is carried by the Effect's timepoint terms and linked through Analysis.acquisitions. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) name: Optional[ExtractedString] = Field(default=None, description="""Short, unique label for the task or paradigm; corresponds conceptually to BIDS TaskName when applicable.""") paradigm: Optional[ExtractedString] = Field(default=None, description="""The name with every qualifier that says how this version varies taken off and moved to its own field: `Stroop task` for an emotional Stroop task, `flanker task` for an auditory flanker task, `n-back task` for a food-picture n-back, `incentive delay task` for a monetary incentive delay task. The same as `name` when nothing qualifies it. @@ -1725,6 +1768,19 @@ class Task(ConfiguredBaseModel): response_modality: Optional[ExtractedResponseModalityList] = Field(default=None, description="""How participants responded. Multivalued because a paradigm often collects more than one -- a button press for accuracy and a spoken report afterwards.""") performance_measures: Optional[ExtractedStringList] = Field(default=None, description="""Behavioral variables recorded and summaries used to verify task performance.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class TaskQualifier(ConfiguredBaseModel): """ @@ -1738,6 +1794,7 @@ class Condition(ConfiguredBaseModel): """ A named condition of a task: a modelled state, event type, or baseline that an analysis can contrast against another. Only what happened inside the paradigm -- the occasion the task was run on and the arm it was run under are Timepoints and Arms, and enter an analysis as levels of their own model terms, compared by `Cell`s on the Effect. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier.""") name: ExtractedString = Field(default=..., description="""Short condition label that distinguishes the condition from other task periods or modeled effects.""") stimulus_content: Optional[ExtractedStringList] = Field(default=None, description="""What this condition's stimuli were about -- their category, not the channel (`Task.stimulus_modality`) or the materials (`Task.stimuli`): a cigarette photograph is `cigarettes`. Faces, scenes, words, tones, music, food, money -- examples, not a vocabulary. @@ -1747,6 +1804,19 @@ class Condition(ConfiguredBaseModel): A rest or fixation period inside another paradigm is a Condition of that paradigm's Task. A standalone resting-state run is not: it is its own Task, with one condition or none. The distinction is which paradigm the participant was in, not what they were doing -- a fixation baseline between working-memory blocks belongs to the working-memory Task. What a tested effect was measured against is not recorded here: a contrast against this condition gives it a `negative` cell, and a contrast against the unmodelled implicit baseline has no second cell.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class ExtractedTaskVariantAxis(ExtractedValue): """ @@ -1807,16 +1877,31 @@ class Assessment(ConfiguredBaseModel): """ A behavioral, cognitive, biological, clinical, or questionnaire-based instrument or procedure that produces participant-level measurements independently of the neuroimaging acquisition or experimental conditions. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier.""") name: ExtractedString = Field(default=..., description="""Name of the assessment, such as Digit Span, MMSE, or BDI.""") description: Optional[ExtractedString] = Field(default=None, description="""Description of the assessment and the measurements it produced, including the purpose the source gives for administering it -- establishing a group's diagnosis, screening a comorbidity, grading severity, characterizing everyone. That purpose is what decides whether a group may name this assessment as its `diagnostic_instrument`, and it is often the only sentence distinguishing the two.""") assessment_type: Optional[ExtractedString] = Field(default=None, description="""Broad type, such as cognitive test, clinical scale, diagnostic interview, questionnaire, behavioral task, physiological measure, or a biospecimen assay such as salivary cortisol or a plasma marker. Free text rather than a vocabulary: create a category if none of these fit.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class Region(ConfiguredBaseModel): """ A brain region the study delimited and used: a connectivity seed or target, the search space an analysis was restricted to, a level of a factor comparing regions, or the signal a model term carries. Declared once and referenced from each, because the same seed written separately in each place is three spellings nothing can join. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier.""") name: ExtractedString = Field(default=..., description="""The region as the source names it -- aSCC, left dlPFC parcel, salience network. Its own words: normalization to an anatomical vocabulary happens downstream and needs the original to map from.""") definition_method: ExtractedRegionDefinition = Field(default=..., description="""How the region was delimited, which is a statement about independence: one drawn from the study's own prior contrast is not independent of an effect tested inside it. Per region, since one analysis may use an atlas mask and a functional ROI at once. Required, so a source that does not say still carries the slot, as `extraction_status: not_reported`. Open vocabulary: write the source's own wording in the field when nothing here fits, because the value is required and a closed range discards the whole region rather than the one unrecognised word. @@ -1826,6 +1911,19 @@ class Region(ConfiguredBaseModel): atlas: Optional[ExtractedString] = Field(default=None, description="""The atlas, parcellation or template it came from, as named -- AAL, WFU_Pickatlas, Schaefer 400 -- with version or parcel index when given. `definition_method: atlas` records that one was used, not which, which is not enough to pool two studies.""") description: Optional[ExtractedString] = Field(default=None, description="""How the source defines the region, including any geometry it states: \"8 mm spheres centered at [0 36 -6]\". Definitional coordinates have no structured slot and belong here, where the evidence span makes them checkable; result coordinates reach the record through the table parser instead.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class ExtractedRegionDefinition(ExtractedValue): """ @@ -1853,6 +1951,7 @@ class Acquisition(ConfiguredBaseModel): """ A modality-agnostic protocol used to acquire neuroimaging or associated physiological data, holding only what every modality has in common. One Acquisition carries one modality: simultaneous recordings such as EEG+fMRI are separate Acquisition records that a Task lists together. Everything modality-specific lives in the class the Modality value instantiates -- MRI, EEG, FNIRS, PET, or OtherModality. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) acquisition_type: Literal["Acquisition"] = Field(default="Acquisition", description="""The concrete subclass this record instantiates. Derived by the mapper from the modality value's `instantiates`, never extracted.""") modality: ExtractedModality = Field(default=..., description="""Imaging or recording modality. A value in the Modality vocabulary also names the class holding this acquisition's modality-specific parameters, through that value's `instantiates`. @@ -1863,6 +1962,19 @@ class Acquisition(ConfiguredBaseModel): It describes the volume behind the REPORTED results. A scan that clipped a structure in participants who were then excluded is `whole_brain`, because the analysed volume was not clipped; calling it `partial` would say the opposite of what the results rest on. A field of view or slice prescription in millimetres is not by itself a statement of coverage. Where only the geometry is given, the value is an inference and `value_source` records it as one. Which structure a partial volume missed is not a field of its own -- the evidence span carries that detail.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class MRI(Acquisition): """ @@ -1875,6 +1987,7 @@ class MRI(Acquisition): echo_time_seconds: Optional[ExtractedNumberList] = Field(default=None, description="""Echo time in seconds, as a list of one value per echo. A multi-echo sequence has several; a single-echo one has a list of length one.""") acquisition_voxel_size_mm: Optional[ExtractedNumberList] = Field(default=None, description="""Original acquisition voxel size [x, y, z] in millimeters, before interpolation or resampling.""") number_of_volumes: Optional[ExtractedInteger] = Field(default=None, description="""Number of acquired volumes contributing to the recording, when reported; a proxy for scan length and temporal sampling in functional acquisitions.""") + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) acquisition_type: Literal["MRI"] = Field(default="MRI", description="""The concrete subclass this record instantiates. Derived by the mapper from the modality value's `instantiates`, never extracted.""") modality: ExtractedModality = Field(default=..., description="""Imaging or recording modality. A value in the Modality vocabulary also names the class holding this acquisition's modality-specific parameters, through that value's `instantiates`. @@ -1885,6 +1998,19 @@ class MRI(Acquisition): It describes the volume behind the REPORTED results. A scan that clipped a structure in participants who were then excluded is `whole_brain`, because the analysed volume was not clipped; calling it `partial` would say the opposite of what the results rest on. A field of view or slice prescription in millimetres is not by itself a statement of coverage. Where only the geometry is given, the value is an inference and `value_source` records it as one. Which structure a partial volume missed is not a field of its own -- the evidence span carries that detail.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class EEG(Acquisition): """ @@ -1895,6 +2021,7 @@ class EEG(Acquisition): eeg_placement_scheme: Optional[ExtractedString] = Field(default=None, description="""Electrode placement system, such as the 10-20 or 10-10 system.""") recording_type: Optional[ExtractedString] = Field(default=None, description="""Recording organization, such as continuous, discontinuous, or epoched.""") eeg_channel_count: Optional[ExtractedInteger] = Field(default=None, description="""Number of simultaneously recorded EEG channels.""") + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) acquisition_type: Literal["EEG"] = Field(default="EEG", description="""The concrete subclass this record instantiates. Derived by the mapper from the modality value's `instantiates`, never extracted.""") modality: ExtractedModality = Field(default=..., description="""Imaging or recording modality. A value in the Modality vocabulary also names the class holding this acquisition's modality-specific parameters, through that value's `instantiates`. @@ -1905,6 +2032,19 @@ class EEG(Acquisition): It describes the volume behind the REPORTED results. A scan that clipped a structure in participants who were then excluded is `whole_brain`, because the analysed volume was not clipped; calling it `partial` would say the opposite of what the results rest on. A field of view or slice prescription in millimetres is not by itself a statement of coverage. Where only the geometry is given, the value is an inference and `value_source` records it as one. Which structure a partial volume missed is not a field of its own -- the evidence span carries that detail.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class FNIRS(Acquisition): """ @@ -1918,6 +2058,7 @@ class FNIRS(Acquisition): source_detector_distances_mm: Optional[ExtractedNumberList] = Field(default=None, description="""Reported source-detector separations in millimeters.""") nirs_placement_scheme: Optional[ExtractedStringList] = Field(default=None, description="""Standard placement system or listed optode positions, such as 10-20, Cz, or Pz.""") recorded_signal_types: Optional[ExtractedStringList] = Field(default=None, description="""Raw or derived channel signal types, such as continuous-wave amplitude, optical density, oxygenated hemoglobin, or deoxygenated hemoglobin.""") + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) acquisition_type: Literal["FNIRS"] = Field(default="FNIRS", description="""The concrete subclass this record instantiates. Derived by the mapper from the modality value's `instantiates`, never extracted.""") modality: ExtractedModality = Field(default=..., description="""Imaging or recording modality. A value in the Modality vocabulary also names the class holding this acquisition's modality-specific parameters, through that value's `instantiates`. @@ -1928,6 +2069,19 @@ class FNIRS(Acquisition): It describes the volume behind the REPORTED results. A scan that clipped a structure in participants who were then excluded is `whole_brain`, because the analysed volume was not clipped; calling it `partial` would say the opposite of what the results rest on. A field of view or slice prescription in millimetres is not by itself a statement of coverage. Where only the geometry is given, the value is an inference and `value_source` records it as one. Which structure a partial volume missed is not a field of its own -- the evidence span carries that detail.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class PET(Acquisition): """ @@ -1941,6 +2095,7 @@ class PET(Acquisition): uptake_time_seconds: Optional[ExtractedNumber] = Field(default=None, description="""Interval from tracer administration to the start of the analyzed scan window, in seconds.""") reconstructed_voxel_size_mm: Optional[ExtractedNumberList] = Field(default=None, description="""Reconstructed PET voxel size [x, y, z] in millimeters.""") spatial_resolution_mm: Optional[ExtractedNumberList] = Field(default=None, description="""Reported reconstructed spatial resolution, such as FWHM dimensions, in millimeters.""") + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) acquisition_type: Literal["PET"] = Field(default="PET", description="""The concrete subclass this record instantiates. Derived by the mapper from the modality value's `instantiates`, never extracted.""") modality: ExtractedModality = Field(default=..., description="""Imaging or recording modality. A value in the Modality vocabulary also names the class holding this acquisition's modality-specific parameters, through that value's `instantiates`. @@ -1951,6 +2106,19 @@ class PET(Acquisition): It describes the volume behind the REPORTED results. A scan that clipped a structure in participants who were then excluded is `whole_brain`, because the analysed volume was not clipped; calling it `partial` would say the opposite of what the results rest on. A field of view or slice prescription in millimetres is not by itself a statement of coverage. Where only the geometry is given, the value is an inference and `value_source` records it as one. Which structure a partial volume missed is not a field of its own -- the evidence span carries that detail.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class OtherModality(Acquisition): """ @@ -1958,6 +2126,7 @@ class OtherModality(Acquisition): """ modality_label: ExtractedString = Field(default=..., description="""The source's own name for the modality or recording technique, verbatim. Always populated, since the Modality vocabulary is coarser than what a source reports.""") reason_no_modality_class_used: Optional[ExtractedString] = Field(default=None, description="""Why this is not an MRI, EEG, FNIRS, or PET record.""") + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) acquisition_type: Literal["OtherModality"] = Field(default="OtherModality", description="""The concrete subclass this record instantiates. Derived by the mapper from the modality value's `instantiates`, never extracted.""") modality: ExtractedModality = Field(default=..., description="""Imaging or recording modality. A value in the Modality vocabulary also names the class holding this acquisition's modality-specific parameters, through that value's `instantiates`. @@ -1968,16 +2137,43 @@ class OtherModality(Acquisition): It describes the volume behind the REPORTED results. A scan that clipped a structure in participants who were then excluded is `whole_brain`, because the analysed volume was not clipped; calling it `partial` would say the opposite of what the results rest on. A field of view or slice prescription in millimetres is not by itself a statement of coverage. Where only the geometry is given, the value is an inference and `value_source` records it as one. Which structure a partial volume missed is not a field of its own -- the evidence span carries that detail.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class Device(ConfiguredBaseModel): """ The scanner, amplifier, or recording system an acquisition was made on, split into the two parts a source reliably reports. Either part can be missing: papers name a model with no manufacturer as often as the reverse. One record is one physical machine, referenced by every acquisition made on it: a study that collected functional, structural and diffusion scans in one session has three Acquisitions and one Device. Two records mean the study says it used two machines -- a second site, or a rescan on different hardware -- which is a claim about the data and not a way of recording that two sequences differed. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier used by Acquisition foreign-key references.""") manufacturer: Optional[ExtractedString] = Field(default=None, description="""Company that made the device, as reported, such as Siemens, GE, Philips, Brain Products, or NIRx.""") model: Optional[ExtractedString] = Field(default=None, description="""Model or product name of the device, as reported, such as Prisma, Signa, Achieva, or actiCHamp.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class ExtractedBrainCoverage(ExtractedValue): """ @@ -2006,6 +2202,7 @@ class Analysis(ConfiguredBaseModel): A distinct reported inferential map or tested effect, linking its source acquisitions, sample, paradigm context, tested effect, preprocessing, model estimation, inference settings, and source result tables. Create one Analysis for each distinct tested effect. Separate Analyses are required when normalized direction, participant-group comparison, method, the pattern of `effect.cells`, seed or connectivity model, decoded variable, component identity, or spatial_scope differ, and when an omnibus effect and its directional post-hoc contrast are both reported. ROI definitions, acquisition summaries, descriptive sample tables, and masks without inferential effects are not Analyses. Nor is a map reported with no inferential test at all -- an ICA or PCA component map presented descriptively, a connectivity matrix given without a test -- there being nothing for such a map to be the effect of. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=...) name: ExtractedString = Field(default=..., description="""Study-local label for the analysis or inferential result, in the source's words.""") definition: ExtractedString = Field(default=..., description="""Precise statement of the tested effect in the source's own words, minimally cleaned. Always source-grounded. For an interaction it must state the direction of the interaction as the source states it; for an ordered contrast it must keep the ordering that the cell directions cannot express.""") @@ -2037,6 +2234,19 @@ class Analysis(ConfiguredBaseModel): mirror_of: Optional[str] = Field(default=None, description="""Present only on an analysis the builder derived by reversing another one, and naming the analysis it reverses. A results table with both signs in one column reports one contrast read two ways. The paper's prose describes one direction, so only that half is extracted and the other is rebuilt arithmetically -- same cells, directions flipped. This slot is what says so. Without it the two halves look like independent findings, and a meta-analysis that counts both counts one sample twice.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class Effect(ConfiguredBaseModel): """ @@ -2073,6 +2283,7 @@ class Measure(ConfiguredBaseModel): One record may be referenced by several Analyses, so every field must hold for all of them; an analysis of a different quantity takes its own record. The test is mechanical: two analyses share a record when `family`, `type`, `specific_metric` and `unit` all match. A study measures one or two quantities and tests them many ways, so sharing is the common case -- eight analyses of one connectivity measure are one record, not eight. Not a property of the thresholding scheme: one correction scheme routinely covers several measures, and one measure is often tested under several schemes. See storage-schema-design-notes.md for the reasoning behind this shape. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier used by Analysis foreign-key references.""") family: ExtractedMeasureFamily = Field(default=..., description="""Broad imaging or signal family, for coarse filtering.""") type: ExtractedMeasureType = Field(default=..., description="""Normalized quantity, for cross-study queries.""") @@ -2080,6 +2291,19 @@ class Measure(ConfiguredBaseModel): specific_metric: Optional[ExtractedString] = Field(default=None, description="""Named metric when type is intentionally broad, such as fractional anisotropy, ALFF, fALFF, or a specific tracer binding potential.""") unit: Optional[ExtractedString] = Field(default=None, description="""The unit the measure is reported in, when the source gives one and it carries information -- mm for a thickness, mL for a volume. Omitted for a unitless quantity such as a correlation or a normalized index.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class Mediation(ConfiguredBaseModel): """ @@ -2115,12 +2339,26 @@ class Preprocessing(ConfiguredBaseModel): """ An ordered preprocessing pipeline applied to acquired data before or as part of statistical modeling. This can be expanded with COBIDAS. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier used by Analysis foreign-key references.""") description: Optional[ExtractedString] = Field(default=None, description="""Narrative overview of the pipeline, its inputs, outputs, and intended purpose.""") software: Optional[ExtractedStringList] = Field(default=None, description="""Software packages or methods used, including version and revision identifiers.""") steps: Optional[ExtractedStringList] = Field(default=None, description="""Ordered free-text preprocessing operation names, listed in execution order; examples include slice-timing correction, motion correction, distortion correction, spatial normalization, and smoothing. The smoothing kernel itself is `smoothing_fwhm_mm` rather than a phrase in this list, because it is the one preprocessing parameter a synthesis filters on.""") smoothing_fwhm_mm: Optional[ExtractedNumberList] = Field(default=None, description="""Full width at half maximum of the applied spatial smoothing kernel, in millimetres: a list of one value for an isotropic kernel and of three for an anisotropic [x, y, z] one. Record 0 when the source states that no smoothing was applied.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class ModelEstimation(ConfiguredBaseModel): """ @@ -2130,6 +2368,7 @@ class ModelEstimation(ConfiguredBaseModel): The test for whether two analyses share a record is mechanical: list the columns each was fitted with. A different dependent variable, a different covariate set, or a different subset of participants means a different design matrix and therefore a separate record, even when family, estimator and software are identical. Two BPM regressions of the same measure on the same regressor, one over the patients and one over patients and controls pooled, are two records. Sharing one record across matrices corrupts the derived adjustment set. The adjustment set is the model's terms minus the terms named by its cells. If several matrices share one record, they claim adjustments that their original models never made, and the record cannot expose the error. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier used by Analysis foreign-key references.""") model_family: ExtractedModelFamily = Field(default=..., description="""Normalized statistical model family, which is the queryable form. Required, so a paper that never says still carries the slot, as `extraction_status: not_reported`. A method the vocabulary cannot name -- ICA, representational similarity, a decoder -- is written here as free text; the Analysis.details subclass also identifies the method.""") model_type: ExtractedString = Field(default=..., description="""The source's own wording for the model, minimally cleaned, such as mass- univariate GLM, meta-analysis, multivariate, predictive model, ICA, or representational similarity analysis.""") @@ -2152,6 +2391,19 @@ class ModelEstimation(ConfiguredBaseModel): May be empty, and legitimately is for a stage that fits only an intercept: a one-sample group test over first-level contrast images has no column of its own. Do not invent an intercept term to fill it. Analyses whose design matrices differ take their own records rather than sharing one.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class ModelTerm(ConfiguredBaseModel): """ @@ -2159,6 +2411,7 @@ class ModelTerm(ConfiguredBaseModel): A term is something the measure was regressed *on*. The quantity being modelled is `Analysis.measure` and never a term of the model that models it. A table laid out with one row-block per measured parameter -- four diffusion parameters, three frequency bands -- is reporting one analysis per parameter over one measure each, not one analysis over a factor whose levels are the parameters. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier, referenced by an Effect's `cells` and its `mediation.mediator`, and by `interaction_with`.""") name: ExtractedString = Field(default=..., description="""Term label, unique within the model: a name appearing twice is one term recorded twice. For a categorical term this is the factor's identity. Often a label the source never writes. A paper reporting 1-back, 2-back and 3-back conditions states the levels and states that they vary together, but frequently names no axis; `load` is then a name supplied for a grouping the source does make. That is permitted, within two limits: the levels must be source-stated, and the name must not assert a comparison or a construct the paper does not.""") @@ -2191,6 +2444,19 @@ class ModelTerm(ConfiguredBaseModel): **Required for a derived column, whatever its `assessment` says.** A change score names its instrument in `assessment` and its derivation here, in the source's own words and including the occasions it spans -- \"percent reduction in BDI, (post-pre)/pre, from baseline to post-treatment\". Nothing else can carry those occasions: `FactorLevel.timepoints` names the occasions a factor *compares*, and a derived column has no levels to hang them on, so where a study has several post-intervention occasions this is the only thing separating a change to the endpoint from a change to a later follow-up. Not the shape of the fitted relationship, which is `functional_form`: a percent change entered linearly is a linear term computed from a difference, so recording the construction as the form would lose whether the fit was linear or quadratic. A transform *of the measure* the vocabulary cannot name still goes in functional_form.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class FactorLevel(ConfiguredBaseModel): """ @@ -2214,6 +2480,7 @@ class InferenceSettings(ConfiguredBaseModel): One record may be referenced by several Analyses, so every field must hold for all of them; an analysis thresholded or corrected differently takes its own record. The test is mechanical: two analyses share a record when the threshold, its type, the correction method, the scope corrected over and the alpha are all the same. A paper that states one correction scheme in Methods and applies it throughout has one record. Inference attaches to the analysis, not to the fit: one model estimated once is often thresholded several ways -- an ROI test Bonferroni-corrected over four regions and a searchlight over the same fit corrected by FDR. See storage-schema-design-notes.md for the reasoning behind this shape. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier used by Analysis foreign-key references.""") inference_level: Optional[ExtractedString] = Field(default=None, description="""Level at which inference is made, such as voxel, vertex, cluster, region, network or other.""") height_threshold_value: Optional[ExtractedNumber] = Field(default=None, description="""The threshold applied to each element's statistic, before correction and before any cluster-level test. Papers name it for whichever of its two jobs they are describing -- the *voxelwise* (or vertexwise) threshold when elements are tested individually, the *cluster-forming* threshold when surviving elements are grouped into clusters -- and it is one number either way, applied once to the map. Record it once here, whichever word the source uses. @@ -2234,11 +2501,25 @@ class InferenceSettings(ConfiguredBaseModel): alpha_level: Optional[ExtractedNumber] = Field(default=None, description="""Family-wise, false-discovery, or nominal alpha threshold when reported.""") number_of_tests: Optional[ExtractedInteger] = Field(default=None, description="""Number of statistical tests considered by the multiple-comparison procedure.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class Table(ConfiguredBaseModel): """ A publication table that supplies coordinates, statistics, or contextual metadata. Raw cell structure and point-level provenance may remain in the repository's coordinate-table representation and be linked with source_path. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable table identifier used by Analysis foreign-key references.""") table_number: Optional[ExtractedString] = Field(default=None, description="""Printed identifier such as Table 2 or Supplementary Table S4.""") title: Optional[ExtractedString] = Field(default=None, description="""Table title, when distinct from the caption.""") @@ -2250,12 +2531,26 @@ class Table(ConfiguredBaseModel): A coordinate table is not an analysis merely by having coordinates in it, and this is the only field that says which it is. Absence means nothing decided: a table deliberately encoded as a non-analysis and one no pass considered are otherwise the same silence, and the coordinates of the first belong on the entity they locate (a Region's description) rather than on a contrast that never produced them. A per-table answer, so it cannot be right for a table whose row groups serve different purposes -- results beside the ROIs they were tested in. Each row group states its own role in a `CoordinateSet`.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class CoordinateSet(ConfiguredBaseModel): """ A group of coordinates reported together -- a row group of a table, or a run of sentences in the text -- and what it is for. The coordinates stay in the parse, as for tables; this records their role and links them to what they belong to: the analysis a result reports, the regions an anchor locates. A set has one role: its dominant one. The groups come from the table parse, which splits a table by what was tested rather than by what each row is for, so a set can mix roles -- a connectivity results table that lists its own seed as a row. Such a set keeps the role of most of its points, and the odd point takes that role with it. This is a known error, about 1.3% of sets, accepted rather than splitting the parse's groups. """ + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""The key the coordinate parse gave this set, `ParsedAnalysis.key` in neuroimaging-paper-parse: `analysis#`, the same key `Analysis.source_table_analysis` names, where `h` hashes the set's name, space and points, so the key survives a re-parse that extracts the same set.""") role: Optional[ExtractedCoordinateRole] = Field(default=None, description="""What these coordinates are for.""") result_kind: Optional[ExtractedResultKind] = Field(default=None, description="""For a `result`, the kind of analysis that produced it. Absent for every other role.""") @@ -2264,12 +2559,26 @@ class CoordinateSet(ConfiguredBaseModel): regions: Optional[list[str]] = Field(default=None, description="""For an `anchor`, the regions these coordinates locate.""") description: Optional[ExtractedString] = Field(default=None, description="""What the set is, in the source's words, when the role alone does not say.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class Study(ConfiguredBaseModel): """ The publication-level record that contains the groups, tasks, design, acquisitions, preprocessing pipelines, model estimations, analyses, and reported result tables belonging to one study. Conditions belong to the task they are part of, not to the study. """ extraction_metadata: ExtractionMetadata = Field(default=..., description="""Provenance for this extraction record.""") + stable_id: Optional[str] = Field(default=None, description="""UUID minted by the store the first time this entity appears, never by an extractor, and kept across re-runs so a later extraction matches the same entity.""") local_id: str = Field(default=..., description="""Stable identifier for the study record, such as a DOI-based or repository-local identifier.""") description: Optional[ExtractedString] = Field(default=None, description="""Brief free-text summary of the study aims and scope.""") hypothesis: Optional[ExtractedStringList] = Field(default=None, description="""The study's own statement of what it expected to find, quoted from the paper -- usually the closing paragraph of the Introduction, in the form \"we hypothesized that\" or \"we predicted that\". Verbatim but for citation markers, one entry per prediction, so a study stating several gets several. @@ -2295,6 +2604,19 @@ class Study(ConfiguredBaseModel): language: Optional[list[str]] = Field(default=None, description="""Languages of this article, copied from the parsed paper's `Bibliography.language` alongside `study_type`, not read off the paper: three-letter codes, `eng` and the rest.""") study_type: Optional[list[str]] = Field(default=None, description="""PubMed publication types for this record, verbatim, copied from the parsed paper's `Bibliography.publication_types`, not read off the paper: it is what the indexers said about the article, which is what a publication-type exclusion asks about.""") + @field_validator('stable_id') + def pattern_stable_id(cls, v): + pattern=re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + if isinstance(v, list): + for element in v: + if isinstance(element, str) and not pattern.search(element): + err_msg = f"Invalid stable_id format: {element}" + raise ValueError(err_msg) + elif isinstance(v, str) and not pattern.search(v): + err_msg = f"Invalid stable_id format: {v}" + raise ValueError(err_msg) + return v + class ExternalDataset(ConfiguredBaseModel): """ diff --git a/tests/test_stable_id.py b/tests/test_stable_id.py new file mode 100644 index 0000000..519cc56 --- /dev/null +++ b/tests/test_stable_id.py @@ -0,0 +1,72 @@ +"""Every extraction entity carries an optional stable_id, a UUID the store mints.""" + +import pytest +from pydantic import BaseModel, ValidationError + +from conftest import load +from study_schema import jsonschema +from study_schema.models import extraction + +STABLE_ID = "3f2b8c1e-9a4d-4e7b-8c2a-1d5e6f7a8b90" + + +def _entity_classes(): + return [ + name + for name, model in vars(extraction).items() + if isinstance(model, type) + and issubclass(model, BaseModel) + and "local_id" in model.model_fields + ] + + +def test_record_with_stable_id_on_its_entity_validates(example_paths): + validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") + record = load(example_paths["record"]) + record["stable_id"] = STABLE_ID + + extraction.Study.model_validate(record) + validator.validate(record, jsonschema.load("extraction-record")) + + +def test_record_without_stable_id_still_validates(example_paths): + validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") + record = load(example_paths["record"]) + record.pop("stable_id") + + extraction.Study.model_validate(record) + validator.validate(record, jsonschema.load("extraction-record")) + + +@pytest.mark.parametrize( + "value", + [ + "not-a-uuid", + "3f2b8c1e9a4d4e7b8c2a1d5e6f7a8b90", + "3F2B8C1E-9A4D-4E7B-8C2A-1D5E6F7A8B90", + "3F2B8C1E-9A4D-4E7B-8C2A-1D5E6F7A8B90x", + ], +) +def test_malformed_stable_id_is_refused(value, example_paths): + validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") + record = load(example_paths["record"]) + record["stable_id"] = value + + with pytest.raises(ValidationError, match="Invalid stable_id format"): + extraction.Study.model_validate(record) + with pytest.raises(validator.ValidationError, match="does not match"): + validator.validate(record, jsonschema.load("extraction-record")) + + +def test_each_entity_class_has_the_slot_in_its_json_schema(): + document = jsonschema.load("extraction-record") + definitions = dict(document.get("$defs", {})) + definitions.setdefault(document["title"], document) + classes = _entity_classes() + + assert len(classes) > 1 + for name in classes: + properties = definitions[name]["properties"] + assert "stable_id" in properties, name + assert "pattern" in properties["stable_id"], name + assert "string" in properties["stable_id"]["type"], name