diff --git a/README.md b/README.md index d8161db..87236fe 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,11 @@ What each side may rely on: - **One shape for every source.** PMC, Europe PMC, pubget, Elsevier, ACE and PDF all emit a `ParsedPaper`, so a consumer never branches on where a paper came from. +- **A preprint is not its version of record.** The header's `doi`, `pmid` and `pmcid` name + the version of record. A preprint server's DOIs, each with its `PreprintServer`, and + PubMed's and PMC's records of the preprint are kept beside them in `preprint_dois`, + `preprint_pmid` and `preprint_pmcid`. `publication_status` states which the paper has + (`published` or `preprint`), and pondie copies it to `Study.publication_status`. - **Bibliography is copied, never extracted.** pondie fills the record's `Study.title`, `authors`, `doi`, `journal`, `publication_year`, `language` and `study_type` from `ParsedPaper.bibliography` and the header's identifiers (`transform: copy` in @@ -320,7 +325,10 @@ so ingestion, pondie and neurostore read "Talairach & Tournoux 1988" the same wa **Examples.** `examples/` holds one paper and one run: a table whose ingestion parse is a single mixed-sign analysis, and pondie's revision that splits it by sign and adds the null result the text states. The tests read them through every model, JSON Schema and layout, and -the checks must find nothing wrong with them. +the checks must find nothing wrong with them. `examples/identifiers/` holds header +identifiers for a preprint-only paper and for a published paper with a preprint DOI, and +under `invalid/` two the models refuse: one without `publication_status`, one with a preprint +DOI that names no server. **Storing coordinates.** The models say what a parse means, not how a corpus of them is kept. `study_schema.parquet` stores coordinate parses as three Parquet tables -- `parses`, diff --git a/examples/corpus/x7Ex4mpleA01/parse/coordinate_parse.json b/examples/corpus/x7Ex4mpleA01/parse/coordinate_parse.json index 24a8436..54becbd 100644 --- a/examples/corpus/x7Ex4mpleA01/parse/coordinate_parse.json +++ b/examples/corpus/x7Ex4mpleA01/parse/coordinate_parse.json @@ -1,12 +1,16 @@ { "header": { "artifact_kind": "coordinate_parse", - "schema_version": "0.6.0", + "schema_version": "0.7.0", "article_id": "art-000001", "identifiers": { "pmid": "00000000", "pmcid": null, "doi": null, + "preprint_dois": null, + "preprint_pmid": null, + "preprint_pmcid": null, + "publication_status": "published", "neurostore_base_study_id": "x7Ex4mpleA01" }, "producer": { diff --git a/examples/corpus/x7Ex4mpleA01/parse/parsed_paper.json b/examples/corpus/x7Ex4mpleA01/parse/parsed_paper.json index 09e1e33..dd66844 100644 --- a/examples/corpus/x7Ex4mpleA01/parse/parsed_paper.json +++ b/examples/corpus/x7Ex4mpleA01/parse/parsed_paper.json @@ -1,12 +1,16 @@ { "header": { "artifact_kind": "parsed_paper", - "schema_version": "0.6.0", + "schema_version": "0.7.0", "article_id": "art-000001", "identifiers": { "pmid": "00000000", "pmcid": null, "doi": null, + "preprint_dois": null, + "preprint_pmid": null, + "preprint_pmcid": null, + "publication_status": "published", "neurostore_base_study_id": "x7Ex4mpleA01" }, "producer": { diff --git a/examples/identifiers/invalid/missing_publication_status.json b/examples/identifiers/invalid/missing_publication_status.json new file mode 100644 index 0000000..deef6cc --- /dev/null +++ b/examples/identifiers/invalid/missing_publication_status.json @@ -0,0 +1,14 @@ +{ + "pmid": "31863619", + "pmcid": null, + "doi": "10.1111/desc.12933", + "preprint_dois": [ + { + "doi": "10.31234/osf.io/5m87x", + "server": "psyarxiv" + } + ], + "preprint_pmid": null, + "preprint_pmcid": null, + "neurostore_base_study_id": null +} diff --git a/examples/identifiers/invalid/preprint_doi_without_server.json b/examples/identifiers/invalid/preprint_doi_without_server.json new file mode 100644 index 0000000..fdd3c95 --- /dev/null +++ b/examples/identifiers/invalid/preprint_doi_without_server.json @@ -0,0 +1,14 @@ +{ + "pmid": "31863619", + "pmcid": null, + "doi": "10.1111/desc.12933", + "preprint_dois": [ + { + "doi": "10.31234/osf.io/5m87x" + } + ], + "preprint_pmid": null, + "preprint_pmcid": null, + "publication_status": "published", + "neurostore_base_study_id": null +} diff --git a/examples/identifiers/preprint_only.json b/examples/identifiers/preprint_only.json new file mode 100644 index 0000000..d9c5c01 --- /dev/null +++ b/examples/identifiers/preprint_only.json @@ -0,0 +1,15 @@ +{ + "pmid": null, + "pmcid": null, + "doi": null, + "preprint_dois": [ + { + "doi": "10.1101/2025.05.13.653896", + "server": "biorxiv" + } + ], + "preprint_pmid": "40462995", + "preprint_pmcid": "PMC12132522", + "publication_status": "preprint", + "neurostore_base_study_id": null +} diff --git a/examples/identifiers/published_with_preprint.json b/examples/identifiers/published_with_preprint.json new file mode 100644 index 0000000..58fb921 --- /dev/null +++ b/examples/identifiers/published_with_preprint.json @@ -0,0 +1,15 @@ +{ + "pmid": "31863619", + "pmcid": null, + "doi": "10.1111/desc.12933", + "preprint_dois": [ + { + "doi": "10.31234/osf.io/5m87x", + "server": "psyarxiv" + } + ], + "preprint_pmid": null, + "preprint_pmcid": null, + "publication_status": "published", + "neurostore_base_study_id": null +} diff --git a/examples/run/revisions/x7Ex4mpleA01.coordinate_parse.json b/examples/run/revisions/x7Ex4mpleA01.coordinate_parse.json index 82e6add..398e4f4 100644 --- a/examples/run/revisions/x7Ex4mpleA01.coordinate_parse.json +++ b/examples/run/revisions/x7Ex4mpleA01.coordinate_parse.json @@ -1,12 +1,16 @@ { "header": { "artifact_kind": "coordinate_parse", - "schema_version": "0.6.0", + "schema_version": "0.7.0", "article_id": "art-000001", "identifiers": { "pmid": "00000000", "pmcid": null, "doi": null, + "preprint_dois": null, + "preprint_pmid": null, + "preprint_pmcid": null, + "publication_status": "published", "neurostore_base_study_id": "x7Ex4mpleA01" }, "producer": { diff --git a/src/study_schema/jsonschema/coordinate-parse.schema.json b/src/study_schema/jsonschema/coordinate-parse.schema.json index 30ab5a3..0f90fce 100644 --- a/src/study_schema/jsonschema/coordinate-parse.schema.json +++ b/src/study_schema/jsonschema/coordinate-parse.schema.json @@ -93,9 +93,10 @@ }, "ArticleIdentifiers": { "additionalProperties": false, - "description": "Every identifier known for the paper when the artifact was written. Carried whole, so no consumer has to choose between them or look one up.", + "description": "Every identifier known for the paper when the artifact was written. Carried whole, so no consumer has to choose between them or look one up. `doi`, `pmid` and `pmcid` name the version of record; a preprint's identifiers are kept beside them in the `preprint_` fields, so a paper and its preprint are one article.", "properties": { "doi": { + "description": "The DOI of the version of record. Never a preprint server's DOI, which is in `preprint_dois`.", "type": [ "string", "null" @@ -109,18 +110,51 @@ ] }, "pmcid": { + "description": "PubMed Central's copy of the version of record. Never PMC's copy of a preprint, which is `preprint_pmcid`.", "type": [ "string", "null" ] }, "pmid": { + "description": "PubMed's record of the version of record. Never PubMed's record of a preprint, which is `preprint_pmid`.", "type": [ "string", "null" ] + }, + "preprint_dois": { + "description": "Every DOI a preprint server registered for the paper. A server that registers each version separately, such as Research Square's `/v2`, gives one entry per version, each kept as registered.", + "items": { + "$ref": "#/$defs/PreprintDoi" + }, + "type": [ + "array", + "null" + ] + }, + "preprint_pmcid": { + "description": "PubMed Central's copy of the preprint.", + "type": [ + "string", + "null" + ] + }, + "preprint_pmid": { + "description": "PubMed's record of the preprint, which PubMed types `Preprint`.", + "type": [ + "string", + "null" + ] + }, + "publication_status": { + "$ref": "#/$defs/PublicationStatus", + "description": "`preprint` when a `preprint_` field is set and `doi`, `pmid` and `pmcid` are all empty, `published` otherwise. The producer states it so that no consumer re-derives it." } }, + "required": [ + "publication_status" + ], "title": "ArticleIdentifiers", "type": "object" }, @@ -1430,6 +1464,63 @@ "title": "PolaritySemantics", "type": "string" }, + "PreprintDoi": { + "additionalProperties": false, + "description": "A DOI a preprint server registered, with the server that registered it.", + "properties": { + "doi": { + "type": "string" + }, + "server": { + "$ref": "#/$defs/PreprintServer", + "description": "Named by the producer from the DOI, so that no consumer needs a table of preprint servers' DOI patterns." + } + }, + "required": [ + "doi", + "server" + ], + "title": "PreprintDoi", + "type": "object" + }, + "PreprintServer": { + "description": "The preprint server that registered a DOI. A server hosted on the Open Science Framework is named by its provider, which has a DOI prefix of its own. A server is added here when a producer learns to recognise its DOIs.", + "enum": [ + "biorxiv", + "medrxiv", + "arxiv", + "researchsquare", + "osf_preprints", + "arabixiv", + "metaarxiv", + "engrxiv", + "focus_archive", + "frenxiv", + "ina_rxiv", + "lawarxiv", + "lis_scholarship_archive", + "marxiv", + "mindrxiv", + "nutrixiv", + "paleorxiv", + "psyarxiv", + "socarxiv", + "sportrxiv", + "africarxiv", + "mediarxiv", + "bodoarxiv", + "edarxiv", + "biohackrxiv", + "ssrn", + "preprints_org", + "authorea", + "techrxiv", + "chemrxiv", + "jmir_preprints" + ], + "title": "PreprintServer", + "type": "string" + }, "Prespecification": { "description": "Whether a contrast was planned before the data were seen. A preregistered primary contrast and a post-hoc sweep from the same paper are not equivalent evidence, and a synthesis that cannot tell them apart cannot address selective reporting.", "enum": [ @@ -1479,6 +1570,15 @@ "title": "Producer", "type": "object" }, + "PublicationStatus": { + "description": "Whether a paper's version of record is known, or only its preprint.", + "enum": [ + "published", + "preprint" + ], + "title": "PublicationStatus", + "type": "string" + }, "Reference": { "additionalProperties": false, "description": "One entry of a paper's reference list.", @@ -2192,5 +2292,5 @@ ], "title": "neuroimaging-paper-parse", "type": "object", - "version": "0.6.0" + "version": "0.7.0" } diff --git a/src/study_schema/jsonschema/parsed-paper.schema.json b/src/study_schema/jsonschema/parsed-paper.schema.json index fbb00ea..1e642d3 100644 --- a/src/study_schema/jsonschema/parsed-paper.schema.json +++ b/src/study_schema/jsonschema/parsed-paper.schema.json @@ -93,9 +93,10 @@ }, "ArticleIdentifiers": { "additionalProperties": false, - "description": "Every identifier known for the paper when the artifact was written. Carried whole, so no consumer has to choose between them or look one up.", + "description": "Every identifier known for the paper when the artifact was written. Carried whole, so no consumer has to choose between them or look one up. `doi`, `pmid` and `pmcid` name the version of record; a preprint's identifiers are kept beside them in the `preprint_` fields, so a paper and its preprint are one article.", "properties": { "doi": { + "description": "The DOI of the version of record. Never a preprint server's DOI, which is in `preprint_dois`.", "type": [ "string", "null" @@ -109,18 +110,51 @@ ] }, "pmcid": { + "description": "PubMed Central's copy of the version of record. Never PMC's copy of a preprint, which is `preprint_pmcid`.", "type": [ "string", "null" ] }, "pmid": { + "description": "PubMed's record of the version of record. Never PubMed's record of a preprint, which is `preprint_pmid`.", "type": [ "string", "null" ] + }, + "preprint_dois": { + "description": "Every DOI a preprint server registered for the paper. A server that registers each version separately, such as Research Square's `/v2`, gives one entry per version, each kept as registered.", + "items": { + "$ref": "#/$defs/PreprintDoi" + }, + "type": [ + "array", + "null" + ] + }, + "preprint_pmcid": { + "description": "PubMed Central's copy of the preprint.", + "type": [ + "string", + "null" + ] + }, + "preprint_pmid": { + "description": "PubMed's record of the preprint, which PubMed types `Preprint`.", + "type": [ + "string", + "null" + ] + }, + "publication_status": { + "$ref": "#/$defs/PublicationStatus", + "description": "`preprint` when a `preprint_` field is set and `doi`, `pmid` and `pmcid` are all empty, `published` otherwise. The producer states it so that no consumer re-derives it." } }, + "required": [ + "publication_status" + ], "title": "ArticleIdentifiers", "type": "object" }, @@ -1430,6 +1464,63 @@ "title": "PolaritySemantics", "type": "string" }, + "PreprintDoi": { + "additionalProperties": false, + "description": "A DOI a preprint server registered, with the server that registered it.", + "properties": { + "doi": { + "type": "string" + }, + "server": { + "$ref": "#/$defs/PreprintServer", + "description": "Named by the producer from the DOI, so that no consumer needs a table of preprint servers' DOI patterns." + } + }, + "required": [ + "doi", + "server" + ], + "title": "PreprintDoi", + "type": "object" + }, + "PreprintServer": { + "description": "The preprint server that registered a DOI. A server hosted on the Open Science Framework is named by its provider, which has a DOI prefix of its own. A server is added here when a producer learns to recognise its DOIs.", + "enum": [ + "biorxiv", + "medrxiv", + "arxiv", + "researchsquare", + "osf_preprints", + "arabixiv", + "metaarxiv", + "engrxiv", + "focus_archive", + "frenxiv", + "ina_rxiv", + "lawarxiv", + "lis_scholarship_archive", + "marxiv", + "mindrxiv", + "nutrixiv", + "paleorxiv", + "psyarxiv", + "socarxiv", + "sportrxiv", + "africarxiv", + "mediarxiv", + "bodoarxiv", + "edarxiv", + "biohackrxiv", + "ssrn", + "preprints_org", + "authorea", + "techrxiv", + "chemrxiv", + "jmir_preprints" + ], + "title": "PreprintServer", + "type": "string" + }, "Prespecification": { "description": "Whether a contrast was planned before the data were seen. A preregistered primary contrast and a post-hoc sweep from the same paper are not equivalent evidence, and a synthesis that cannot tell them apart cannot address selective reporting.", "enum": [ @@ -1479,6 +1570,15 @@ "title": "Producer", "type": "object" }, + "PublicationStatus": { + "description": "Whether a paper's version of record is known, or only its preprint.", + "enum": [ + "published", + "preprint" + ], + "title": "PublicationStatus", + "type": "string" + }, "Reference": { "additionalProperties": false, "description": "One entry of a paper's reference list.", @@ -2227,5 +2327,5 @@ ], "title": "neuroimaging-paper-parse", "type": "object", - "version": "0.6.0" + "version": "0.7.0" } diff --git a/src/study_schema/jsonschema/storage-study.schema.json b/src/study_schema/jsonschema/storage-study.schema.json index 25671a2..d0dd44c 100644 --- a/src/study_schema/jsonschema/storage-study.schema.json +++ b/src/study_schema/jsonschema/storage-study.schema.json @@ -3309,6 +3309,15 @@ "title": "Prespecification", "type": "string" }, + "PublicationStatus": { + "description": "Whether a paper's version of record is known, or only its preprint.", + "enum": [ + "published", + "preprint" + ], + "title": "PublicationStatus", + "type": "string" + }, "Region": { "additionalProperties": false, "description": "A brain region the study delimited and used: a connectivity seed or target, the search space an analysis was restricted to, a level of a factor comparing regions, or the signal a model term carries. Declared once and referenced from each, because the same seed written separately in each place is three spellings nothing can join.", @@ -3778,7 +3787,7 @@ ] }, "doi": { - "description": "Digital Object Identifier, without requiring a particular URL form.", + "description": "Digital Object Identifier of the version of record, without requiring a particular URL form. Never a preprint server's DOI.", "type": [ "string", "null" @@ -3886,6 +3895,17 @@ "null" ] }, + "publication_status": { + "anyOf": [ + { + "$ref": "#/$defs/PublicationStatus" + }, + { + "type": "null" + } + ], + "description": "Whether the version of record is known or only the preprint, copied from the parsed paper's header identifiers. `study_type` is independent of it: it keeps PubMed's publication types verbatim, `Preprint` included." + }, "publication_year": { "description": "Year in which the study was first published or made publicly available.", "minimum": 1600, @@ -4577,7 +4597,7 @@ ] }, "doi": { - "description": "Digital Object Identifier, without requiring a particular URL form.", + "description": "Digital Object Identifier of the version of record, without requiring a particular URL form. Never a preprint server's DOI.", "type": [ "string", "null" @@ -4685,6 +4705,17 @@ "null" ] }, + "publication_status": { + "anyOf": [ + { + "$ref": "#/$defs/PublicationStatus" + }, + { + "type": "null" + } + ], + "description": "Whether the version of record is known or only the preprint, copied from the parsed paper's header identifiers. `study_type` is independent of it: it keeps PubMed's publication types verbatim, `Preprint` included." + }, "publication_year": { "description": "Year in which the study was first published or made publicly available.", "minimum": 1600, @@ -4760,5 +4791,5 @@ ], "title": "neuroimaging-study-storage", "type": "object", - "version": "0.7.0" + "version": "0.8.0" } diff --git a/src/study_schema/linkml/extraction-to-storage.map.yaml b/src/study_schema/linkml/extraction-to-storage.map.yaml index 18808d5..11b2882 100644 --- a/src/study_schema/linkml/extraction-to-storage.map.yaml +++ b/src/study_schema/linkml/extraction-to-storage.map.yaml @@ -135,6 +135,13 @@ derivations: notes: >- DOI from the parsed paper's header identifiers. Not extracted: a bibliographic fact the metadata lookup states is not one a model should be asked to read off the page. + Study.publication_status: + transform: copy + source_artifact: neuroimaging-paper-parse/ParsedPaper + source_field: ArticleIdentifiers.publication_status + notes: >- + Stated by the producer of the parsed paper from the identifiers it holds. Not + extracted: whether a version of record exists is not something the text says. Study.id: transform: generate notes: Assigned by the ingestion pipeline; typically DOI-derived (doi:10.../record) or a UUID. diff --git a/src/study_schema/linkml/neuroimaging-paper-parse.yaml b/src/study_schema/linkml/neuroimaging-paper-parse.yaml index a0e0b57..197170a 100644 --- a/src/study_schema/linkml/neuroimaging-paper-parse.yaml +++ b/src/study_schema/linkml/neuroimaging-paper-parse.yaml @@ -29,7 +29,7 @@ description: >- record so the stored analyses match the analyses the record describes. license: CC0-1.0 -version: 0.6.0 +version: 0.7.0 prefixes: linkml: https://w3id.org/linkml/ diff --git a/src/study_schema/linkml/neuroimaging-paper-parse/header.yaml b/src/study_schema/linkml/neuroimaging-paper-parse/header.yaml index 4abfab6..6380b54 100644 --- a/src/study_schema/linkml/neuroimaging-paper-parse/header.yaml +++ b/src/study_schema/linkml/neuroimaging-paper-parse/header.yaml @@ -11,6 +11,7 @@ prefixes: imports: - linkml:types - parse_enums + - ../neuroimaging-study-storage/analysis_enums default_prefix: parse default_range: string @@ -51,18 +52,63 @@ classes: ArticleIdentifiers: description: >- Every identifier known for the paper when the artifact was written. Carried whole, so - no consumer has to choose between them or look one up. + no consumer has to choose between them or look one up. `doi`, `pmid` and `pmcid` name + the version of record; a preprint's identifiers are kept beside them in the + `preprint_` fields, so a paper and its preprint are one article. attributes: pmid: range: string + description: >- + PubMed's record of the version of record. Never PubMed's record of a preprint, + which is `preprint_pmid`. pmcid: range: string + description: >- + PubMed Central's copy of the version of record. Never PMC's copy of a preprint, + which is `preprint_pmcid`. doi: range: string + description: >- + The DOI of the version of record. Never a preprint server's DOI, which is in + `preprint_dois`. + preprint_dois: + range: PreprintDoi + multivalued: true + inlined_as_list: true + description: >- + Every DOI a preprint server registered for the paper. A server that registers each + version separately, such as Research Square's `/v2`, gives one entry per version, + each kept as registered. + preprint_pmid: + range: string + description: PubMed's record of the preprint, which PubMed types `Preprint`. + preprint_pmcid: + range: string + description: PubMed Central's copy of the preprint. + publication_status: + range: PublicationStatus + required: true + description: >- + `preprint` when a `preprint_` field is set and `doi`, `pmid` and `pmcid` are all + empty, `published` otherwise. The producer states it so that no consumer re-derives + it. neurostore_base_study_id: range: string description: Set once the paper has been uploaded; neurostore's BaseStudy id. + PreprintDoi: + description: A DOI a preprint server registered, with the server that registered it. + attributes: + doi: + range: string + required: true + server: + range: PreprintServer + required: true + description: >- + Named by the producer from the DOI, so that no consumer needs a table of preprint + servers' DOI patterns. + Producer: description: The program and stage that wrote an artifact. attributes: diff --git a/src/study_schema/linkml/neuroimaging-paper-parse/parse_enums.yaml b/src/study_schema/linkml/neuroimaging-paper-parse/parse_enums.yaml index 54d0294..2c5fd81 100644 --- a/src/study_schema/linkml/neuroimaging-paper-parse/parse_enums.yaml +++ b/src/study_schema/linkml/neuroimaging-paper-parse/parse_enums.yaml @@ -2,10 +2,10 @@ id: https://neurostuff.org/schemas/neuroimaging-paper-parse/parse_enums name: neuroimaging-paper-parse-enums title: Neuroimaging Paper Parse Vocabularies description: >- - The vocabularies of the parsed paper and the coordinate parse. Coordinate roles and - anchor kinds are not here: they are the storage schema's `CoordinateRole` and - `AnchorKind`, imported, so that the parse's role and the record's role are written - in the same words. + The vocabularies of the parsed paper and the coordinate parse. Coordinate roles, anchor + kinds and publication status are not here: they are the storage schema's + `CoordinateRole`, `AnchorKind` and `PublicationStatus`, imported, so that the parse and + the record write them in the same words. prefixes: linkml: https://w3id.org/linkml/ parse: https://neurostuff.org/schemas/neuroimaging-paper-parse/ @@ -44,6 +44,75 @@ enums: pdf: description: A PDF converted to text. + PreprintServer: + description: >- + The preprint server that registered a DOI. A server hosted on the Open Science + Framework is named by its provider, which has a DOI prefix of its own. A server is + added here when a producer learns to recognise its DOIs. + permissible_values: + biorxiv: + description: bioRxiv, DOI prefix 10.1101 and, since 2025-11, 10.64898 (openRxiv). + medrxiv: + description: medRxiv, DOI prefix 10.1101 and, since 2025-11, 10.64898 (openRxiv). + arxiv: + description: arXiv, DOI prefix 10.48550, the DOI registered for an arXiv id. + researchsquare: + description: Research Square, DOI prefix 10.21203. + osf_preprints: + description: OSF Preprints, DOI prefix 10.31219. + arabixiv: + description: Arabixiv (OSF), DOI prefix 10.31221. + metaarxiv: + description: MetaArXiv (OSF), DOI prefix 10.31222. + engrxiv: + description: engrXiv (OSF), DOI prefix 10.31224. + focus_archive: + description: FocUS Archive (OSF), DOI prefix 10.31225. + frenxiv: + description: Frenxiv (OSF), DOI prefix 10.31226. + ina_rxiv: + description: INA-Rxiv (OSF), DOI prefix 10.31227. + lawarxiv: + description: LawArXiv (OSF), DOI prefix 10.31228. + lis_scholarship_archive: + description: LIS Scholarship Archive (OSF), DOI prefix 10.31229. + marxiv: + description: MarXiv (OSF), DOI prefix 10.31230. + mindrxiv: + description: MindRxiv (OSF), DOI prefix 10.31231. + nutrixiv: + description: NutriXiv (OSF), DOI prefix 10.31232. + paleorxiv: + description: PaleorXiv (OSF), DOI prefix 10.31233. + psyarxiv: + description: PsyArXiv (OSF), DOI prefix 10.31234. + socarxiv: + description: SocArXiv (OSF), DOI prefix 10.31235. + sportrxiv: + description: SportRxiv (OSF), DOI prefix 10.31236. + africarxiv: + description: AfricArXiv (OSF), DOI prefix 10.31730. + mediarxiv: + description: MediArXiv (OSF), DOI prefix 10.33767. + bodoarxiv: + description: BodoArXiv (OSF), DOI prefix 10.34055. + edarxiv: + description: EdArXiv (OSF), DOI prefix 10.35542. + biohackrxiv: + description: BioHackrXiv (OSF), DOI prefix 10.37044. + ssrn: + description: SSRN, DOI prefix 10.2139. + preprints_org: + description: Preprints.org, DOI prefix 10.20944. + authorea: + description: Authorea, DOI prefix 10.22541. + techrxiv: + description: TechRxiv, DOI prefix 10.36227. + chemrxiv: + description: ChemRxiv, DOI prefix 10.26434. + jmir_preprints: + description: JMIR Preprints, DOI prefix 10.2196. + SectionKind: description: What a span of the text is. permissible_values: diff --git a/src/study_schema/linkml/neuroimaging-study-storage.yaml b/src/study_schema/linkml/neuroimaging-study-storage.yaml index ec1e167..cabba9d 100644 --- a/src/study_schema/linkml/neuroimaging-study-storage.yaml +++ b/src/study_schema/linkml/neuroimaging-study-storage.yaml @@ -30,7 +30,7 @@ description: >- stage can map and which facets have no field to map. license: CC0-1.0 -version: 0.7.0 +version: 0.8.0 see_also: - https://www.humanbrainmapping.org/files/2016/COBIDASreport.pdf - https://github.com/ohbm/cobidas_schema diff --git a/src/study_schema/linkml/neuroimaging-study-storage/analysis_enums.yaml b/src/study_schema/linkml/neuroimaging-study-storage/analysis_enums.yaml index db323d3..e05222b 100644 --- a/src/study_schema/linkml/neuroimaging-study-storage/analysis_enums.yaml +++ b/src/study_schema/linkml/neuroimaging-study-storage/analysis_enums.yaml @@ -1056,3 +1056,16 @@ enums: node: in_subset: [mvp, model_extracted] description: An atlas parcel or network node definition. + + PublicationStatus: + in_subset: [mvp, deterministic] + description: Whether a paper's version of record is known, or only its preprint. + permissible_values: + published: + in_subset: [mvp, deterministic] + description: >- + A DOI, PubMed record or PMC copy of the version of record is known. The paper may + have been a preprint too. + preprint: + in_subset: [mvp, deterministic] + description: Only a preprint's identifiers are known. diff --git a/src/study_schema/linkml/neuroimaging-study-storage/study.yaml b/src/study_schema/linkml/neuroimaging-study-storage/study.yaml index d77c2bb..51d0f60 100644 --- a/src/study_schema/linkml/neuroimaging-study-storage/study.yaml +++ b/src/study_schema/linkml/neuroimaging-study-storage/study.yaml @@ -84,7 +84,16 @@ classes: doi: in_subset: [mvp, deterministic] range: string - description: Digital Object Identifier, without requiring a particular URL form. + description: >- + Digital Object Identifier of the version of record, without requiring a particular + URL form. Never a preprint server's DOI. + publication_status: + in_subset: [mvp, deterministic] + range: PublicationStatus + description: >- + Whether the version of record is known or only the preprint, copied from the + parsed paper's header identifiers. `study_type` is independent of it: it keeps + PubMed's publication types verbatim, `Preprint` included. publication_year: in_subset: [mvp, deterministic] range: integer diff --git a/src/study_schema/models/paper_parse.py b/src/study_schema/models/paper_parse.py index db21bf6..b641f75 100644 --- a/src/study_schema/models/paper_parse.py +++ b/src/study_schema/models/paper_parse.py @@ -32,7 +32,7 @@ metamodel_version = "1.12.0" -version = "0.6.0" +version = "0.7.0" class ConfiguredBaseModel(BaseModel): @@ -118,6 +118,136 @@ class DocumentSource(str, Enum): """ +class PreprintServer(str, Enum): + """ + The preprint server that registered a DOI. A server hosted on the Open Science Framework is named by its provider, which has a DOI prefix of its own. A server is added here when a producer learns to recognise its DOIs. + """ + biorxiv = "biorxiv" + """ + bioRxiv, DOI prefix 10.1101 and, since 2025-11, 10.64898 (openRxiv). + """ + medrxiv = "medrxiv" + """ + medRxiv, DOI prefix 10.1101 and, since 2025-11, 10.64898 (openRxiv). + """ + arxiv = "arxiv" + """ + arXiv, DOI prefix 10.48550, the DOI registered for an arXiv id. + """ + researchsquare = "researchsquare" + """ + Research Square, DOI prefix 10.21203. + """ + osf_preprints = "osf_preprints" + """ + OSF Preprints, DOI prefix 10.31219. + """ + arabixiv = "arabixiv" + """ + Arabixiv (OSF), DOI prefix 10.31221. + """ + metaarxiv = "metaarxiv" + """ + MetaArXiv (OSF), DOI prefix 10.31222. + """ + engrxiv = "engrxiv" + """ + engrXiv (OSF), DOI prefix 10.31224. + """ + focus_archive = "focus_archive" + """ + FocUS Archive (OSF), DOI prefix 10.31225. + """ + frenxiv = "frenxiv" + """ + Frenxiv (OSF), DOI prefix 10.31226. + """ + ina_rxiv = "ina_rxiv" + """ + INA-Rxiv (OSF), DOI prefix 10.31227. + """ + lawarxiv = "lawarxiv" + """ + LawArXiv (OSF), DOI prefix 10.31228. + """ + lis_scholarship_archive = "lis_scholarship_archive" + """ + LIS Scholarship Archive (OSF), DOI prefix 10.31229. + """ + marxiv = "marxiv" + """ + MarXiv (OSF), DOI prefix 10.31230. + """ + mindrxiv = "mindrxiv" + """ + MindRxiv (OSF), DOI prefix 10.31231. + """ + nutrixiv = "nutrixiv" + """ + NutriXiv (OSF), DOI prefix 10.31232. + """ + paleorxiv = "paleorxiv" + """ + PaleorXiv (OSF), DOI prefix 10.31233. + """ + psyarxiv = "psyarxiv" + """ + PsyArXiv (OSF), DOI prefix 10.31234. + """ + socarxiv = "socarxiv" + """ + SocArXiv (OSF), DOI prefix 10.31235. + """ + sportrxiv = "sportrxiv" + """ + SportRxiv (OSF), DOI prefix 10.31236. + """ + africarxiv = "africarxiv" + """ + AfricArXiv (OSF), DOI prefix 10.31730. + """ + mediarxiv = "mediarxiv" + """ + MediArXiv (OSF), DOI prefix 10.33767. + """ + bodoarxiv = "bodoarxiv" + """ + BodoArXiv (OSF), DOI prefix 10.34055. + """ + edarxiv = "edarxiv" + """ + EdArXiv (OSF), DOI prefix 10.35542. + """ + biohackrxiv = "biohackrxiv" + """ + BioHackrXiv (OSF), DOI prefix 10.37044. + """ + ssrn = "ssrn" + """ + SSRN, DOI prefix 10.2139. + """ + preprints_org = "preprints_org" + """ + Preprints.org, DOI prefix 10.20944. + """ + authorea = "authorea" + """ + Authorea, DOI prefix 10.22541. + """ + techrxiv = "techrxiv" + """ + TechRxiv, DOI prefix 10.36227. + """ + chemrxiv = "chemrxiv" + """ + ChemRxiv, DOI prefix 10.26434. + """ + jmir_preprints = "jmir_preprints" + """ + JMIR Preprints, DOI prefix 10.2196. + """ + + class SectionKind(str, Enum): """ What a span of the text is. @@ -1283,6 +1413,20 @@ class AnchorKind(str, Enum): """ +class PublicationStatus(str, Enum): + """ + Whether a paper's version of record is known, or only its preprint. + """ + published = "published" + """ + A DOI, PubMed record or PMC copy of the version of record is known. The paper may have been a preprint too. + """ + preprint = "preprint" + """ + Only a preprint's identifiers are known. + """ + + class ArtifactHeader(ConfiguredBaseModel): """ @@ -1299,14 +1443,26 @@ class ArtifactHeader(ConfiguredBaseModel): class ArticleIdentifiers(ConfiguredBaseModel): """ - Every identifier known for the paper when the artifact was written. Carried whole, so no consumer has to choose between them or look one up. + Every identifier known for the paper when the artifact was written. Carried whole, so no consumer has to choose between them or look one up. `doi`, `pmid` and `pmcid` name the version of record; a preprint's identifiers are kept beside them in the `preprint_` fields, so a paper and its preprint are one article. """ - pmid: Optional[str] = Field(default=None) - pmcid: Optional[str] = Field(default=None) - doi: Optional[str] = Field(default=None) + pmid: Optional[str] = Field(default=None, description="""PubMed's record of the version of record. Never PubMed's record of a preprint, which is `preprint_pmid`.""") + pmcid: Optional[str] = Field(default=None, description="""PubMed Central's copy of the version of record. Never PMC's copy of a preprint, which is `preprint_pmcid`.""") + doi: Optional[str] = Field(default=None, description="""The DOI of the version of record. Never a preprint server's DOI, which is in `preprint_dois`.""") + preprint_dois: Optional[list[PreprintDoi]] = Field(default=None, description="""Every DOI a preprint server registered for the paper. A server that registers each version separately, such as Research Square's `/v2`, gives one entry per version, each kept as registered.""") + preprint_pmid: Optional[str] = Field(default=None, description="""PubMed's record of the preprint, which PubMed types `Preprint`.""") + preprint_pmcid: Optional[str] = Field(default=None, description="""PubMed Central's copy of the preprint.""") + publication_status: PublicationStatus = Field(default=..., description="""`preprint` when a `preprint_` field is set and `doi`, `pmid` and `pmcid` are all empty, `published` otherwise. The producer states it so that no consumer re-derives it.""") neurostore_base_study_id: Optional[str] = Field(default=None, description="""Set once the paper has been uploaded; neurostore's BaseStudy id.""") +class PreprintDoi(ConfiguredBaseModel): + """ + A DOI a preprint server registered, with the server that registered it. + """ + doi: str = Field(default=...) + server: PreprintServer = Field(default=..., description="""Named by the producer from the DOI, so that no consumer needs a table of preprint servers' DOI patterns.""") + + class Producer(ConfiguredBaseModel): """ The program and stage that wrote an artifact. @@ -1637,6 +1793,7 @@ class AnalysisVerdict(ConfiguredBaseModel): # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model ArtifactHeader.model_rebuild() ArticleIdentifiers.model_rebuild() +PreprintDoi.model_rebuild() Producer.model_rebuild() InputRef.model_rebuild() TextSpan.model_rebuild() diff --git a/src/study_schema/models/storage.py b/src/study_schema/models/storage.py index 129aa02..33bfce6 100644 --- a/src/study_schema/models/storage.py +++ b/src/study_schema/models/storage.py @@ -32,7 +32,7 @@ metamodel_version = "1.12.0" -version = "0.7.0" +version = "0.8.0" class ConfiguredBaseModel(BaseModel): @@ -1405,6 +1405,20 @@ class AnchorKind(str, Enum): """ +class PublicationStatus(str, Enum): + """ + Whether a paper's version of record is known, or only its preprint. + """ + published = "published" + """ + A DOI, PubMed record or PMC copy of the version of record is known. The paper may have been a preprint too. + """ + preprint = "preprint" + """ + Only a preprint's identifiers are known. + """ + + class StudyDesign(ConfiguredBaseModel): """ @@ -2004,7 +2018,8 @@ class Study(ConfiguredBaseModel): design: Optional[StudyDesign] = Field(default=None, description="""The study's design outside its tasks: how participants were assigned, the arms they were assigned to, and the occasions data were collected on. The narrative design description is this record's own `description`.""") authors: Optional[list[str]] = Field(default=None, description="""Authors of the study or publication, in published order.""") journal: Optional[str] = Field(default=None, description="""Journal or other publication venue.""") - doi: Optional[str] = Field(default=None, description="""Digital Object Identifier, without requiring a particular URL form.""") + doi: Optional[str] = Field(default=None, description="""Digital Object Identifier of the version of record, without requiring a particular URL form. Never a preprint server's DOI.""") + publication_status: Optional[PublicationStatus] = Field(default=None, description="""Whether the version of record is known or only the preprint, copied from the parsed paper's header identifiers. `study_type` is independent of it: it keeps PubMed's publication types verbatim, `Preprint` included.""") publication_year: Optional[int] = Field(default=None, description="""Year in which the study was first published or made publicly available.""", ge=1600) language: Optional[list[str]] = Field(default=None, description="""Languages of the article, as the E-utilities API reports them -- the three-letter codes PubMed assigns, `eng` and the rest. Filled from the same `esummary` call that fills `study_type`, and not read off the paper: the text a reader is given is in one language whatever the article is indexed as. At least six of the benchmark's sixteen meta-analyses state \"English language\" as an inclusion criterion, and it was unexpressible. On the 1,804 records measured it excludes exactly one paper, so it buys precision rather than recall; the criterion is stated, so the field states it.""") diff --git a/storage-parameter-priorities.yaml b/storage-parameter-priorities.yaml index df611fa..ef6bf3f 100644 --- a/storage-parameter-priorities.yaml +++ b/storage-parameter-priorities.yaml @@ -29,6 +29,7 @@ Study: authors: n/a journal: n/a doi: n/a + publication_status: n/a publication_year: n/a external_datasets: 1 study_type: 0 diff --git a/tests/test_identifiers.py b/tests/test_identifiers.py new file mode 100644 index 0000000..ae86435 --- /dev/null +++ b/tests/test_identifiers.py @@ -0,0 +1,100 @@ +"""A header's identifiers tell the version of record from the preprint, and say which the +paper has.""" + +import pytest +from pydantic import ValidationError + +from conftest import EXAMPLES, load +from study_schema import jsonschema +from study_schema.models import paper_parse, storage + +IDENTIFIERS = EXAMPLES / "identifiers" +VALID = ["preprint_only", "published_with_preprint"] +INVALID = { + "missing_publication_status": "publication_status", + "preprint_doi_without_server": "server", +} + + +def _paper_with(identifiers: dict, example_paths) -> dict: + paper = load(example_paths["parsed_paper"]) + paper["header"]["identifiers"] = identifiers + return paper + + +def _status(identifiers: dict) -> str: + """The rule `ArticleIdentifiers.publication_status` documents.""" + record = any(identifiers.get(k) for k in ("doi", "pmid", "pmcid")) + preprint = any(identifiers.get(k) for k in ("preprint_dois", "preprint_pmid", "preprint_pmcid")) + return "preprint" if preprint and not record else "published" + + +@pytest.mark.parametrize("name", VALID) +def test_identifier_examples_validate(name, example_paths): + validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") + identifiers = load(IDENTIFIERS / f"{name}.json") + paper = _paper_with(identifiers, example_paths) + + parsed = paper_parse.ParsedPaper.model_validate(paper) + validator.validate(paper, jsonschema.load("parsed-paper")) + assert parsed.header.identifiers.publication_status == _status(identifiers) + + +def test_a_preprint_only_paper_holds_no_version_of_record_id(): + ids = paper_parse.ArticleIdentifiers.model_validate(load(IDENTIFIERS / "preprint_only.json")) + assert (ids.doi, ids.pmid, ids.pmcid) == (None, None, None) + assert ids.preprint_pmid and ids.preprint_pmcid + assert [d.server for d in ids.preprint_dois] == ["biorxiv"] + assert ids.publication_status == "preprint" + + +def test_a_published_paper_keeps_its_preprint_doi_beside_its_own(): + ids = paper_parse.ArticleIdentifiers.model_validate(load(IDENTIFIERS / "published_with_preprint.json")) + assert ids.doi == "10.1111/desc.12933" + assert [(d.doi, d.server) for d in ids.preprint_dois] == [ + ("10.31234/osf.io/5m87x", "psyarxiv") + ] + assert ids.publication_status == "published" + + +@pytest.mark.parametrize("name, field", INVALID.items()) +def test_invalid_identifier_examples_are_refused(name, field, example_paths): + validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") + paper = _paper_with(load(IDENTIFIERS / "invalid" / f"{name}.json"), example_paths) + + with pytest.raises(ValidationError, match=field): + paper_parse.ParsedPaper.model_validate(paper) + with pytest.raises(validator.ValidationError, match=field): + validator.validate(paper, jsonschema.load("parsed-paper")) + + +def test_a_preprint_server_outside_the_vocabulary_is_refused(): + ids = load(IDENTIFIERS / "published_with_preprint.json") + ids["preprint_dois"][0]["server"] = "osf" + with pytest.raises(ValidationError, match="server"): + paper_parse.ArticleIdentifiers.model_validate(ids) + + +def test_header_and_storage_share_one_publication_status_vocabulary(): + assert {v.value for v in paper_parse.PublicationStatus} == { + v.value for v in storage.PublicationStatus + } == {"published", "preprint"} + + +@pytest.mark.parametrize("status", ["published", "preprint"]) +def test_storage_study_carries_the_publication_status(status): + validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") + study = {"id": "x7Ex4mpleA01", "title": "An example", "study_type": ["Preprint"], "publication_status": status} + + assert storage.Study.model_validate(study).publication_status == status + validator.validate(study, jsonschema.load("storage-study")) + + +def test_storage_refuses_an_unknown_publication_status(): + validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") + study = {"id": "x7Ex4mpleA01", "title": "An example", "study_type": ["Preprint"], "publication_status": "in_press"} + + with pytest.raises(ValidationError, match="publication_status"): + storage.Study.model_validate(study) + with pytest.raises(validator.ValidationError): + validator.validate(study, jsonschema.load("storage-study")) diff --git a/tools/benchmark_parquet.py b/tools/benchmark_parquet.py index 0140f71..e36e10f 100644 --- a/tools/benchmark_parquet.py +++ b/tools/benchmark_parquet.py @@ -43,7 +43,7 @@ def parse(i): origin="table", table_id=tid, name=f"Condition {a} > Baseline", name_is_printed=True, coordinate_space="MNI", role="result", statistic="t", thresholds=[pp.Threshold(level="height", quantity="p", value=0.001, correction="uncorrected")], points=pts)) hdr = pp.ArtifactHeader(artifact_kind="coordinate_parse", schema_version=pp.version, article_id=f"art-{i:06d}", - identifiers=pp.ArticleIdentifiers(pmid=str(10_000_000+i)), producer=pp.Producer(name="ns-pond-ingestion-workflow", version="1.0", stage="parse"), created_at="2026-10-09T12:00:00Z") + identifiers=pp.ArticleIdentifiers(pmid=str(10_000_000+i), publication_status="published"), producer=pp.Producer(name="ns-pond-ingestion-workflow", version="1.0", stage="parse"), created_at="2026-10-09T12:00:00Z") return pp.CoordinateParse(header=hdr, parse_id=sha, text_sha256=sha, analyses=analyses, tables=[pp.TableReading(table_id=f"tbl{t+1}", reading="coordinates") for t in range(3)]) N = int(sys.argv[2]) if len(sys.argv) > 2 else 5000