From 580dcd097504d3b1c61d4379478071cd60b28bc2 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Thu, 10 Sep 2026 15:45:33 +0100 Subject: [PATCH 1/4] Copy files from warehouse to elt-pipeline --- .../base/base_statusdisplay__cycles.sql | 18 ++++++++++ .../base/base_statusdisplay__cycles.yml | 11 ++++++ .../base_statusdisplay__cycles__phases.sql | 22 ++++++++++++ .../base_statusdisplay__cycles__phases.yml | 16 +++++++++ .../accelerator/stg_statusdisplay__cycles.sql | 35 +++++++++++++++++++ .../accelerator/stg_statusdisplay__cycles.yml | 25 +++++++++++++ 6 files changed, 127 insertions(+) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.sql create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.yml create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.sql create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.yml create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.sql new file mode 100644 index 00000000..1edcab38 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.sql @@ -0,0 +1,18 @@ +with source as ( + + select * from {{ source('accelerator_statusdisplay', 'cycles') }} + +), + +renamed as ( + + select + + _dlt_id as dlt_id, + label as name + + from source + +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.yml new file mode 100644 index 00000000..d3354677 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles.yml @@ -0,0 +1,11 @@ +models: + - name: base_statusdisplay__cycles + description: > + Each row contains the cycle id & label along with a primary key referenced + in the secondary base_statusdisplay__cycles__phases table where + information describing a particular phase of a cycle is stored. + columns: + - name: dlt_id + description: Primary key - unique identifier for the cycle from the source system + - name: name + description: The cycle label/name diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.sql new file mode 100644 index 00000000..afb362df --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.sql @@ -0,0 +1,22 @@ +-- See base_statusdisplay__cycles.sql for a description of why this table exists. + +with source as ( + + select * from {{ source('accelerator_statusdisplay', 'cycles__phases') }} + +), + +renamed as ( + + select + type as phase, + target, + start as started_at, + {{ adapter.quote("end") }} as ended_at, + _dlt_parent_id as dlt_cycles_id + + from source + +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.yml new file mode 100644 index 00000000..80a073c2 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_statusdisplay__cycles__phases.yml @@ -0,0 +1,16 @@ +models: + - name: base_statusdisplay__cycles__phases + description: > + Each row in the cycles__phases table corrseponds to a section of time + within a given cycle. + columns: + - name: phase + description: The name of the phase within the cycle + - name: target + description: The target station number that was running during this phase + - name: started_at + description: Timestamp when the phase started + - name: ended_at + description: Timestamp when the phase ended + - name: dlt_cycles_id + description: Foreign key referencing base_statusdisplay__cycles (dlt_id) diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql new file mode 100644 index 00000000..b049809b --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql @@ -0,0 +1,35 @@ +-- Simple join to denormalize the cycles and cycles__phases tables +-- that are not much use as separated table. + +{# Denormalize cycles & cycles__phases to include the cyce label for each phase #} + +with + +base_cycles as ( + + select * from {{ ref('base_statusdisplay__cycles') }} + +), + +base_cycles__phases as ( + + select * from {{ ref('base_statusdisplay__cycles__phases') }} + +), + +join_cycle_labels_and_phases as ( + + select + + {{ adapter.quote('name') }}, + started_at, + ended_at, + phase, + {{ adapter.quote('target') }} + + from base_cycles + join base_cycles__phases on base_cycles.dlt_id = base_cycles__phases.dlt_cycles_id + +) + +select * from join_cycle_labels_and_phases diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.yml new file mode 100644 index 00000000..39f11e3f --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.yml @@ -0,0 +1,25 @@ +models: + - name: stg_statusdisplay__cycles + description: > + Each record defines a phase of a cycle, including which target was + running. Multiple records exist for a given period of time if more than + one target station was running. + +unit_tests: + - name: test_stg_statusdisplay__cycles + model: stg_statusdisplay__cycles + given: + - input: ref('base_statusdisplay__cycles') + format: sql + fixture: base_statusdisplay__cycles + - input: ref('base_statusdisplay__cycles__phases') + format: sql + fixture: base_statusdisplay__cycles__phases + expect: + rows: + - { + name: "2024/2", + started_at: "2024-07-09 07:30:00 UTC", + ended_at: "2024-07-24 07:30:00 UTC", + target: 1, + } From bb195763d621c0358de8b838129fd351d910cba3 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Thu, 10 Sep 2026 16:35:28 +0100 Subject: [PATCH 2/4] Add record to accelerator sources file --- .../models/staging/accelerator/_accelerator__sources.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml index 28b76ead..6cde418c 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -10,3 +10,8 @@ sources: - name: LogBookChapter - name: Logbooks - name: MoreEntryColumns + - name: accelerator_statusdisplay + database: facility_ops_landing + tables: + - name: cycles_phases + - name: cycles From 0350c3bdb961844348219e381f305fbfe6451d26 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 11 Sep 2026 11:10:14 +0100 Subject: [PATCH 3/4] Corrected comment, referencing https://github.com/ISISNeutronMuon/analytics-data-platform/pull/470#discussion_r3988121951 --- .../models/staging/accelerator/stg_statusdisplay__cycles.sql | 2 +- .../models/staging/accelerator/stg_statusdisplay__cycles.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql index b049809b..da926e26 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql @@ -1,7 +1,7 @@ -- Simple join to denormalize the cycles and cycles__phases tables -- that are not much use as separated table. -{# Denormalize cycles & cycles__phases to include the cyce label for each phase #} +{# Denormalize cycles & cycles__phases to include the cycle label for each phase #} with diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql b/warehouses/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql index b049809b..da926e26 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql +++ b/warehouses/facility_ops/transform/models/staging/accelerator/stg_statusdisplay__cycles.sql @@ -1,7 +1,7 @@ -- Simple join to denormalize the cycles and cycles__phases tables -- that are not much use as separated table. -{# Denormalize cycles & cycles__phases to include the cyce label for each phase #} +{# Denormalize cycles & cycles__phases to include the cycle label for each phase #} with From eac7a8a8bc66009cb92217d7dc400846e28c1059 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 11 Sep 2026 11:14:36 +0100 Subject: [PATCH 4/4] Corrected nomenclature disparity, referencing https://github.com/ISISNeutronMuon/analytics-data-platform/pull/470#discussion_r3980831361 --- .../models/staging/accelerator/_accelerator__sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml index 6cde418c..cd3edf07 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -13,5 +13,5 @@ sources: - name: accelerator_statusdisplay database: facility_ops_landing tables: - - name: cycles_phases + - name: cycles__phases - name: cycles