justfile

# =======
# Configs
# =======

set shell := ["bash", "-uc"]
set dotenv-load

# -----
# Paths
# -----

local_dir := "local/"
init_sql_path := join(local_dir, "init.sql")

engine_db_path := join(local_dir, env_var("ENGINE_DB"))
stage_db_path := join(local_dir, env_var("STAGE_DB"))
secure_stage_db_path := join(local_dir, env_var("SECURE_STAGE_DB"))
graphs_mart_db_path := join(local_dir, env_var("GRAPHS_MART_DB"))
analytics_mart_db_path := join(local_dir, env_var("ANALYTICS_MART_DB"))

# --------
# Datasets
# --------

ds_dsn_url := "https://www.kaggle.com/datasets/andreagarritano/deezer-social-networks"
ds_msdsl_url := "https://www.kaggle.com/datasets/undefinenull/million-song-dataset-spotify-lastfm"
ds_dd_url := "https://huggingface.co/datasets/ShreyaR/DepressionDetection"
ds_dd_monitor_url := "https://huggingface.co/datasets/joangaes/depression"


# ======
# Common
# ======

default:
    just -l

check binary:
    @echo -n "Checking {{binary}}... "
    @which {{binary}} >/dev/null && test -x $(which {{binary}}) \
        || (echo "failed ({{binary}} not found)"; exit 1)
    @echo ok

check-dlctl:
    just check dlctl

check-duckdb:
    just check duckdb

check-curl:
    just check curl

check-terraform:
    just check terraform

check-docker:
    just check docker

check-psql:
    just check psql

check-pgloader:
    just check pgloader

confirm:
    #!/bin/bash
    while true; do
        read -p "Are you sure? [y/N] " yn
        yn=${yn:-N}
        case $yn in
            [Yy]*) exit 0;;
            [Nn]*) exit 1;;
        esac
    done


# ===========
# Development
# ===========

check-dev:
    just check git
    just check uv
    just check nbdime
    just check nbstripout

setup-dev:
    just check git || sudo apt install -y git
    just check uv || curl -LsSf https://astral.sh/uv/install.sh | sh
    just check nbdime || nbdime config-git --enable --global
    just check nbstripout || nbstripout --install --global


# ========
# DuckLake
# ========

check-init-sql:
    test -r {{init_sql_path}} || just generate-init-sql

generate-init-sql: check-dlctl
    dlctl tools generate-init-sql --path {{init_sql_path}}

lakehouse: check-duckdb check-init-sql
    duckdb -init {{init_sql_path}} {{engine_db_path}}


# ====================
# GraphRAG with KùzuDB
# ====================

graphrag-ingest: check-dlctl
    dlctl ingest dataset {{ds_dsn_url}}
    dlctl ingest dataset {{ds_msdsl_url}}

graphrag-transform:
    dlctl transform -m "+marts.graphs.music_taste"

graphrag-export:
    dlctl export dataset graphs "music_taste"

graphrag-load:
    dlctl graph load --overwrite "music_taste"

graphrag-tl: graphrag-transform graphrag-export graphrag-load

graphrag-etl: graphrag-ingest graphrag-tl

graphrag-embeddings: check-dlctl
    dlctl graph compute embeddings "music_taste" -d 256 -b 9216 -e 5
    dlctl graph reindex "music_taste"

graphrag: check-dlctl
    dlctl graph rag "music_taste" -i

graphrag-all: graphrag-etl graphrag-embeddings graphrag


# =============================
# Economic Competition Networks
# =============================

econ-compnet-ingest: check-dlctl
    dlctl ingest dataset -t "atlas" "The Atlas of Economic Complexity"

econ-compnet-transform: check-dlctl
    dlctl transform -m "+marts.graphs.econ_comp"

econ-compnet-export: check-dlctl
    dlctl export dataset graphs "econ_comp"

econ-compnet-load: check-dlctl
    dlctl graph load --overwrite "econ_comp"

econ-compnet-tl: econ-compnet-transform econ-compnet-export econ-compnet-load

econ-compnet-etl: econ-compnet-ingest econ-compnet-tl

econ-compnet-scoring: check-dlctl
    dlctl graph compute con-score "econ_comp" "Country" "CompetesWith"

econ-compnet-all: econ-compnet-etl econ-compnet-scoring


# ===================================================
# MLOps: A/B Testing with MLflow, Kafka, and DuckLake
# ===================================================

# ----
# ETL
# ----

mlops-ingest: check-dlctl
    dlctl ingest dataset {{ds_dd_url}}
    dlctl ingest dataset {{ds_dd_monitor_url}}

mlops-transform: check-dlctl
    dlctl transform -m "+stage.depression_detection"

mlops-tl: mlops-transform

mlops-etl: mlops-ingest mlops-tl

# --------
# Training
# --------

mlops-train-logreg-tfidf: check-dlctl
    dlctl ml train "dd" --method "logreg" --features "tfidf"

mlops-train-logreg-embeddings: check-dlctl
    dlctl ml train "dd" --method "logreg" --features "embeddings"

mlops-train-logreg: mlops-train-logreg-tfidf mlops-train-logreg-embeddings

mlops-train-xgboost-tfidf: check-dlctl
    dlctl ml train "dd" --method "xgboost" --features "tfidf"

mlops-train-xgboost-embeddings: check-dlctl
    dlctl ml train "dd" --method "xgboost" --features "embeddings"

mlops-train-xgboost: mlops-train-xgboost-tfidf mlops-train-xgboost-embeddings

mlops-train: mlops-train-logreg mlops-train-xgboost

# --------------
# ETL + Training
# --------------

mlops-all: mlops-etl mlops-train

# ------
# Server
# ------

mlops-serve: check-dlctl
    dlctl ml server

mlops_test_inference_payload := '''
{
    "models": [
        {
            "name": "dd_logreg_tfidf",
            "version": "latest"
        },
        {
            "name": "dd_xgboost_embeddings",
            "version": "latest"
        }
    ],
    "data": "hello twitter i m on a one week leave from school bc i have depression how are you all d",
    "log_to_lakehouse": true
}
'''

mlops-test-inference: check-curl
    curl -f -X POST "http://localhost:8000/inference" \
        -H "Content-Type: application/json" \
        -d '{{mlops_test_inference_payload}}'
    @echo
    curl -f -X GET "http://localhost:8000/inference/logs/flush"

mlops-test-feedback uuid feedback: check-curl
    curl -f -X PATCH "http://localhost:8000/inference" \
        -H "Content-Type: application/json" \
        -d '{"inference_uuid": "{{uuid}}", "feedback": {{feedback}}}'
    curl -f -X GET "http://localhost:8000/inference/logs/flush"

# ----------
# Monitoring
# ----------

mlops-simulate-inference: check-dlctl
    dlctl ml simulate "dd" \
        --sample-fraction 0.01 \
        --model-uri "models:/dd_xgboost_embeddings/latest" \
        --model-uri "models:/dd_logreg_tfidf/latest"

mlops-monitor-compute: check-dlctl
    dlctl ml monitor compute "dd" \
        --model-uri "models:/dd_xgboost_embeddings/latest" \
        --model-uri "models:/dd_logreg_tfidf/latest"

mlops-monitor-plot: check-dlctl
    dlctl ml monitor plot "dd" \
        --model-uri "models:/dd_xgboost_embeddings/latest" \
        --model-uri "models:/dd_logreg_tfidf/latest"


# ==============
# Data Lab Infra
# ==============

docker_shared_context := "docker-shared"

# -------------
# Config Checks
# -------------

infra-config-check-foundation: check-terraform
    @echo -n "Checking foundation configs... "
    @test -f infra/foundation/terraform.tfvars \
        || (echo "failed (terraform.tfvars: not found)"; exit 1)
    @echo ok

infra-config-check-platform: check-terraform
    @echo -n "Checking platform configs... "
    @test -f infra/platform/terraform.tfvars \
        || (echo "failed: terraform.tfvars not found"; exit 1)
    @test -f infra/platform/state.config \
        || (echo "state.config: not found"; exit 2)
    @echo ok

infra-config-check-services: check-docker
    @echo -n "Checking {{docker_shared_context}} docker context... "
    @docker context ls --format "{{{{.Name}}" | grep -q '^{{docker_shared_context}}$' \
        || (echo "failed: {{docker_shared_context}} docker context not configured"; exit 1)
    @echo ok

infra-config-check-all: infra-config-check-foundation \
    infra-config-check-platform \
    infra-config-check-services

# ---------------
# Initializations
# ---------------

infra-foundation-init: infra-config-check-foundation
    terraform -chdir=infra/foundation init

infra-platform-init: infra-config-check-platform
    terraform -chdir=infra/platform init -backend-config=state.config

infra-init: infra-foundation-init infra-platform-init

# ------------
# Provisioning
# ------------

infra-provision-foundation: infra-config-check-foundation
    terraform -chdir=infra/foundation apply

infra-provision-platform: infra-config-check-platform
    terraform -chdir=infra/platform apply

infra-provision-services: infra-config-check-services
    terraform -chdir=infra/services/gitlab apply
    docker -c {{docker_shared_context}} compose -p datalab -f infra/services/docker/compose.yml up -d

infra-provision-all: infra-provision-foundation \
    infra-provision-platform \
    infra-provision-services

infra-provision-local:
    docker compose -p datalab -f infra/services/docker/compose.yml --profile dev up -d

# -----------
# Destruction
# -----------

infra-destroy-foundation:
    terraform -chdir=infra/foundation destroy

infra-destroy-platform:
    terraform -chdir=infra/platform destroy

infra-destroy-services: confirm
    docker -c {{docker_shared_context}} compose -p datalab -f infra/services/docker/compose.yml down
    terraform -chdir=infra/services/gitlab destroy -auto-approve

infra-destroy-all: infra-destroy-services \
    infra-destroy-platform \
    infra-destroy-foundation

infra-destroy-local:
    docker compose -p datalab -f infra/services/docker/compose.yml --profile dev down

# ---------
# Utilities
# ---------

infra-show-tf-credentials layer:
    @[[ " foundation platform " == *" {{layer}} "* ]] \
        || (echo "{{layer}}: invalid layer"; exit 1)
    @terraform -chdir=infra/{{layer}} output -json \
        | jq -r 'to_entries[] \
        | select(.value.sensitive==true) \
        | "\(.key) = \(.value.value)"'

infra-show-credentials: infra-config-check-all
    @echo
    @echo "=========="
    @echo "Foundation"
    @echo "=========="
    @just infra-show-tf-credentials foundation
    @echo
    @echo "========"
    @echo "Platform"
    @echo "========"
    @just infra-show-tf-credentials platform


# ====================================================
# Migrating DuckLake Catalog From SQLite to PostgreSQL
# ====================================================

migrate_lakehouse_fix_script := "./scripts/ducklake_sqlite_to_postgres_fix.sql"

test-lakehouse-catalog-connection: check-psql
    #!/bin/bash

    export PGHOST=$PSQL_CATALOG_HOST
    export PGPORT=$PSQL_CATALOG_PORT
    export PGDATABASE=$PSQL_CATALOG_DB
    export PGUSER=$PSQL_CATALOG_USER
    export PGPASSWORD=$PSQL_CATALOG_PASSWORD

    echo -n "Testing lakehouse catalog connection... "

    if output=$(psql -c '\q' 2>&1); then
        echo "ok"
    else
        echo "failed"
        echo $output
    fi

migrate-lakehouse-catalog catalog: check-pgloader test-lakehouse-catalog-connection
    #!/bin/bash

    set -e

    export PGHOST=$PSQL_CATALOG_HOST
    export PGPORT=$PSQL_CATALOG_PORT
    export PGDATABASE=$PSQL_CATALOG_DB
    export PGUSER=$PSQL_CATALOG_USER
    export PGPASSWORD=$PSQL_CATALOG_PASSWORD

    case "{{catalog}}" in
        "stage")
            sqlite_db_path="{{stage_db_path}}"
            psql_schema="$PSQL_CATALOG_STAGE_SCHEMA"
            ;;
        "secure_stage")
            sqlite_db_path="{{secure_stage_db_path}}"
            psql_schema="$PSQL_CATALOG_SECURE_STAGE_SCHEMA"
            ;;
        "graphs_mart")
            sqlite_db_path="{{graphs_mart_db_path}}"
            psql_schema="$PSQL_CATALOG_GRAPHS_MART_SCHEMA"
            ;;
        "analytics_mart")
            sqlite_db_path="{{analytics_mart_db_path}}"
            psql_schema="$PSQL_CATALOG_ANALYTICS_MART_SCHEMA"
            ;;
    esac

    echo "Migrating {{catalog}} catalog from SQLite to PostgreSQL..."

    psql_conn_str="postgresql://$PGUSER:$PGPASSWORD@$PGHOST:$PGPORT/$PGDATABASE"

    psql -c "CREATE SCHEMA IF NOT EXISTS $psql_schema"
    pgloader --set search_path="'$psql_schema'" $sqlite_db_path $psql_conn_str
    PGOPTIONS="--search-path=$psql_schema" psql -f "{{migrate_lakehouse_fix_script}}"

migrate-lakehouse-catalog-all:
    just migrate-lakehouse-catalog stage
    just migrate-lakehouse-catalog secure_stage
    just migrate-lakehouse-catalog graphs_mart
    just migrate-lakehouse-catalog analytics_mart


# ======
# Global
# ======

global-etl: graphrag-etl econ-compnet-etl mlops-etl
global-ingest: graphrag-ingest econ-compnet-ingest mlops-ingest
global-tl: graphrag-tl econ-compnet-tl mlops-tl