diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000..ed3d0e0 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,24 @@ +# CI + +The `ci.yml` workflow runs on every push/PR to `master`. It: + +1. Installs OS tools (Java, Node, Vault, Babashka) via `scripts/install --no-deps` +2. Opens the GitHub Actions runner IP on the Vault security group so it can authenticate +3. Fetches AWS credentials from Vault for the private S3 Maven repo +4. Caches Maven (`~/.m2`) and npm (`node_modules`) dependencies between runs +5. Installs project dependencies via `scripts/install` +6. Runs `scripts/test` (JVM, ClojureScript, and Babashka test suites) +7. Revokes the runner IP from the Vault security group + +## Required secrets + +| Secret | Purpose | +|---|---| +| `GH_ACTIONS_AWS_ACCESS_KEY_ID` | AWS key for managing the Vault security group | +| `GH_ACTIONS_AWS_SECRET_ACCESS_KEY` | AWS secret for managing the Vault security group | +| `VAULT_GITHUB_TOKEN` | GitHub token for `vault login -method=github` | + +## Composite actions + +- **`vault-allow`** — adds the runner's IP to the Vault SG and logs into Vault +- **`vault-revoke`** — removes the runner's IP from the Vault SG (runs even if tests fail) diff --git a/.github/actions/vault-allow/action.yml b/.github/actions/vault-allow/action.yml new file mode 100644 index 0000000..a8dc4ee --- /dev/null +++ b/.github/actions/vault-allow/action.yml @@ -0,0 +1,43 @@ +name: Allow runner IP on Vault security group +description: > + Gets the runner's public IP, opens port 443 on the Vault SG, + and logs into Vault via GitHub token so subsequent steps can + call vault commands without additional auth. + +inputs: + aws-access-key-id: + required: true + aws-secret-access-key: + required: true + github-token: + description: GitHub token for vault login (method=github) + required: true + +outputs: + ip: + description: Runner public IP address (pass to vault-revoke) + value: ${{ steps.ip.outputs.ip }} + +runs: + using: composite + steps: + - id: ip + shell: bash + run: echo "ip=$(curl -s https://checkip.amazonaws.com)" >> $GITHUB_OUTPUT + + - shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} + run: | + aws ec2 authorize-security-group-ingress \ + --region us-west-2 \ + --group-id sg-702d6801 \ + --protocol tcp --port 443 \ + --cidr ${{ steps.ip.outputs.ip }}/32 + + - shell: bash + env: + VAULT_ADDR: https://vault.techascent.com + GITHUB_TOKEN: ${{ inputs.github-token }} + run: vault login -method=github token="$GITHUB_TOKEN" >/dev/null diff --git a/.github/actions/vault-revoke/action.yml b/.github/actions/vault-revoke/action.yml new file mode 100644 index 0000000..4608661 --- /dev/null +++ b/.github/actions/vault-revoke/action.yml @@ -0,0 +1,25 @@ +name: Revoke runner IP from Vault security group +description: Removes the runner's IP from the Vault SG (use with if:always()) + +inputs: + ip: + description: Runner IP from vault-allow output + required: true + aws-access-key-id: + required: true + aws-secret-access-key: + required: true + +runs: + using: composite + steps: + - shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} + run: | + aws ec2 revoke-security-group-ingress \ + --region us-west-2 \ + --group-id sg-702d6801 \ + --protocol tcp --port 443 \ + --cidr ${{ inputs.ip }}/32 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3afb468 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,68 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + workflow_dispatch: + +jobs: + test: + name: Test (JVM + CLJS + Babashka) + runs-on: ubuntu-latest + env: + VAULT_ADDR: https://vault.techascent.com + + steps: + - uses: actions/checkout@v4 + + - name: Install OS tools (vault, babashka, etc.) + run: scripts/install --no-deps + + - name: Allow runner IP on Vault + id: vault + uses: ./.github/actions/vault-allow + with: + aws-access-key-id: ${{ secrets.GH_ACTIONS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GH_ACTIONS_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ secrets.VAULT_GITHUB_TOKEN }} + + - name: Fetch AWS credentials (core) for S3 Maven repo + run: scripts/aws-creds core --write-profile default --force + + - name: Verify AWS credentials + run: aws sts get-caller-identity + + - name: Set up Java 21 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Cache Maven deps + uses: actions/cache@v4 + with: + path: ~/.m2 + key: m2-${{ hashFiles('deps.edn') }} + restore-keys: m2- + + - name: Cache npm deps + uses: actions/cache@v4 + with: + path: node_modules + key: npm-${{ hashFiles('package.json') }} + + - name: Install dependencies + run: scripts/install + + - name: Run tests + run: scripts/test + + - name: Revoke runner IP from Vault + if: always() + uses: ./.github/actions/vault-revoke + with: + ip: ${{ steps.vault.outputs.ip }} + aws-access-key-id: ${{ secrets.GH_ACTIONS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.GH_ACTIONS_AWS_SECRET_ACCESS_KEY }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 0457f73..a5db36b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added +- Babashka support via cross-platform `core.cljc` and `files.bb` +- `CONFIG_DIR` env var support for Babashka scripts without `bb.edn` +- `bb.edn` for classpath configuration +- Babashka tests in `scripts/test` - ClojureScript (Node.js) implementation of `tech.config.core` - shadow-cljs test setup for ClojureScript tests - `build.clj` and `VERSION` file for tools.build-based releases @@ -16,6 +20,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Updated Clojure dependency to 1.12.4 ### Removed +- `.lein-env` and `.boot-env` file reading from environ (legacy Leiningen/Boot support) - `project.clj` (replaced by `deps.edn`) - `pom.xml` (generated by tools.build at release time) - `.travis.yml` diff --git a/README.md b/README.md index 9e5d5df..e732229 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # tech.config -A Clojure and ClojureScript configuration library that abstracts configuration -from files and environment variables. +A Clojure, ClojureScript, and Babashka configuration library that abstracts +configuration from files and environment variables. The library works by reading config files named `*-config.edn` from the resources directory (classpath in Clojure, a configurable directory in ClojureScript/Node). @@ -67,6 +67,20 @@ The ClojureScript version reads config from: (config/print-config) ``` +### Babashka + +The Babashka version reads config from classpath directories (configured via +`:paths` in `bb.edn`) and supports `CONFIG_DIR` for scripts that run without a +`bb.edn`. + +```clojure +#!/usr/bin/env bb +(require '[tech.config.core :as config]) + +(config/get-config :my-setting) +(config/get-config :my-setting "default-value") +``` + ### Precedence Hierarchy #### Clojure @@ -134,10 +148,16 @@ npx shadow-cljs compile test node target/test.js ``` -Or via deps.edn: +#### Running Babashka tests + +```bash +bb test/bb_test_runner.clj +``` + +#### Running all tests ```bash -clojure -M:test-cljs && node target/test.js +scripts/test ``` #### Releasing diff --git a/bb.edn b/bb.edn new file mode 100644 index 0000000..da629ce --- /dev/null +++ b/bb.edn @@ -0,0 +1 @@ +{:paths ["src" "resources" "test" "test/resources"]} diff --git a/scripts/install b/scripts/install index b3885b8..746cdbf 100755 --- a/scripts/install +++ b/scripts/install @@ -125,7 +125,8 @@ elif [ "$_os" = "linux" ]; then else spin "Installing aws-cli (latest)" _tmpdir=$(mktemp -d) - curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \ + _aws_arch=$([ "$_arch" = "arm64" ] && echo "aarch64" || echo "x86_64") + curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-${_aws_arch}.zip" \ -o "$_tmpdir/awscliv2.zip" unzip -q "$_tmpdir/awscliv2.zip" -d "$_tmpdir" sudo "$_tmpdir/aws/install" \ @@ -158,6 +159,31 @@ else ok "vault already installed ($(vault version | head -1))" fi +# ── Babashka ────────────────────────────────────────────────────────────────── + +header "Babashka" + +if ! command -v bb &>/dev/null; then + bb_version=$(curl -fsSL "https://api.github.com/repos/babashka/babashka/releases/latest" \ + | sed -n 's/.*"tag_name": "v\([^"]*\)".*/\1/p') + spin "Downloading babashka ${bb_version}" + _tmpdir=$(mktemp -d) + _bb_os=$([ "$_os" = "darwin" ] && echo "macos" || echo "linux") + _bb_arch=$([ "$_arch" = "arm64" ] && echo "aarch64" || echo "amd64") + # Linux aarch64 only ships as -static; amd64 and macOS have non-static builds. + _bb_suffix="" + [[ "$_bb_os" == "linux" && "$_bb_arch" == "aarch64" ]] && _bb_suffix="-static" + curl -fsSL \ + "https://github.com/babashka/babashka/releases/download/v${bb_version}/babashka-${bb_version}-${_bb_os}-${_bb_arch}${_bb_suffix}.tar.gz" \ + -o "$_tmpdir/bb.tar.gz" + tar xzf "$_tmpdir/bb.tar.gz" -C "$_tmpdir" + _install_bin "$_tmpdir/bb" bb + rm -rf "$_tmpdir" + clear_spin; ok "babashka ${bb_version} installed" +else + ok "babashka already installed ($(bb --version))" +fi + if [ "$_no_deps" = false ]; then # ── npm deps ────────────────────────────────────────────────────────────────── diff --git a/scripts/test b/scripts/test index 7c18686..1f47a9d 100755 --- a/scripts/test +++ b/scripts/test @@ -12,4 +12,7 @@ npm install --silent npx shadow-cljs compile test node target/test.js +header "Babashka tests" +bb test/bb_test_runner.clj + ok "All tests passed" diff --git a/src/tech/config/core.clj b/src/tech/config/core.clj deleted file mode 100644 index 05f9a95..0000000 --- a/src/tech/config/core.clj +++ /dev/null @@ -1,275 +0,0 @@ -(ns tech.config.core - (:require [tech.config.environ :refer [read-env]] - [clojure.java.classpath :as cp] - [clojure.set :as set] - [clojure.edn :as edn] - [clojure.java.io :as io] - [clojure.pprint :as pprint]) - (:import [java.io File] - [java.util.jar JarFile])) - -(def ^:dynamic *config-map* nil) -(def ^:dynamic *config-sources* {}) -(def ^:dynamic *config-keys* #{}) ;; Only keys found in config _files_. - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Helper functions -(defn- input-stream-to-map - "Helper function for reading a classpath jar into a map." - [stream] - (with-open [^java.io.BufferedReader s stream] - (edn/read (java.io.PushbackReader. s)))) - -(defn- is-config-file? - [^java.io.File file] - (.contains (.getName file) "config.edn")) - -(defn- is-config-jarentry? - [^java.util.jar.JarEntry entry] - (.contains (.getName entry) "config.edn")) - -(defn- coercing-merge - "Takes two maps and merges the source into the dest while trying to coerce - values into the type of the destination map. This is so that a base config - (e.g. in a library) can specify types and they can be overwritten by strings - from the outside (e.g. either via the command line or the environment) and - have those values become the correct type." - [dest src] - (reduce (fn [dest-map [k v]] - (if-not (contains? dest-map k) - (assoc dest-map k v) - (let [d (get dest-map k) - bad-type-fn (fn [x type_] - (->> (format "Config value: %s should be a %s." x type_) - (IllegalArgumentException.) - (throw)))] - (->> (cond - (nil? d) v - (string? d) (str v) - (float? d) (Double. (str v)) - (integer? d) (Integer. (str v)) - (keyword? d) (if (and (string? v) (not-empty v) (= \: (first v))) - (keyword (subs v 1)) - (keyword v)) - (instance? Boolean d) (boolean (Boolean. (str v))) - (map? d) (cond - (map? v) v - (string? v) (->> (edn/read-string v) - ((fn [x] (if (map? x) x - (bad-type-fn x "map"))))) - :default (bad-type-fn v "map")) - (seq? d) (cond - (seq? v) v - (string? v) (->> (edn/read-string v) - ((fn [x] (if (seq? x) x - (bad-type-fn x "seq"))))) - :default (bad-type-fn v "seq")) - (vector? d) (cond - (vector? v) v - (string? v) (->> (edn/read-string v) - ((fn [x] (if (vector? x) x - (bad-type-fn x "seq"))))) - :default (bad-type-fn v "seq")) - :default (edn/read-string v)) - (assoc dest-map k))))) - dest - src)) - -(defn- get-config-streams - "Returns BufferedReaders for *-config.edn files found in jar-files." - [jar-files] - (->> jar-files - (map (fn [^java.util.jar.JarFile jarfile] - [jarfile - (filter is-config-jarentry? - (enumeration-seq (.entries jarfile)))])) - (filter #(> (count (second %)) 0)) - ;; Flatten the config map (make sure that all of the values map to their source) - ((fn [m] - (for [[k v] m - val_ v] - [k val_]))) - (mapv (fn [[^java.util.jar.JarFile jarfile ^java.util.jar.JarEntry jarentry]] - [(str (.getName jarfile) "/" (.getName jarentry)) - (io/reader (.getInputStream jarfile jarentry))])))) - -(defn classpath-directories - [] - (->> (cp/system-classpath) - (filter #(.isDirectory ^java.io.File %)))) - - -(defn classpath-jarfiles - [] - (->> (cp/system-classpath) - (filter cp/jar-file?) - (map #(JarFile. ^File %)))) - - -(defn- file-config - "Loops through all of the .edn files in the jars as well as resources and - coerce-merges them reverse alphabetically with app-config and user-config - taking precedence over the remaining, respectively." - [] - (let [short-name-fn (partial re-find #"[^\/]+$") - move-to-end-fn (fn [entry coll] - (let [m (group-by #(= (short-name-fn (first %)) entry) coll)] - (concat (get m false) (get m true)))) - update-config-sources? (empty? *config-sources*)] - (->> (classpath-directories) - (map file-seq) - (flatten) - (filter is-config-file?) - (map (fn [^java.io.File f] [(.getName f) (io/reader f)])) - (concat (get-config-streams (classpath-jarfiles))) - (sort-by (comp short-name-fn first)) - (reverse) - (move-to-end-fn "app-config.edn") - (move-to-end-fn "user-config.edn") - (reduce (fn [eax [path file]] - (let [m (input-stream-to-map file)] - (when update-config-sources? - (doseq [[k _] m] - (alter-var-root #'*config-keys* #(conj % k)) - (alter-var-root #'*config-sources* #(assoc % k (short-name-fn path))))) - (coercing-merge eax m))) {})))) - -(defn- build-config - "Squashes the environment onto the config-*.edn files." - ([config-map] - (let [env (read-env) - final-map (coercing-merge config-map env) - print-map (->> final-map - (filter #(*config-keys* (first %))) - (into {}))] - (doseq [k (set/intersection (set (keys env)) - (set (keys print-map)))] - (alter-var-root #'*config-sources* #(assoc % k "environment"))) - final-map)) - ([] - (build-config (file-config)))) - -(defn get-config-map - [] - (when-not *config-map* - (alter-var-root #'*config-map* (fn [_] (build-config)))) - *config-map*) - -(defn reload-config! - "Refreshes the config (e.g. re-reading .edn files)" - [] - (alter-var-root #'*config-map* (fn [_] nil))) - - -(defmacro static-configuration - "Macro meant to be used during AOT compile to define the jar and classpath based - configuration once into a variable. - - Example: - -```clojure - (def static-config (config/static-configuration)) - (config/set-static-config! static-config) -```" - [] - (let [config-map (file-config) - config-sources *config-sources* - config-keys *config-keys*] - `{:config-sources ~config-sources - :config-keys ~config-keys - :config-map ~config-map})) - - -(defn set-static-configuration! - "Given a map of static configuration information, combine with environment variables - and set the config global vars. The outcome of this should be identical to - calling (build-config) when no config information has been requested yet. - - This is meant to be used with the `static-configuration` macro." - [static-config] - (alter-var-root #'*config-map* (constantly (build-config (:config-map static-config)))) - (alter-var-root #'*config-keys* (constantly (:config-keys static-config))) - (alter-var-root #'*config-sources* (constantly (:config-sources static-config))) - :ok) - - -(defn get-configurable-options - "This function returns all keys that are specified in .edn files, excluding - the automatic variables such as os-*." - [] - (-> (get-config-map) - (keys) - (set) - (set/difference #{:os-arch :os-name :os-version}))) - -(defn get-config-table-str - "Returns a nice string representation of the current config map." - [& {:keys [no-redact redact-keys] - :or {no-redact false}}] - (let [table (->> (get-config-map) - (filter #(*config-keys* (first %))) - (sort-by first) - (map (fn [[k v]] - {:key k - :value v - :source (get *config-sources* k)}))) - key-width (min 30 (apply max (map (comp count str :key) table))) - val-width (min 30 (apply max (map (comp count str :value) table))) - src-width (min 20 (apply max (map (comp count str :source) table)))] - (with-out-str - (with-bindings {#'pprint/*print-right-margin* nil} - (println (format (str "%-" key-width "s %-" val-width "s %-" src-width "s") "Key" "Value" "Source")) - (println (apply str (repeat (+ key-width val-width src-width 2) "-"))) - (doseq [{:keys [key value source]} table] - (let [print-value (cond (and (not no-redact) - redact-keys - ((set redact-keys) key)) - "[REDACTED]" - (and (not no-redact) - (not redact-keys) - (or (.contains (str key) "secret") - (.contains (str key) "private"))) - "[REDACTED]" - :default (if (string? value) - (format "\"%s\"" value) - value))] - (println (format (str "%-" key-width "s %-" val-width "s %-" src-width "s") - key print-value source)))))))) - -(defn unchecked-get-config - "Get app config. Unlike `get-config`, doesn't coerce arguments and can return nil for missing config." - [k] - ((get-config-map) k)) - -(defn get-config - "Get app config. Accepts a key such as \"PORT\" or :port." - ([k] - (let [retval (unchecked-get-config k)] - (when (nil? retval) - (throw (IllegalArgumentException. (format "Missing config value: %s" k)))) - retval)) - ([k read-string?] - (let [raw-config (get-config k)] - (if (and (string? raw-config) read-string?) - (edn/read-string raw-config) - raw-config)))) - -(defn set-config! - "Very dangerous, but useful during testing. Set a config value. - See: `with-config`" - [key value] - (let [old-map (get-config-map)] - (alter-var-root #'*config-map* assoc key value) - (old-map key))) - -(defmacro with-config - [config-key-vals & body] - `(let [new-map# (#'tech.config.core/coercing-merge (get-config-map) (apply hash-map ~config-key-vals)) - new-keys# (take-nth 2 ~config-key-vals) - new-sources# (->> new-keys# - (map (fn [new-var#] [new-var# "with-config"])) - (into {}))] - (with-bindings {#'*config-map* new-map# - #'*config-sources* (merge *config-sources* new-sources#) - #'*config-keys* (set/union *config-keys* (set new-keys#))} - ~@body))) diff --git a/src/tech/config/core.cljc b/src/tech/config/core.cljc new file mode 100644 index 0000000..a56c14c --- /dev/null +++ b/src/tech/config/core.cljc @@ -0,0 +1,268 @@ +(ns tech.config.core + (:require [tech.config.environ :refer [read-env]] + [tech.config.files :as files] + #?(:clj [clojure.set :as set]) + #?(:clj [clojure.edn :as edn] + :cljs [cljs.reader :as edn]) + #?(:clj [clojure.pprint :as pprint]) + [clojure.string :as str])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Platform helpers — tiny wrappers over parsing differences + +(defn- numeric? [v] + #?(:clj (or (float? v) (integer? v)) + :cljs (number? v))) + +(defn- parse-number + "Parses v as a number, preserving the type of base (int vs float) on JVM." + [v base] + #?(:clj (if (integer? base) + (Integer/parseInt (str v)) + (Double/parseDouble (str v))) + :cljs (let [n (js/parseFloat (str v))] + (if (js/isNaN n) + (throw (ex-info (str "Config value should be a number: " v) {:value v})) + n)))) + +(defn- parse-bool + "Parses v as a boolean." + [v] + #?(:clj (Boolean/parseBoolean (str v)) + :cljs (cond + (boolean? v) v + (string? v) (= (str/lower-case v) "true") + :else (boolean v)))) + +(defn- char-colon + "The colon character, platform-appropriate for comparison with (first s)." + [] + #?(:clj \: :cljs ":")) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Coercing merge — fully cross-platform + +(defn- coercing-merge + "Takes two maps and merges the source into the dest while trying to coerce + values into the type of the destination map. This is so that a base config + (e.g. in a library) can specify types and they can be overwritten by strings + from the outside (e.g. either via the command line or the environment) and + have those values become the correct type." + [dest src] + (reduce (fn [dest-map [k v]] + (if-not (contains? dest-map k) + (assoc dest-map k v) + (let [d (get dest-map k) + bad-type (fn [x type_] + (throw (ex-info (str "Config value: " x " should be a " type_ ".") + {:key k :value x :expected-type type_})))] + (->> (cond + (nil? d) v + (string? d) (str v) + (numeric? d) (parse-number v d) + (keyword? d) (if (and (string? v) (not-empty v) (= (first v) (char-colon))) + (keyword (subs v 1)) + (keyword v)) + (boolean? d) (parse-bool v) + (map? d) (cond + (map? v) v + (string? v) (let [x (edn/read-string v)] + (if (map? x) x (bad-type x "map"))) + :else (bad-type v "map")) + (sequential? d) (cond + (sequential? v) v + (string? v) (let [x (edn/read-string v)] + (if (sequential? x) x (bad-type x "sequential"))) + :else (bad-type v "sequential")) + :else (if (string? v) (edn/read-string v) v)) + (assoc dest-map k))))) + dest + src)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; File config — shared ordering, delegates discovery to tech.config.files + +(defn- order-config-files + "Sorts config file pairs reverse-alphabetically with app-config and + user-config taking precedence (applied last)." + [file-pairs] + (let [short-name (partial re-find #"[^\/]+$") + move-to-end (fn [entry coll] + (let [m (group-by #(= (short-name (first %)) entry) coll)] + (concat (get m false) (get m true))))] + (->> file-pairs + (sort-by (comp short-name first)) + reverse + (move-to-end "app-config.edn") + (move-to-end "user-config.edn")))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Config state — JVM/bb use dynamic vars, CLJS uses atom + +#?(:clj (def ^:dynamic *config-map* nil)) +#?(:clj (def ^:dynamic *config-sources* {})) +#?(:clj (def ^:dynamic *config-keys* #{})) + +(defn- file-config + "Reads and merges all config files, tracking sources and keys." + [] + (let [ordered (order-config-files (files/config-file-maps))] + #?(:clj + (reduce (fn [cfg [path file-map]] + (let [short-name (re-find #"[^\/]+$" path)] + (doseq [[k _] file-map] + (alter-var-root #'*config-keys* conj k) + (alter-var-root #'*config-sources* assoc k short-name)) + (coercing-merge cfg file-map))) + {} ordered) + + :cljs + (reduce (fn [[cfg src keys] [filename file-map]] + [(coercing-merge cfg file-map) + (reduce (fn [s k] (assoc s k filename)) src (cljs.core/keys file-map)) + (into keys (cljs.core/keys file-map))]) + [{} {} #{}] ordered)))) + +(defn- build-config + "Squashes the environment onto the config files." + #?(:clj + ([config-map] + (let [env (read-env) + final-map (coercing-merge config-map env) + print-map (->> final-map + (filter #(*config-keys* (first %))) + (into {}))] + (doseq [k (set/intersection (set (keys env)) + (set (keys print-map)))] + (alter-var-root #'*config-sources* assoc k "environment")) + final-map)) + :cljs + ([config-map env-src-map] + (let [env (read-env)] + [(coercing-merge config-map env) + (reduce (fn [s k] (assoc s k "environment")) env-src-map (keys env))]))) + ([] + #?(:clj (build-config (file-config)) + :cljs (let [[cfg src keys] (file-config) + [final-cfg final-src] (build-config cfg src)] + [final-cfg final-src keys])))) + +#?(:cljs + (defonce ^:private state + (let [[cfg src keys] (build-config)] + (atom {:config cfg :sources src :file-keys keys})))) + +(defn get-config-map [] + #?(:clj (do (when-not *config-map* + (alter-var-root #'*config-map* (fn [_] (build-config)))) + *config-map*) + :cljs (:config @state))) + +(defn reload-config! + "Refreshes the config (e.g. re-reading .edn files)" + [] + #?(:clj (alter-var-root #'*config-map* (fn [_] nil)) + :cljs (let [[cfg src keys] (build-config)] + (reset! state {:config cfg :sources src :file-keys keys})))) + +(defn get-configurable-options + "Returns all keys specified in .edn files." + [] + #?(:clj (-> (get-config-map) keys set (set/difference #{:os-arch :os-name :os-version})) + :cljs (:file-keys @state))) + +(defn unchecked-get-config + "Get app config. Unlike `get-config`, can return nil for missing config." + [k] + (get (get-config-map) k)) + +(defn get-config + "Get app config. Accepts a key such as :port. + Single arity throws if key is missing. Two-arity returns default." + ([k] + (let [v (unchecked-get-config k)] + (when (nil? v) + (throw (ex-info (str "Missing config value: " k) {:key k}))) + v)) + ([k default] + (let [v (unchecked-get-config k)] + (if (nil? v) default v)))) + +(defn set-config! + "Very dangerous, but useful during testing. Set a config value." + [key value] + (let [old-val (unchecked-get-config key)] + #?(:clj (alter-var-root #'*config-map* assoc key value) + :cljs (swap! state assoc-in [:config key] value)) + old-val)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Config display + +(defn- sensitive? [k] + (let [s (str k)] + (or (str/includes? s "secret") + (str/includes? s "private") + (str/includes? s "password") + (str/includes? s "token")))) + +(defn get-config-table-str + "Returns a string representation of the current config map." + [& {:keys [no-redact redact-keys] + :or {no-redact false}}] + (let [#?@(:clj [config-keys *config-keys* + sources *config-sources*] + :cljs [config-keys (:file-keys @state) + sources (:sources @state)]) + config (get-config-map) + rows (->> config-keys sort + (map (fn [k] {:key k :value (get config k) :source (get sources k "unknown")}))) + col (fn [w s] (let [s (str s)] (str s (apply str (repeat (max 1 (- w (count s))) " ")))))] + (str (col 30 "Key") (col 30 "Value") "Source\n" + (apply str (repeat 80 "-")) "\n" + (apply str + (map (fn [{:keys [key value source]}] + (let [pv (cond (and (not no-redact) redact-keys ((set redact-keys) key)) "[REDACTED]" + (and (not no-redact) (not redact-keys) (sensitive? key)) "[REDACTED]" + (string? value) (str "\"" value "\"") + :else value)] + (str (col 30 key) (col 30 pv) source "\n"))) + rows))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; JVM-only features + +#?(:clj + (defmacro static-configuration + "Macro meant to be used during AOT compile to define the jar and classpath + based configuration once into a variable." + [] + (let [config-map (file-config) + config-sources *config-sources* + config-keys *config-keys*] + `{:config-sources ~config-sources + :config-keys ~config-keys + :config-map ~config-map}))) + +#?(:clj + (defn set-static-configuration! + "Given a map of static configuration information, combine with environment + variables and set the config global vars." + [static-config] + (alter-var-root #'*config-map* (constantly (build-config (:config-map static-config)))) + (alter-var-root #'*config-keys* (constantly (:config-keys static-config))) + (alter-var-root #'*config-sources* (constantly (:config-sources static-config))) + :ok)) + +#?(:clj + (defmacro with-config + [config-key-vals & body] + `(let [new-map# (#'tech.config.core/coercing-merge (get-config-map) (apply hash-map ~config-key-vals)) + new-keys# (take-nth 2 ~config-key-vals) + new-sources# (->> new-keys# + (map (fn [new-var#] [new-var# "with-config"])) + (into {}))] + (with-bindings {#'*config-map* new-map# + #'*config-sources* (merge *config-sources* new-sources#) + #'*config-keys* (set/union *config-keys* (set new-keys#))} + ~@body)))) diff --git a/src/tech/config/core.cljs b/src/tech/config/core.cljs deleted file mode 100644 index a4b687c..0000000 --- a/src/tech/config/core.cljs +++ /dev/null @@ -1,165 +0,0 @@ -(ns tech.config.core - (:require ["fs" :as fs] - ["path" :as path] - [cljs.reader :as edn] - [clojure.string :as str] - [tech.config.environ :refer [read-env]])) - -(def ^:private config-dir - (or (.. js/process -env -CONFIG_DIR) "resources")) - -(defn- coercing-merge - "Takes two maps and merges the source into the dest while trying to coerce - values into the type of the destination map. This is so that a base config - (e.g. in a library) can specify types and they can be overwritten by strings - from the outside (e.g. either via the command line or the environment) and - have those values become the correct type." - [dest src] - (reduce (fn [dest-map [k v]] - (if-not (contains? dest-map k) - (assoc dest-map k v) - (let [d (get dest-map k)] - (->> (cond - (nil? d) v - (string? d) (str v) - (number? d) (if (string? v) - (let [n (js/parseFloat v)] - (if (js/isNaN n) - (throw (ex-info (str "Config value: " v " should be a number.") {:key k})) - n)) - v) - (keyword? d) (if (and (string? v) (not-empty v) (= (first v) ":")) - (keyword (subs v 1)) - (keyword v)) - (boolean? d) (cond - (boolean? v) v - (string? v) (= (str/lower-case v) "true") - :else (boolean v)) - (map? d) (cond - (map? v) v - (string? v) (let [x (edn/read-string v)] - (if (map? x) x - (throw (ex-info (str "Config value: " v " should be a map.") {:key k})))) - :else (throw (ex-info (str "Config value: " v " should be a map.") {:key k}))) - (vector? d) (cond - (vector? v) v - (string? v) (let [x (edn/read-string v)] - (if (vector? x) x - (throw (ex-info (str "Config value: " v " should be a vector.") {:key k})))) - :else (throw (ex-info (str "Config value: " v " should be a vector.") {:key k}))) - :else (if (string? v) - (edn/read-string v) - v)) - (assoc dest-map k))))) - dest - src)) - -(defn- read-files - "Reads *-config.edn files from the config directory, sorted reverse - alphabetically with app-config and user-config taking precedence." - [] - (if-not (.existsSync fs config-dir) - [{} {} #{}] - (let [move-to-end-fn (fn [entry coll] - (let [grouped (group-by #(= (first %) entry) coll)] - (concat (get grouped false) (get grouped true))))] - (->> (.readdirSync fs config-dir) - js->clj - (filter #(str/ends-with? % "config.edn")) - sort - reverse - (map (fn [filename] [filename (path/join config-dir filename)])) - (move-to-end-fn "app-config.edn") - (move-to-end-fn "user-config.edn") - (reduce (fn [[cfg src keys] [filename full-path]] - (let [file-map (edn/read-string (.readFileSync fs full-path "utf8")) - file-keys (cljs.core/keys file-map)] - [(coercing-merge cfg file-map) - (merge src (into {} (map #(vector % filename) file-keys))) - (into keys file-keys)])) - [{} {} #{}]))))) - -(defonce ^:private state - (let [[file-cfg file-src file-keys] (read-files) - env (read-env) - env-src (into {} (map #(vector % "environment") (keys env)))] - (atom {:config (coercing-merge file-cfg env) - :sources (merge file-src env-src) - :file-keys (into file-keys (keys file-cfg))}))) - -(defn get-config-map - [] - (:config @state)) - -(defn reload-config! - "Refreshes the config (e.g. re-reading .edn files)" - [] - (let [[file-cfg file-src file-keys] (read-files) - env (read-env) - env-src (into {} (map #(vector % "environment") (keys env)))] - (reset! state {:config (coercing-merge file-cfg env) - :sources (merge file-src env-src) - :file-keys (into file-keys (keys file-cfg))}))) - -(defn get-configurable-options - "Returns all keys specified in .edn files." - [] - (:file-keys @state)) - -(defn unchecked-get-config - "Get app config. Unlike `get-config`, can return nil for missing config." - [k] - (get (:config @state) k)) - -(defn get-config - "Get app config. Accepts a key such as :port." - ([k] - (let [v (unchecked-get-config k)] - (when (nil? v) - (throw (ex-info (str "Missing config value: " k) {:key k}))) - v)) - ([k default] - (get (:config @state) k default))) - -(defn set-config! - "Very dangerous, but useful during testing. Set a config value." - [key value] - (let [old-val (unchecked-get-config key)] - (swap! state assoc-in [:config key] value) - old-val)) - -(defn- sensitive? [k] - (let [s (name k)] - (some #(str/includes? s %) ["secret" "private" "password" "token"]))) - -(defn get-config-table-str - "Returns a string representation of the current config map." - [& {:keys [no-redact redact-keys] - :or {no-redact false}}] - (let [{:keys [config sources file-keys]} @state - col (fn [w s] (let [s (str s)] (str s (apply str (repeat (max 1 (- w (count s))) " "))))) - rows (->> file-keys sort - (map (fn [k] {:key k - :value (get config k) - :source (get sources k "unknown")})))] - (str (col 30 "Key") (col 30 "Value") "Source\n" - (apply str (repeat 80 "-")) "\n" - (apply str - (map (fn [{:keys [key value source]}] - (let [print-value (cond (and (not no-redact) - redact-keys - ((set redact-keys) key)) - "[REDACTED]" - (and (not no-redact) - (not redact-keys) - (sensitive? key)) - "[REDACTED]" - (string? value) (str "\"" value "\"") - :else value)] - (str (col 30 key) (col 30 print-value) source "\n"))) - rows))))) - -(defn print-config - "Prints the current config table." - [] - (println (get-config-table-str))) diff --git a/src/tech/config/environ.cljc b/src/tech/config/environ.cljc index 5862799..c22fb70 100644 --- a/src/tech/config/environ.cljc +++ b/src/tech/config/environ.cljc @@ -1,17 +1,11 @@ (ns tech.config.environ - "Copy of environ that exposes 'read-env'" - (:require #?(:clj [clojure.edn :as edn] :cljs [cljs.reader :as edn]) - #?(:clj [clojure.java.io :as io]) - #?(:cljs [goog.object :as obj]) + "Reads environment variables and system properties into a config map." + (:require #?(:cljs [goog.object :as obj]) [clojure.string :as str])) - #?(:cljs (def ^:private nodejs? (exists? js/require))) -#?(:cljs (def ^:private fs - (when nodejs? (js/require "fs")))) - #?(:cljs (def ^:private process (when nodejs? (js/require "process")))) @@ -21,17 +15,6 @@ (str/replace "." "-") (keyword))) -(defn- sanitize-key [k] - (let [s (keywordize (name k))] - (if-not (= k s) (println "Warning: environ key" k "has been corrected to" s)) - s)) - -(defn- sanitize-val [k v] - (if (string? v) - v - (do (println "Warning: environ value" (pr-str v) "for key" k "has been cast to string") - (str v)))) - (defn- read-system-env [] (->> #?(:clj (System/getenv) :cljs (zipmap (obj/getKeys (.-env process)) @@ -44,18 +27,6 @@ (map (fn [[k v]] [(keywordize k) v])) (into {})))) -(defn- slurp-file [f] - #?(:clj (when-let [f (io/file f)] - (when (.exists f) - (slurp f))) - :cljs (when (.existsSync fs f) - (str (.readFileSync fs f))))) - -(defn- read-env-file [f] - (when-let [content (slurp-file f)] - (into {} (for [[k v] (edn/read-string content)] - [(sanitize-key k) (sanitize-val k v)])))) - (defn- warn-on-overwrite [ms] (doseq [[k kvs] (group-by key (apply concat ms)) :let [vs (map val kvs)] @@ -69,12 +40,8 @@ (defn read-env [] #?(:clj (merge-env - (read-env-file ".lein-env") - (read-env-file (io/resource ".boot-env")) (read-system-env) (read-system-props)) :cljs (if nodejs? - (merge-env - (read-env-file ".lein-env") - (read-system-env)) + (read-system-env) {}))) diff --git a/src/tech/config/files.bb b/src/tech/config/files.bb new file mode 100644 index 0000000..df39adb --- /dev/null +++ b/src/tech/config/files.bb @@ -0,0 +1,33 @@ +(ns tech.config.files + "Babashka file discovery: classpath directories only (no jar scanning)." + (:require [clojure.edn :as edn] + [clojure.java.io :as io] + [clojure.string :as str]) + (:import [java.io File])) + +(defn- config-file? [^File f] + (and (.isFile f) + (.contains (.getName f) "config.edn"))) + +(defn- classpath-directories [] + (let [cp-dirs (->> (str/split (System/getProperty "java.class.path") #":") + (map #(File. ^String %)) + (filter #(.isDirectory ^File %))) + cp-set (set (map #(.getCanonicalPath ^File %) cp-dirs)) + ;; CONFIG_DIR allows scripts without bb.edn to specify a config directory + extra (when-let [f (some-> (System/getenv "CONFIG_DIR") (File.))] + (when (and (.isDirectory f) (not (cp-set (.getCanonicalPath f)))) + [f]))] + (concat cp-dirs extra))) + +(defn config-file-maps + "Returns an unordered seq of [filename config-map] pairs from + classpath directories." + [] + (->> (classpath-directories) + (mapcat file-seq) + (filter config-file?) + (map (fn [^File f] + [(.getName f) + (with-open [rdr (io/reader f)] + (edn/read (java.io.PushbackReader. rdr)))])))) diff --git a/src/tech/config/files.clj b/src/tech/config/files.clj new file mode 100644 index 0000000..56f1e30 --- /dev/null +++ b/src/tech/config/files.clj @@ -0,0 +1,50 @@ +(ns tech.config.files + "JVM file discovery: classpath directories + jar scanning." + (:require [clojure.java.classpath :as cp] + [clojure.edn :as edn] + [clojure.java.io :as io]) + (:import [java.io File] + [java.util.jar JarFile JarEntry])) + +(defn- config-file? [^File f] + (and (.isFile f) + (.contains (.getName f) "config.edn"))) + +(defn- config-jarentry? [^JarEntry entry] + (.contains (.getName entry) "config.edn")) + +(defn- read-edn-stream [stream] + (with-open [^java.io.BufferedReader s stream] + (edn/read (java.io.PushbackReader. s)))) + +(defn- dir-config-files + "Returns [filename config-map] pairs from classpath directories." + [] + (->> (cp/system-classpath) + (filter #(.isDirectory ^File %)) + (mapcat file-seq) + (filter config-file?) + (map (fn [^File f] + [(.getName f) (read-edn-stream (io/reader f))])))) + +(defn- jar-config-files + "Returns [filename config-map] pairs from jars on the classpath." + [] + (let [jars (->> (cp/system-classpath) + (filter cp/jar-file?) + (map #(JarFile. ^File %)))] + (->> jars + (mapcat (fn [^JarFile jarfile] + (->> (enumeration-seq (.entries jarfile)) + (filter config-jarentry?) + (map (fn [^JarEntry entry] + [(str (.getName jarfile) "/" (.getName entry)) + (read-edn-stream + (io/reader (.getInputStream jarfile entry)))]))))) + vec))) + +(defn config-file-maps + "Returns an unordered seq of [filename config-map] pairs from all + classpath directories and jars." + [] + (concat (dir-config-files) (jar-config-files))) diff --git a/src/tech/config/files.cljs b/src/tech/config/files.cljs new file mode 100644 index 0000000..1d66ecd --- /dev/null +++ b/src/tech/config/files.cljs @@ -0,0 +1,22 @@ +(ns tech.config.files + "Node.js file discovery: reads *-config.edn from a filesystem directory." + (:require ["fs" :as fs] + ["path" :as path] + [cljs.reader :as edn] + [clojure.string :as str])) + +(def ^:private config-dir + (or (.. js/process -env -CONFIG_DIR) "resources")) + +(defn config-file-maps + "Returns an unordered seq of [filename config-map] pairs from + the config directory." + [] + (when (.existsSync fs config-dir) + (->> (.readdirSync fs config-dir) + js->clj + (filter #(str/ends-with? % "config.edn")) + (map (fn [filename] + [filename + (edn/read-string + (.readFileSync fs (path/join config-dir filename) "utf8"))]))))) diff --git a/test/bb_test_runner.clj b/test/bb_test_runner.clj new file mode 100644 index 0000000..dfd0efc --- /dev/null +++ b/test/bb_test_runner.clj @@ -0,0 +1,14 @@ +(ns bb-test-runner + (:require [clojure.test :as t])) + +;; Set system properties that the JVM tests get via -D flags +(System/setProperty "overwrite" "80") +(System/setProperty "env-config-overwrite" "true") +(System/setProperty "complex-type-env-overwrite-map" "{:a 1 :b 3}") +(System/setProperty "complex-type-env-overwrite-vec" "[:c :b :a]") +(System/setProperty "complex-type-env-overwrite-seq" "(:c :b :a)") + +(require 'tech.config.config-test) + +(let [{:keys [fail error]} (t/run-tests 'tech.config.config-test)] + (System/exit (if (zero? (+ fail error)) 0 1))) diff --git a/test/tech/config/config_test.clj b/test/tech/config/config_test.clj index a385b1c..e522e75 100644 --- a/test/tech/config/config_test.clj +++ b/test/tech/config/config_test.clj @@ -8,14 +8,14 @@ (deftest config-test (testing "Config Test" - (is (string? (get-config :os-arch))) - (is (thrown? IllegalArgumentException (get-config :some-bs-val))))) + (is (string? (get-config :os-arch))) + (is (thrown? Exception (get-config :some-bs-val))))) (deftest types-test (testing "Entries are properly coerced" - (is (integer? (get-config :app-config-overwrite))) - (is (= (get-config :app-config-overwrite) 1)) - (is (= (get-config :user-config-overwrite) 2)))) + (is (integer? (get-config :app-config-overwrite))) + (is (= (get-config :app-config-overwrite) 1)) + (is (= (get-config :user-config-overwrite) 2)))) (deftest with-config-test (testing "Make sure with-config can coerce values properly." @@ -60,10 +60,9 @@ (with-config [:boolean "true"] (is (true? (get-config :boolean))))) -(deftest read-string-test - (with-config [:m "{:a :b}"] - (is (nil? (:a (get-config :m)))) - (is (= :b (:a (get-config :m true)))))) +(deftest default-value-test + (is (= "fallback" (get-config :nonexistent-key "fallback"))) + (is (thrown? Exception (get-config :nonexistent-key)))) (deftest file-order-test (testing "Make sure files are merged in reverse-alphabetical order." @@ -82,8 +81,8 @@ (with-config [:complex-type-map "{:a 1 :b 4}"] (is (= (:b (get-config :complex-type-map)) 4))) - (is (thrown? IllegalArgumentException (with-config [:complex-type-map [:a :b :c]]))) - (is (thrown? IllegalArgumentException (with-config [:complex-type-map "[:a :b :c]"]))))) + (is (thrown? Exception (with-config [:complex-type-map [:a :b :c]]))) + (is (thrown? Exception (with-config [:complex-type-map "[:a :b :c]"]))))) (deftest reload-config-test (testing "Make sure that we can reload the config and get new values from .edn files"