From 184b67b4639fd08b9767fee607977fb19c99bfd0 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 17:32:19 +0100 Subject: [PATCH 01/16] Update deployment and security workflow This commit updates the publish workflow to use the NPM environment instead of the legacy NPM-OIDC context and bumps the Socket CLI version in the security reachability workflow from 2.7.2 to 2.8.1. --- .github/workflows/Publish.yml | 4 ++-- .github/workflows/Security-Reachability.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Publish.yml b/.github/workflows/Publish.yml index 6640695..c5c379a 100644 --- a/.github/workflows/Publish.yml +++ b/.github/workflows/Publish.yml @@ -40,8 +40,8 @@ jobs: contents: none id-token: write - # The deploy step runs in the NPM OIDC authorized context - environment: NPM-OIDC + # The deploy step runs in the NPM authorized context + environment: NPM # Set of commands to run for the build job steps: diff --git a/.github/workflows/Security-Reachability.yml b/.github/workflows/Security-Reachability.yml index 511e43a..a45f5f2 100644 --- a/.github/workflows/Security-Reachability.yml +++ b/.github/workflows/Security-Reachability.yml @@ -80,7 +80,7 @@ jobs: # validate:package:skip-reachability command so workflow validation does not modify its checkout. - name: Install Socket CLI background: true - run: sfw pip install socketsecurity==2.7.2 uv --upgrade + run: sfw pip install socketsecurity==2.8.1 uv --upgrade # Bring job back to sync execution by awaiting for all async jobs to finish before continuing - name: Steps - Convert Back To Synchronous Execution - Packages Updates/Setup From f241021ff214dc21b5f7497b39541e80aab06c78 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 17:54:43 +0100 Subject: [PATCH 02/16] fix: retain package imports map for compiled # aliases The TypeScript path alias is preserved in emitted JavaScript, so Node needs the package imports map to resolve internal #/* module references at runtime. Without it, consumers receive ERR_PACKAGE_IMPORT_NOT_DEFINED. --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index 8fb0989..383e490 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,9 @@ "type": "git", "url": "git+https://github.com/Software-Hardware-Integration-Lab/LogEngine.git" }, + "imports": { + "#/*": "./bin/*" + }, "exports": { ".": { "types": "./bin/index.d.ts", From baa2a96204bb10980ae862b08b61b489b0404f8e Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 17:55:36 +0100 Subject: [PATCH 03/16] Add optional Azure Blob storage support This release adds @azure/storage-blob as an optional peer dependency and dev dependency, and bumps the package version to 0.0.4. It also updates the Vitest toolchain to 5.0.1 and refreshes the lockfile to match the new dependency graph. --- package-lock.json | 609 ++++++++++++++++++++++++++++++++++++++++------ package.json | 15 +- 2 files changed, 542 insertions(+), 82 deletions(-) diff --git a/package-lock.json b/package-lock.json index 14e45d1..8d5735e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,23 +1,247 @@ { "name": "@software-hardware-integration-lab/log-engine", - "version": "0.0.3", + "version": "0.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@software-hardware-integration-lab/log-engine", - "version": "0.0.3", + "version": "0.0.4", "hasInstallScript": true, "dependencies": { "typia": "~12.1.1" }, "devDependencies": { + "@azure/storage-blob": "~12.33.0", "@software-hardware-integration-lab/development-utilities": "~3.0.0", "@types/node": "~26.5.0", - "@vitest/coverage-v8": "~5.0.0", + "@vitest/coverage-v8": "~5.0.1", "ts-patch": "~4.0.1", "typescript": "~6.0.3", - "vitest": "~5.0.0" + "vitest": "~5.0.1" + }, + "peerDependencies": { + "@azure/storage-blob": "~12.33.0" + }, + "peerDependenciesMeta": { + "@azure/storage-blob": { + "optional": true + } + } + }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.5.0.tgz", + "integrity": "sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.7.0.tgz", + "integrity": "sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.6.0.tgz", + "integrity": "sha512-e7lX/dk//F6Qf7BB6PTY4+p2yuOQtyOeHGyapYHNwqSp2OnYpwQt49A/Nin2XmKBQ69pwagR4k/lQBq8lbHQkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.5.9", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.33.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.33.0.tgz", + "integrity": "sha512-2SX8oP8PyblUcAFZSg39c8Ls+tFjavM6sBeV+qpw33mRzRhI/5hrFJmJ/x0H9xx5l6ECPvgSP8uPxqTeVbHNIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.4.1", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.5.0.tgz", + "integrity": "sha512-bttzuhQiCIwrkzjPDA+AtAR7dg19L/CC6ztcqJ5LfvWpXuys9mHp0UQ0udYnoUvv9SCT9KTR5kqFvFr0e6k0lQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.4.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" } }, "node_modules/@babel/code-frame": { @@ -589,6 +813,19 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@oxc-project/types": { "version": "0.149.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.149.0.tgz", @@ -1330,6 +1567,21 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "dev": true, + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@typia/core": { "version": "12.1.1", "resolved": "https://registry.npmjs.org/@typia/core/-/core-12.1.1.tgz", @@ -1367,9 +1619,9 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-5.0.0.tgz", - "integrity": "sha512-toMg6PZGCIa/lQNCDoASrfb1ly4hsUKXFtFYC9kD4t78o5Y6LyNJU7AENt8eHPr3quYdxaxK7hj2mnbFfUk9NA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-5.0.1.tgz", + "integrity": "sha512-FRC8ACiudC3dI6MTplzRSYWHDRnIv2IPfbzs4FdoJNsMal/35sWV8hwIfV8ZcqzSPy+uXHeMVONt9CEqtOU17w==", "dev": true, "license": "MIT", "dependencies": { @@ -1386,8 +1638,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "5.0.0", - "vitest": "5.0.0" + "@vitest/browser": "5.0.1", + "vitest": "5.0.1" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -1419,14 +1671,14 @@ } }, "node_modules/@vitest/mocker": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.0.tgz", - "integrity": "sha512-66PGTMIiVJP3t4a5yxU9qPtf7MdTBs8jmToMvy+HVflB3Yy13WJZTtPePdvU+wjRV02SKK5doLbSA6o9pwOmiA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.1.tgz", + "integrity": "sha512-6K1DoBNAPGvuOcSsGA4D6x+5zEEff/KmOOP3uetT2TrGpVfI+HRHRnJJfKi5ib/g1vx8IYHQD8s0pbJz8WQI7Q==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "0.3.31", - "@vitest/spy": "5.0.0", + "@vitest/spy": "5.0.1", "estree-walker": "^3.0.3", "magic-string": "^1.2.3" }, @@ -1447,9 +1699,9 @@ } }, "node_modules/@vitest/spy": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.0.tgz", - "integrity": "sha512-uy+luWBAPw9XfthoHi5AkfHUnuPYEESjl0p/r+meoBnU8bxg5GDQ3Ey8MjcJ6sqahkL4PFyrvfMJJBw7LbU06g==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.1.tgz", + "integrity": "sha512-rbto/mF/SGERxEgYOek7Xm6B9b+y+mVoo+f4b2LymYO8zM1b7uB5nHuhVMTP2hxdzgxvGiZYGxGIaMvL5y180Q==", "dev": true, "license": "MIT", "funding": { @@ -1479,6 +1731,16 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", @@ -1535,6 +1797,19 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/are-docs-informative": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/are-docs-informative/-/are-docs-informative-0.0.2.tgz", @@ -2264,6 +2539,16 @@ "node": ">=0.10.0" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/expect-type": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", @@ -2295,6 +2580,47 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz", + "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -2511,6 +2837,34 @@ ], "license": "MIT" }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -2527,6 +2881,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/iconv-lite/node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -2609,17 +2969,6 @@ "node": ">=12.0.0" } }, - "node_modules/is-core-module": { - "name": "@socketregistry/is-core-module", - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@socketregistry/is-core-module/-/is-core-module-1.0.11.tgz", - "integrity": "sha512-obrSzvIfJXKQthA3u1RmkjLHuA1QDtLm0SbXJxGs2CQfXZY9Eql5/pBGSV1hIUWKcpdcNphFgnJMC5BITcTXsQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -2662,15 +3011,18 @@ "node": ">=18" } }, - "node_modules/is-unicode-supported": { - "name": "@socketregistry/is-unicode-supported", - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@socketregistry/is-unicode-supported/-/is-unicode-supported-1.0.5.tgz", - "integrity": "sha512-l3wz0cknjyGlI2iCyZxp50FJhtUFXkdZR6CfUU7OfNxE7I4CRBdsvORLgV+JPwqQQErRO/CZgKsbDHefd3puYA==", - "license": "MIT", - "engines": { - "node": ">=18" - } + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" }, "node_modules/isexe": { "version": "2.0.0", @@ -3100,6 +3452,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/log-symbols/node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -3111,13 +3475,13 @@ } }, "node_modules/magic-string": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.2.3.tgz", - "integrity": "sha512-Bpb0W2TbLKOZ7vJnOUnVRGq3WL2p+ISV29M6hYPL1AFCpyKZpdr5ytiXoTSSxRVhg8YW7f65+6gbG8WG6PCa/g==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.1.tgz", + "integrity": "sha512-8lyCu36ErXR0J9uaGKlKQoiLZKmtI63YGLE8G2o9jyRPdr4X47LusSOwgOJOzcVtp81fTAAjxR7BwKz682Jhow==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" + "@jridgewell/sourcemap-codec": "^1.6.0" } }, "node_modules/magicast": { @@ -3181,9 +3545,9 @@ "license": "ISC" }, "node_modules/nanoid": { - "version": "3.3.18", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", - "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", "dev": true, "funding": [ { @@ -3294,6 +3658,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ora/node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -3362,6 +3738,22 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -3531,6 +3923,33 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/resolve/node_modules/hasown": { + "name": "@socketregistry/hasown", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@socketregistry/hasown/-/hasown-1.0.7.tgz", + "integrity": "sha512-MZ5dyXOtiEc7q3801T+2EmKkxrd55BOSQnG8z/8/IkIJzDxqBxGGBKVyixqFm3W657TyUEBfIT9iWgSB6ipFsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/resolve/node_modules/is-core-module": { + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", + "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/restore-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", @@ -3606,26 +4025,6 @@ "tslib": "^2.1.0" } }, - "node_modules/safe-buffer": { - "name": "@socketregistry/safe-buffer", - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@socketregistry/safe-buffer/-/safe-buffer-1.0.9.tgz", - "integrity": "sha512-eV4uYchI1+vQeKpFG+aBlhVQ/AaaPTTXaan+ReiNn/izy8U9hfT4WC8l4g8o8BC3zaeNnsNVxec14hJH/y2y3g==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/safer-buffer": { - "name": "@socketregistry/safer-buffer", - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/@socketregistry/safer-buffer/-/safer-buffer-1.0.10.tgz", - "integrity": "sha512-jbEY37bJn51W9pP1pXxIoGcQbmbi9EQDtnXfWBjGLNvKC1iEyNLOaGm8ee7dN7Z+KgJdQbrrDjjD3HbGeOFC4A==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/semver": { "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", @@ -3733,6 +4132,26 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -3759,6 +4178,22 @@ "node": ">=8" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -4036,17 +4471,17 @@ "license": "MIT" }, "node_modules/vite": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", - "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", + "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", "dev": true, "license": "MIT", "peer": true, "dependencies": { "lightningcss": "^1.33.0", - "picomatch": "^4.0.5", - "postcss": "^8.5.26", - "rolldown": "~1.2.4", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.6", "tinyglobby": "^0.2.17" }, "bin": { @@ -4063,7 +4498,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.4.0 || ^0.5.0", + "@vitejs/devtools": "^0.7.1", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -4115,14 +4550,14 @@ } }, "node_modules/vitest": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.0.tgz", - "integrity": "sha512-gpsMNoRhMjMktVxPtstOH4/PJuPyovVaMDr4oDilXaGH1EcqM2OE96SoHT2VIQ6fTGtTjqmHDrEu2X9RQiXf8Q==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.1.tgz", + "integrity": "sha512-iA95lQbKEkvrtTkdAgnWbXfbipWiiWe/hDl2P5tMi6WFwD76G0NxXAGp/M9EOcYupeGJRr6wppMc7CoA41TQjg==", "dev": true, "license": "MIT", "dependencies": { "@types/chai": "^5.2.2", - "@vitest/mocker": "5.0.0", + "@vitest/mocker": "5.0.1", "chai": "^6.2.2", "es-module-lexer": "^2.3.2", "expect-type": "^1.4.0", @@ -4148,12 +4583,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "5.0.0", - "@vitest/browser-preview": "5.0.0", + "@vitest/browser-playwright": "5.0.1", + "@vitest/browser-preview": "5.0.1", "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", - "@vitest/coverage-istanbul": "5.0.0", - "@vitest/coverage-v8": "5.0.0", - "@vitest/ui": "5.0.0", + "@vitest/coverage-istanbul": "5.0.1", + "@vitest/coverage-v8": "5.0.1", + "@vitest/ui": "5.0.1", "happy-dom": "*", "jsdom": "*", "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" @@ -4263,6 +4698,22 @@ "node": ">=8" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/package.json b/package.json index 383e490..1ea42df 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "0.0.3", + "version": "0.0.4", "name": "@software-hardware-integration-lab/log-engine", "description": "Logging engine with ability to add plugins for any destination", "author": { @@ -51,12 +51,21 @@ "typia": "~12.1.1" }, "devDependencies": { + "@azure/storage-blob": "^12.33.0", "@software-hardware-integration-lab/development-utilities": "~3.0.0", "@types/node": "~26.5.0", - "@vitest/coverage-v8": "~5.0.0", + "@vitest/coverage-v8": "~5.0.1", "ts-patch": "~4.0.1", "typescript": "~6.0.3", - "vitest": "~5.0.0" + "vitest": "~5.0.1" + }, + "peerDependencies": { + "@azure/storage-blob": "^12.33.0" + }, + "peerDependenciesMeta": { + "@azure/storage-blob": { + "optional": true + } }, "overrides": { "function-bind": "npm:@socketregistry/function-bind@^1", From 8cfcfdf1268027baca46e951412a014700189f50 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 17:55:43 +0100 Subject: [PATCH 04/16] Update Security-Reachability.yml --- .github/workflows/Security-Reachability.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Security-Reachability.yml b/.github/workflows/Security-Reachability.yml index a45f5f2..5ca7f4b 100644 --- a/.github/workflows/Security-Reachability.yml +++ b/.github/workflows/Security-Reachability.yml @@ -80,7 +80,7 @@ jobs: # validate:package:skip-reachability command so workflow validation does not modify its checkout. - name: Install Socket CLI background: true - run: sfw pip install socketsecurity==2.8.1 uv --upgrade + run: sfw pip install socketsecurity==2.8.2 uv --upgrade # Bring job back to sync execution by awaiting for all async jobs to finish before continuing - name: Steps - Convert Back To Synchronous Execution - Packages Updates/Setup From 90612e1020043890fe38e9ab6c8d3a4dafb1a0f0 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 17:56:19 +0100 Subject: [PATCH 05/16] Add Azure Storage log destination Introduce an Azure Storage destination plugin that writes operational and audit logs to append blobs, plus shared option types and exported entry points. Add tests covering container setup, hourly blob rotation, disabled streams, validation, oversized records, append failures, and disposal behavior. --- src/index.ts | 7 + .../plugins/AzureStorageDestination.ts | 21 ++ src/plugins/AzureStorageDestination.ts | 326 ++++++++++++++++++ tests/plugins/AzureStorageDestination.test.ts | 238 +++++++++++++ 4 files changed, 592 insertions(+) create mode 100644 src/interfaces/plugins/AzureStorageDestination.ts create mode 100644 src/plugins/AzureStorageDestination.ts create mode 100644 tests/plugins/AzureStorageDestination.test.ts diff --git a/src/index.ts b/src/index.ts index 2bb1f5d..b6e9f07 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,8 @@ export { FileDestination } from './plugins/FileDestination.js'; export { LogAnalyticsDestination } from './plugins/LogAnalyticsDestination.js'; +export { AzureStorageDestination } from './plugins/AzureStorageDestination.js'; + export { LogLevel } from './interfaces/LogEngine.js'; export type { @@ -40,4 +42,9 @@ export type { LogAnalyticsUploaderFactory } from './interfaces/plugins/LogAnalyticsDestination.js'; +export type { + AzureStorageDestinationOptions, + ResolvedAzureStorageDestinationOptions +} from './interfaces/plugins/AzureStorageDestination.js'; + export type { LoggingPluginContract } from './interfaces/plugins/LoggingPlugin.js'; diff --git a/src/interfaces/plugins/AzureStorageDestination.ts b/src/interfaces/plugins/AzureStorageDestination.ts new file mode 100644 index 0000000..2de8d59 --- /dev/null +++ b/src/interfaces/plugins/AzureStorageDestination.ts @@ -0,0 +1,21 @@ +import { DEFAULT_LOGGING_PLUGIN_CONFIGURATION_OPTIONS, type LoggingPluginConfigurationOptions } from './LoggingPlugin.js'; +import type { tags } from 'typia'; + +/** Configuration options for Azure Storage append-blob logging behavior. */ +export interface AzureStorageDestinationOptions extends LoggingPluginConfigurationOptions { + /** Maximum number of UTF-8 bytes permitted in one append-blob write. */ + 'maxAppendBlockBytes'?: number & tags.Minimum<1>; +} + +/** Fully resolved options used internally by the Azure Storage destination. */ +export interface ResolvedAzureStorageDestinationOptions extends LoggingPluginConfigurationOptions { + /** Maximum number of UTF-8 bytes permitted in one append-blob write. */ + 'maxAppendBlockBytes': number & tags.Minimum<1>; +} + +/** Static default options for Azure Storage append-blob logging. */ +export const DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS: ResolvedAzureStorageDestinationOptions = { + ...DEFAULT_LOGGING_PLUGIN_CONFIGURATION_OPTIONS, + // The stable limit supported by all Append Blob service API versions. + 'maxAppendBlockBytes': 4 * 1024 * 1024 +}; diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts new file mode 100644 index 0000000..3f078c0 --- /dev/null +++ b/src/plugins/AzureStorageDestination.ts @@ -0,0 +1,326 @@ +import type { OperationalLog, AuditLog } from '#/interfaces/LogEngine.js'; +import type { AppendBlobClient, ContainerClient } from '@azure/storage-blob'; +import { LoggingPlugin } from './base/LoggingPlugin.js'; +import { DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS, type AzureStorageDestinationOptions, type ResolvedAzureStorageDestinationOptions } from '#/interfaces/plugins/AzureStorageDestination.js'; +import { SerializableAuditLog } from '#/classes/SerializableAuditLog.js'; +import { SerializableOperationalLog } from '#/classes/SerializableOperationalLog.js'; +import { assertGuardEquals } from 'typia'; + +interface LogTypeCollection { + 'activeBlob': AppendBlobClient | undefined; + 'appendQueue': Promise; + 'activeBlobDate': number | undefined; + 'blobContainer': ContainerClient | undefined; +} + +/** + * Azure Storage destination for operational and audit logs. + * Utilizes Azure Blob Storage append blobs for log storage. + */ +export class AzureStorageDestination extends LoggingPlugin { + readonly #operationalCollection: LogTypeCollection | undefined = void 0; + readonly #auditCollection: LogTypeCollection | undefined = void 0; + readonly #appliedOptions: ResolvedAzureStorageDestinationOptions; + #isDisposed = false; + + private constructor( + operationalLogContainer?: ContainerClient, + auditLogContainer?: ContainerClient, + configuration: AzureStorageDestinationOptions = DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS + ) { + super(configuration.id ?? 'AzureStorageDestination'); + + this.#appliedOptions = AzureStorageDestination.#resolveConfigurationOptions(configuration); + + this.#operationalCollection = { + 'activeBlob': void 0, + 'appendQueue': Promise.resolve(), + 'activeBlobDate': void 0, + 'blobContainer': operationalLogContainer + }; + + this.#auditCollection = { + 'activeBlob': void 0, + 'appendQueue': Promise.resolve(), + 'activeBlobDate': void 0, + 'blobContainer': auditLogContainer + }; + } + + /** + * Creates an Azure Storage destination backed by one or both supplied blob containers. + * @param operationalLogContainer Optional Azure Blob container for operational logs. + * @param auditLogContainer Optional Azure Blob container for audit logs. + * @param configuration Optional base configuration that controls log and diagnostic output. + * @returns Initialized Azure Storage destination. + */ + public static async create( + operationalLogContainer?: ContainerClient, + auditLogContainer?: ContainerClient, + configuration: AzureStorageDestinationOptions = DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS + ): Promise { + if (!operationalLogContainer && !auditLogContainer) { + throw new Error('At least one of operationalLogContainer or auditLogContainer must be provided.'); + } + + if (operationalLogContainer && !AzureStorageDestination.#isContainerClient(operationalLogContainer)) { + throw new TypeError('operationalLogContainer must be an Azure Blob ContainerClient.'); + } + + if (auditLogContainer && !AzureStorageDestination.#isContainerClient(auditLogContainer)) { + throw new TypeError('auditLogContainer must be an Azure Blob ContainerClient.'); + } + + const resolvedOptions = AzureStorageDestination.#resolveConfigurationOptions(configuration); + + if (operationalLogContainer) { + if (resolvedOptions.getShouldWriteOperationalLogs && !resolvedOptions.getShouldWriteOperationalLogs()) { + throw new Error('Operational logs are disabled by configuration but an operational log container was provided.'); + } + + try { + await operationalLogContainer.createIfNotExists(); + } catch (error: unknown) { + if (!(error instanceof Error)) { + throw error; + } + + throw new Error('Failed to create or access the Azure Storage Blob container (operational).', { + 'cause': error + }); + } + } + + if (auditLogContainer) { + if (resolvedOptions.getShouldWriteAuditLogs && !resolvedOptions.getShouldWriteAuditLogs()) { + throw new Error('Audit logs are disabled by configuration but an audit log container was provided.'); + } + + try { + await auditLogContainer.createIfNotExists(); + } catch (error: unknown) { + if (!(error instanceof Error)) { + throw error; + } + + throw new Error('Failed to create or access the Azure Storage Blob container (audit).', { + 'cause': error + }); + } + } + + return new AzureStorageDestination(operationalLogContainer, auditLogContainer, resolvedOptions); + } + + public override async log(log: OperationalLog): Promise { + // #region Input validation + /* v8 ignore next */ + assertGuardEquals(log); + // #endregion Input validation + + if (this.#isDisposed || ( + this.#appliedOptions.getShouldWriteOperationalLogs && + !this.#appliedOptions.getShouldWriteOperationalLogs() + ) + ) { + return; + } + + try { + await this.#append( + `${ new SerializableOperationalLog(log).serialize('json') }\n`, + 'operational' + ); + } catch (error: unknown) { + AzureStorageDestination.writeConfiguredDebugInfo( + this.#appliedOptions, + error, + 'Failed to append operational log to Azure Blob Storage.' + ); + } + + return Promise.resolve(); + } + + public override async auditLog(log: AuditLog): Promise { + // #region Input validation + /* v8 ignore next */ + assertGuardEquals(log); + // #endregion Input validation + + if (this.#isDisposed || ( + this.#appliedOptions.getShouldWriteAuditLogs && + !this.#appliedOptions.getShouldWriteAuditLogs() + ) + ) { + return; + } + + try { + await this.#append( + `${ new SerializableAuditLog(log).serialize('json') }\n`, + 'audit' + ); + } catch (error: unknown) { + AzureStorageDestination.writeConfiguredDebugInfo( + this.#appliedOptions, + error, + 'Failed to append audit log to Azure Blob Storage.' + ); + } + } + + public override dispose(): void { + this.#isDisposed = true; + + void Promise.all([ + this.#operationalCollection?.appendQueue, + this.#auditCollection?.appendQueue + ]).finally(() => { + if (this.#operationalCollection) { + this.#operationalCollection.activeBlob = void 0; + + this.#operationalCollection.activeBlobDate = void 0; + } + + if (this.#auditCollection) { + this.#auditCollection.activeBlob = void 0; + + this.#auditCollection.activeBlobDate = void 0; + } + }); + } + + #append(content: string, type: 'audit' | 'operational'): Promise { + if (this.#isDisposed) { + return Promise.reject(new Error('Azure Storage destination has been disposed.')); + } + + const collection = type === 'audit' ? this.#auditCollection : this.#operationalCollection; + + if (!collection) { + return Promise.reject(new Error(`Failed to access the ${ type } log collection.`)); + } + + const contentSize = Buffer.byteLength(content); + + if (contentSize > this.#appliedOptions.maxAppendBlockBytes) { + throw new RangeError(`Azure Storage append blob record is ${ contentSize } bytes; ` + + `the configured maximum is ${ this.#appliedOptions.maxAppendBlockBytes } bytes.`); + } + + // Queue the append operation to ensure sequential writes. + const appendOperation = collection.appendQueue + .then(async () => { + // Check if a new blob needs to be created based on the current time. + const timeResult = AzureStorageDestination.#shouldCreateNewBlob(collection.activeBlobDate); + + return timeResult.shouldCreate + ? { + 'activeBlob': await this.#createNewBlob(type, timeResult.activeHour), + 'activeBlobDate': timeResult.activeHour + } + : void 0; + }) + .then((newActiveBlob) => { + if (newActiveBlob) { + collection.activeBlob = newActiveBlob.activeBlob; + + collection.activeBlobDate = newActiveBlob.activeBlobDate; + } + + // Ensure the active blob is initialized before appending. + if (!collection.activeBlob) { + throw new Error('Active blob is not initialized.'); + } + + // Append the content to the active blob and discard the result. + return collection.activeBlob + .appendBlock(content, Buffer.byteLength(content)) + .then(() => void 0); + }); + + // Keep the queue usable after an individual append failure. + collection.appendQueue = appendOperation.catch(() => void 0); + + return appendOperation; + } + + async #createNewBlob(type: 'audit' | 'operational', activeHour: number): Promise { + const activeHourDate = new Date(activeHour); + + // Format the blob name based on the current hour + let blobName = activeHourDate.toISOString(); + + // Construct the final blob name in the format YYYYMMDDHH.audit.log or YYYYMMDDHH.operational.log. + blobName = `${ blobName.slice(0, 10) }${ blobName.slice(11, 13) }.${ type }.log`.replaceAll('-', ''); + + const collection = type === 'audit' ? this.#auditCollection : this.#operationalCollection; + + if (!collection?.blobContainer) { + // Todo - log internal don't throw + this.#writeDebugLog(`Failed to access the ${ type } log collection.`); + + return void 0; + } + + const blobClient = collection.blobContainer.getAppendBlobClient(blobName); + + const result = await blobClient.createIfNotExists(); + + if (result.errorCode) { + this.#writeDebugLog('Failed to create new blob:', result.errorCode); + } + + return blobClient; + } + + static #isContainerClient(value: unknown): value is ContainerClient { + return typeof value === 'object' && + value !== null && + typeof (value as { 'createIfNotExists'?: unknown; }).createIfNotExists === 'function' && + typeof (value as { 'getAppendBlobClient'?: unknown; }).getAppendBlobClient === 'function'; + } + + static #resolveConfigurationOptions(configuration: AzureStorageDestinationOptions): ResolvedAzureStorageDestinationOptions { + const resolvedOptions = AzureStorageDestination.resolveConfigurationOptions( + configuration, + DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS + ); + + // #region Input validation + /* v8 ignore next */ + assertGuardEquals(resolvedOptions.validationInput as Omit); + // #endregion Input validation + + return { + ...DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS, + ...configuration, + 'getShouldWriteAuditLogs': resolvedOptions.getShouldWriteAuditLogs, + 'getShouldWriteDebugInfo': resolvedOptions.getShouldWriteDebugInfo, + 'getShouldWriteOperationalLogs': resolvedOptions.getShouldWriteOperationalLogs + }; + } + + static #shouldCreateNewBlob(currentActiveDate?: number): { + 'shouldCreate': boolean, + 'activeHour': number; + } { + const activeHour = new Date().setMinutes(0, 0, 0); + + return { + activeHour, + 'shouldCreate': currentActiveDate === void 0 || activeHour > currentActiveDate + }; + } + + #writeDebugLog(message: string, ...optionalParams: unknown[]): void { + if ( + this.#appliedOptions.getShouldWriteDebugInfo && + this.#appliedOptions.getShouldWriteDebugInfo() + ) { + // eslint-disable-next-line no-console + console.debug(message, ...optionalParams); + } + } +} diff --git a/tests/plugins/AzureStorageDestination.test.ts b/tests/plugins/AzureStorageDestination.test.ts new file mode 100644 index 0000000..54b8d5a --- /dev/null +++ b/tests/plugins/AzureStorageDestination.test.ts @@ -0,0 +1,238 @@ +import type { AppendBlobClient, ContainerClient } from '@azure/storage-blob'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { AzureStorageDestination } from '#/plugins/AzureStorageDestination.js'; +import { LogLevel, type AuditLog, type OperationalLog } from '#/interfaces/LogEngine.js'; + +const uuid = '00000000-0000-0000-0000-000000000001'; + +const operational: OperationalLog = { + 'additionalContext': void 0, + 'correlationId': uuid, + 'level': LogLevel.Information, + 'message': 'operational entry', + 'requestId': void 0, + 'stack': void 0, + 'tenantId': void 0, + 'timeGenerated': new Date('2025-01-02T03:04:05.678Z'), + 'userId': 'user' +}; + +const audit: AuditLog = { + 'after': 'new', + 'before': 'old', + 'category': 'Update', + 'correlationId': uuid, + 'message': 'audit entry', + 'requestId': void 0, + 'tenantId': void 0, + 'timeGenerated': new Date('2025-01-02T03:04:05.678Z'), + 'userId': 'user' +}; + +interface BlobContainerTestDouble { + 'appendBlock': ReturnType; + 'blobNames': string[]; + 'createIfNotExists': ReturnType; + 'container': ContainerClient; +} + +function createContainer(): BlobContainerTestDouble { + const appendBlock = vi.fn(() => Promise.resolve({})); + + const blobNames: string[] = []; + + const appendBlob = { + 'appendBlock': appendBlock, + 'createIfNotExists': vi.fn(() => Promise.resolve({ 'succeeded': true })) + } as unknown as AppendBlobClient; + + const createIfNotExists = vi.fn(() => Promise.resolve({ 'succeeded': true })); + + const container = { + createIfNotExists, + 'getAppendBlobClient': vi.fn((name: string) => { + blobNames.push(name); + + return appendBlob; + }) + } as unknown as ContainerClient; + + return { + appendBlock, + blobNames, + createIfNotExists, + container + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe('AzureStorageDestination', () => { + it('should initialize supplied containers and append JSON Lines to separate stream blobs', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-02T03:04:05.678Z')); + + const operationalContainer = createContainer(); + const auditContainer = createContainer(); + + const destination = await AzureStorageDestination.create(operationalContainer.container, auditContainer.container); + + await destination.log(operational); + await destination.auditLog(audit); + + expect(operationalContainer.createIfNotExists).toHaveBeenCalledOnce(); + expect(auditContainer.createIfNotExists).toHaveBeenCalledOnce(); + expect(operationalContainer.blobNames).toEqual(['2025010203.operational.log']); + expect(auditContainer.blobNames).toEqual(['2025010203.audit.log']); + + const [[operationalContent, operationalContentLength]] = operationalContainer.appendBlock.mock.calls as [[string, number]]; + const [[auditContent, auditContentLength]] = auditContainer.appendBlock.mock.calls as [[string, number]]; + + expect(operationalContent).toMatch(/\n$/u); + expect(JSON.parse(operationalContent)).toMatchObject({ 'message': 'operational entry' }); + expect(operationalContentLength).toBe(Buffer.byteLength(operationalContent)); + expect(auditContent).toMatch(/\n$/u); + expect(JSON.parse(auditContent)).toMatchObject({ 'message': 'audit entry' }); + expect(auditContentLength).toBe(Buffer.byteLength(auditContent)); + }); + + it('should reuse a stream blob within the current hour and rotate it in the next hour', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-02T03:04:05.678Z')); + + const operationalContainer = createContainer(); + const destination = await AzureStorageDestination.create(operationalContainer.container); + + await destination.log(operational); + await destination.log(operational); + + vi.setSystemTime(new Date('2025-01-02T04:00:00.000Z')); + + await destination.log(operational); + + expect(operationalContainer.blobNames).toEqual([ + '2025010203.operational.log', + '2025010204.operational.log' + ]); + expect(operationalContainer.appendBlock).toHaveBeenCalledTimes(3); + }); + + it('should skip disabled streams without appending records', async () => { + const operationalContainer = createContainer(); + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'getShouldWriteAuditLogs': () => false + }); + + await destination.auditLog(audit); + + expect(operationalContainer.appendBlock).not.toHaveBeenCalled(); + }); + + it('should reject invalid containers and configurations during creation', async () => { + const operationalContainer = createContainer(); + + await expect(AzureStorageDestination.create()).rejects.toThrow('At least one'); + await expect(AzureStorageDestination.create({} as ContainerClient)).rejects.toThrow('operationalLogContainer'); + await expect(AzureStorageDestination.create(void 0, {} as ContainerClient)).rejects.toThrow('auditLogContainer'); + await expect(AzureStorageDestination.create(operationalContainer.container, void 0, { + 'getShouldWriteOperationalLogs': () => false + })).rejects.toThrow('Operational logs are disabled'); + await expect(AzureStorageDestination.create(void 0, operationalContainer.container, { + 'getShouldWriteAuditLogs': () => false + })).rejects.toThrow('Audit logs are disabled'); + await expect(AzureStorageDestination.create(operationalContainer.container, void 0, { + 'maxAppendBlockBytes': 0 as never + })).rejects.toThrow(); + }); + + it('should reject unsuccessful operational and audit container initialization', async () => { + const operationalContainer = createContainer(); + const auditContainer = createContainer(); + + operationalContainer.createIfNotExists.mockRejectedValueOnce(new Error('operational init failed')); + auditContainer.createIfNotExists.mockRejectedValueOnce(new Error('audit init failed')); + + await expect(AzureStorageDestination.create(operationalContainer.container)) + .rejects.toThrow('Failed to create or access the Azure Storage Blob container (operational).'); + await expect(AzureStorageDestination.create(void 0, auditContainer.container)) + .rejects.toThrow('Failed to create or access the Azure Storage Blob container (audit).'); + }); + + it('should report a missing stream container without throwing to the caller', async () => { + const operationalContainer = createContainer(); + const diagnostic = vi.spyOn(console, 'log').mockImplementation(() => void 0); + const debug = vi.spyOn(console, 'debug').mockImplementation(() => void 0); + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'getShouldWriteDebugInfo': () => true + }); + + await expect(destination.auditLog(audit)).resolves.toBeUndefined(); + + expect(debug).toHaveBeenCalledWith('Failed to access the audit log collection.'); + expect(diagnostic).toHaveBeenCalledWith(expect.stringContaining('Active blob is not initialized.')); + }); + + it('should report append failures and continue processing later records', async () => { + const error = new Error('append failed'); + const operationalContainer = createContainer(); + + operationalContainer.appendBlock + .mockRejectedValueOnce(error) + .mockResolvedValueOnce({}); + + const diagnostic = vi.spyOn(console, 'log').mockImplementation(() => void 0); + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'getShouldWriteDebugInfo': () => true + }); + + await destination.log(operational); + await destination.log(operational); + + expect(operationalContainer.appendBlock).toHaveBeenCalledTimes(2); + expect(diagnostic).toHaveBeenCalledWith(expect.stringContaining('Failed to append operational log')); + expect(diagnostic).toHaveBeenCalledWith(expect.stringContaining('append failed')); + }); + + it('should report oversized records without creating an append blob', async () => { + const operationalContainer = createContainer(); + const diagnostic = vi.spyOn(console, 'log').mockImplementation(() => void 0); + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'getShouldWriteDebugInfo': () => true, + 'maxAppendBlockBytes': 1 + }); + + await destination.log(operational); + + expect(operationalContainer.blobNames).toEqual([]); + expect(diagnostic).toHaveBeenCalledWith(expect.stringContaining('configured maximum is 1 bytes')); + }); + + it('should let queued appends finish and ignore records submitted after disposal', async () => { + let finishAppend: (() => void) | undefined; + + const operationalContainer = createContainer(); + + operationalContainer.appendBlock.mockImplementationOnce(() => new Promise((resolve) => { + finishAppend = resolve; + })); + + const destination = await AzureStorageDestination.create(operationalContainer.container); + + const firstLog = destination.log(operational); + + await vi.waitFor(() => expect(operationalContainer.appendBlock).toHaveBeenCalledOnce()); + + destination.dispose(); + + const laterLog = destination.log(operational); + + finishAppend?.(); + + await Promise.all([firstLog, laterLog]); + + expect(operationalContainer.appendBlock).toHaveBeenCalledOnce(); + }); +}); From eadb01cb7784741f41bf8e1aa83d78a8126dd8a4 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 17:56:24 +0100 Subject: [PATCH 06/16] Update README.md --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/README.md b/README.md index 518e1d8..fbf4656 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,46 @@ await logEngine.addPlugin(FileDestination, { The default directory is `./logs/dev`, the default rotation interval is 60 minutes, and the default retention period is one day. Host applications can conditionally add this destination, using the runtime write callbacks to control it. +### AzureStorageDestination + +`AzureStorageDestination` appends newline-delimited JSON records to Azure Append Blobs. The Azure Blob SDK is an optional peer dependency: applications that use this destination must install it, while applications using other destinations do not need it. + +```bash +npm install @azure/storage-blob @azure/identity +``` + +Create authenticated `ContainerClient` instances in the host application and pass them to the destination factory. The first container receives operational records and the second receives audit records; either container may be omitted, but at least one must be supplied. + +```typescript +import { DefaultAzureCredential } from '@azure/identity'; +import { BlobServiceClient } from '@azure/storage-blob'; +import { + AzureStorageDestination, + LogEngine +} from '@software-hardware-integration-lab/log-engine'; + +const credential = new DefaultAzureCredential(); +const blobService = new BlobServiceClient('https://example.blob.core.windows.net', credential); +const operationalContainer = blobService.getContainerClient('operational-logs'); +const auditContainer = blobService.getContainerClient('audit-logs'); + +const azureStorageDestination = await AzureStorageDestination.create( + operationalContainer, + auditContainer, + { + 'maxAppendBlockBytes': 4 * 1024 * 1024 + } +); + +await logEngine.addPlugin({ + 'create': async () => azureStorageDestination +}); +``` + +The destination creates containers when necessary and uses separate hourly blobs for operational and audit streams. Append Blob records are capped at 4 MiB by default, the stable limit across Azure Storage service versions; set `maxAppendBlockBytes` only after confirming the service version and account capabilities used by the host application. + +Configure blob retention with an Azure Storage lifecycle-management policy scoped to the destination container or prefix. The plugin deliberately has only data-plane responsibilities and does not create, replace, or delete storage-account lifecycle rules. + ### LogAnalyticsDestination `LogAnalyticsDestination` sends operational and audit logs through host-provided structural uploaders. The package intentionally does not depend on an Azure SDK: the host owns credentials and SDK clients, while the destination only needs an object with `upload(ruleId, streamName, logs)`. From 570622640818c3ceeee8096dfc8727b3d517b2ce Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 18:02:00 +0100 Subject: [PATCH 07/16] Bump Azure blob and Node types Update the lockfile to allow the Azure Blob Storage dependency to resolve with the caret range and refresh @types/node to 26.5.1. This keeps dependency resolution aligned with the current package constraints and avoids stale lockfile metadata. --- package-lock.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8d5735e..9335ef2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "typia": "~12.1.1" }, "devDependencies": { - "@azure/storage-blob": "~12.33.0", + "@azure/storage-blob": "^12.33.0", "@software-hardware-integration-lab/development-utilities": "~3.0.0", "@types/node": "~26.5.0", "@vitest/coverage-v8": "~5.0.1", @@ -21,7 +21,7 @@ "vitest": "~5.0.1" }, "peerDependencies": { - "@azure/storage-blob": "~12.33.0" + "@azure/storage-blob": "^12.33.0" }, "peerDependenciesMeta": { "@azure/storage-blob": { @@ -1231,9 +1231,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.0.tgz", - "integrity": "sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A==", + "version": "26.5.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.1.tgz", + "integrity": "sha512-CzNm2FezW4VR/LjG6yUdiEgLE/rAQ9Slj5gCu/C2VrdcW7I0ahNZ8DRbHT7zOZ6r3ONgd/bsQIeSaoDGrd1C6g==", "devOptional": true, "license": "MIT", "dependencies": { From 3f0ae60da81abe54604fd0ff25d9c35305872fd6 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 18:04:10 +0100 Subject: [PATCH 08/16] Change byte length function to satisfy tsc warning --- tests/plugins/AzureStorageDestination.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/plugins/AzureStorageDestination.test.ts b/tests/plugins/AzureStorageDestination.test.ts index 54b8d5a..2ccb89f 100644 --- a/tests/plugins/AzureStorageDestination.test.ts +++ b/tests/plugins/AzureStorageDestination.test.ts @@ -93,10 +93,10 @@ describe('AzureStorageDestination', () => { expect(operationalContent).toMatch(/\n$/u); expect(JSON.parse(operationalContent)).toMatchObject({ 'message': 'operational entry' }); - expect(operationalContentLength).toBe(Buffer.byteLength(operationalContent)); + expect(operationalContentLength).toBe(new TextEncoder().encode(operationalContent).byteLength); expect(auditContent).toMatch(/\n$/u); expect(JSON.parse(auditContent)).toMatchObject({ 'message': 'audit entry' }); - expect(auditContentLength).toBe(Buffer.byteLength(auditContent)); + expect(auditContentLength).toBe(new TextEncoder().encode(auditContent).byteLength); }); it('should reuse a stream blob within the current hour and rotate it in the next hour', async () => { From 6aef1e5a496a2587bbf4ae28e7ae4f3973624be2 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 19:35:15 +0100 Subject: [PATCH 09/16] Decouple Azure storage client types Replace direct `@azure/storage-blob` type imports with local structural interfaces for append blob and container clients. This keeps the Azure storage destination compatible with the SDK at runtime while avoiding a hard dependency on its public TypeScript declarations. --- .../plugins/AzureStorageDestination.ts | 24 +++++++++++++++++++ src/plugins/AzureStorageDestination.ts | 19 +++++++-------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/interfaces/plugins/AzureStorageDestination.ts b/src/interfaces/plugins/AzureStorageDestination.ts index 2de8d59..2a1102f 100644 --- a/src/interfaces/plugins/AzureStorageDestination.ts +++ b/src/interfaces/plugins/AzureStorageDestination.ts @@ -1,6 +1,30 @@ import { DEFAULT_LOGGING_PLUGIN_CONFIGURATION_OPTIONS, type LoggingPluginConfigurationOptions } from './LoggingPlugin.js'; import type { tags } from 'typia'; +/** + * Minimal contract for an Azure Blob append-blob client. Structurally compatible with + * `@azure/storage-blob`'s `AppendBlobClient`, so real SDK instances satisfy this type without + * requiring `@azure/storage-blob` to resolve the public declarations of this package. + */ +export interface AzureAppendBlobClientLike { + /** Appends a block of content to the append blob. */ + 'appendBlock': (content: string, contentLength: number) => Promise; + /** Creates the append blob if it does not already exist. */ + 'createIfNotExists': () => Promise<{ 'errorCode'?: string; }>; +} + +/** + * Minimal contract for an Azure Blob container client. Structurally compatible with + * `@azure/storage-blob`'s `ContainerClient`, so real SDK instances satisfy this type without + * requiring `@azure/storage-blob` to resolve the public declarations of this package. + */ +export interface AzureBlobContainerLike { + /** Creates the container if it does not already exist. */ + 'createIfNotExists': () => Promise; + /** Gets a client for the named append blob within this container. */ + 'getAppendBlobClient': (blobName: string) => AzureAppendBlobClientLike; +} + /** Configuration options for Azure Storage append-blob logging behavior. */ export interface AzureStorageDestinationOptions extends LoggingPluginConfigurationOptions { /** Maximum number of UTF-8 bytes permitted in one append-blob write. */ diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts index 3f078c0..6ad9bc2 100644 --- a/src/plugins/AzureStorageDestination.ts +++ b/src/plugins/AzureStorageDestination.ts @@ -1,16 +1,15 @@ import type { OperationalLog, AuditLog } from '#/interfaces/LogEngine.js'; -import type { AppendBlobClient, ContainerClient } from '@azure/storage-blob'; import { LoggingPlugin } from './base/LoggingPlugin.js'; -import { DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS, type AzureStorageDestinationOptions, type ResolvedAzureStorageDestinationOptions } from '#/interfaces/plugins/AzureStorageDestination.js'; +import { DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS, type AzureAppendBlobClientLike, type AzureBlobContainerLike, type AzureStorageDestinationOptions, type ResolvedAzureStorageDestinationOptions } from '#/interfaces/plugins/AzureStorageDestination.js'; import { SerializableAuditLog } from '#/classes/SerializableAuditLog.js'; import { SerializableOperationalLog } from '#/classes/SerializableOperationalLog.js'; import { assertGuardEquals } from 'typia'; interface LogTypeCollection { - 'activeBlob': AppendBlobClient | undefined; + 'activeBlob': AzureAppendBlobClientLike | undefined; 'appendQueue': Promise; 'activeBlobDate': number | undefined; - 'blobContainer': ContainerClient | undefined; + 'blobContainer': AzureBlobContainerLike | undefined; } /** @@ -24,8 +23,8 @@ export class AzureStorageDestination extends LoggingPlugin { #isDisposed = false; private constructor( - operationalLogContainer?: ContainerClient, - auditLogContainer?: ContainerClient, + operationalLogContainer?: AzureBlobContainerLike, + auditLogContainer?: AzureBlobContainerLike, configuration: AzureStorageDestinationOptions = DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS ) { super(configuration.id ?? 'AzureStorageDestination'); @@ -55,8 +54,8 @@ export class AzureStorageDestination extends LoggingPlugin { * @returns Initialized Azure Storage destination. */ public static async create( - operationalLogContainer?: ContainerClient, - auditLogContainer?: ContainerClient, + operationalLogContainer?: AzureBlobContainerLike, + auditLogContainer?: AzureBlobContainerLike, configuration: AzureStorageDestinationOptions = DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS ): Promise { if (!operationalLogContainer && !auditLogContainer) { @@ -246,7 +245,7 @@ export class AzureStorageDestination extends LoggingPlugin { return appendOperation; } - async #createNewBlob(type: 'audit' | 'operational', activeHour: number): Promise { + async #createNewBlob(type: 'audit' | 'operational', activeHour: number): Promise { const activeHourDate = new Date(activeHour); // Format the blob name based on the current hour @@ -275,7 +274,7 @@ export class AzureStorageDestination extends LoggingPlugin { return blobClient; } - static #isContainerClient(value: unknown): value is ContainerClient { + static #isContainerClient(value: unknown): value is AzureBlobContainerLike { return typeof value === 'object' && value !== null && typeof (value as { 'createIfNotExists'?: unknown; }).createIfNotExists === 'function' && From 7d5e6b3c20ab6c7b644c3214bb6210f3e786d00f Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 19:38:14 +0100 Subject: [PATCH 10/16] Add maxAppendBlockBytes to AzureStorageDestination options Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: pr0uxx --- src/plugins/AzureStorageDestination.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts index 6ad9bc2..f19626f 100644 --- a/src/plugins/AzureStorageDestination.ts +++ b/src/plugins/AzureStorageDestination.ts @@ -295,6 +295,7 @@ export class AzureStorageDestination extends LoggingPlugin { return { ...DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS, ...configuration, + 'maxAppendBlockBytes': configuration.maxAppendBlockBytes ?? DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS.maxAppendBlockBytes, 'getShouldWriteAuditLogs': resolvedOptions.getShouldWriteAuditLogs, 'getShouldWriteDebugInfo': resolvedOptions.getShouldWriteDebugInfo, 'getShouldWriteOperationalLogs': resolvedOptions.getShouldWriteOperationalLogs From da7dad1f35736eab894475958076d0f4917b6e2f Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 19:47:53 +0100 Subject: [PATCH 11/16] Remove disabled logs validation checks Remove validation that threw errors when operational or audit log containers were provided but logging was disabled by configuration. This allows containers to be provided without enforcing that logging must be enabled. --- .github/workflows/Security-Reachability.yml | 2 +- src/plugins/AzureStorageDestination.ts | 8 -------- tests/plugins/AzureStorageDestination.test.ts | 6 ------ 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/Security-Reachability.yml b/.github/workflows/Security-Reachability.yml index 5ca7f4b..33e187b 100644 --- a/.github/workflows/Security-Reachability.yml +++ b/.github/workflows/Security-Reachability.yml @@ -80,7 +80,7 @@ jobs: # validate:package:skip-reachability command so workflow validation does not modify its checkout. - name: Install Socket CLI background: true - run: sfw pip install socketsecurity==2.8.2 uv --upgrade + run: sfw pip install socketsecurity==2.9.0 uv --upgrade # Bring job back to sync execution by awaiting for all async jobs to finish before continuing - name: Steps - Convert Back To Synchronous Execution - Packages Updates/Setup diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts index f19626f..b36ce5e 100644 --- a/src/plugins/AzureStorageDestination.ts +++ b/src/plugins/AzureStorageDestination.ts @@ -73,10 +73,6 @@ export class AzureStorageDestination extends LoggingPlugin { const resolvedOptions = AzureStorageDestination.#resolveConfigurationOptions(configuration); if (operationalLogContainer) { - if (resolvedOptions.getShouldWriteOperationalLogs && !resolvedOptions.getShouldWriteOperationalLogs()) { - throw new Error('Operational logs are disabled by configuration but an operational log container was provided.'); - } - try { await operationalLogContainer.createIfNotExists(); } catch (error: unknown) { @@ -91,10 +87,6 @@ export class AzureStorageDestination extends LoggingPlugin { } if (auditLogContainer) { - if (resolvedOptions.getShouldWriteAuditLogs && !resolvedOptions.getShouldWriteAuditLogs()) { - throw new Error('Audit logs are disabled by configuration but an audit log container was provided.'); - } - try { await auditLogContainer.createIfNotExists(); } catch (error: unknown) { diff --git a/tests/plugins/AzureStorageDestination.test.ts b/tests/plugins/AzureStorageDestination.test.ts index 2ccb89f..bb2f6d7 100644 --- a/tests/plugins/AzureStorageDestination.test.ts +++ b/tests/plugins/AzureStorageDestination.test.ts @@ -137,12 +137,6 @@ describe('AzureStorageDestination', () => { await expect(AzureStorageDestination.create()).rejects.toThrow('At least one'); await expect(AzureStorageDestination.create({} as ContainerClient)).rejects.toThrow('operationalLogContainer'); await expect(AzureStorageDestination.create(void 0, {} as ContainerClient)).rejects.toThrow('auditLogContainer'); - await expect(AzureStorageDestination.create(operationalContainer.container, void 0, { - 'getShouldWriteOperationalLogs': () => false - })).rejects.toThrow('Operational logs are disabled'); - await expect(AzureStorageDestination.create(void 0, operationalContainer.container, { - 'getShouldWriteAuditLogs': () => false - })).rejects.toThrow('Audit logs are disabled'); await expect(AzureStorageDestination.create(operationalContainer.container, void 0, { 'maxAppendBlockBytes': 0 as never })).rejects.toThrow(); From d1d3e15e5ad859a038f62852e5b1af2da1fd4ae9 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 19:49:53 +0100 Subject: [PATCH 12/16] Change activeHour to use UTC minutes Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: pr0uxx --- src/plugins/AzureStorageDestination.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts index b36ce5e..19e024e 100644 --- a/src/plugins/AzureStorageDestination.ts +++ b/src/plugins/AzureStorageDestination.ts @@ -298,7 +298,7 @@ export class AzureStorageDestination extends LoggingPlugin { 'shouldCreate': boolean, 'activeHour': number; } { - const activeHour = new Date().setMinutes(0, 0, 0); + const activeHour = new Date().setUTCMinutes(0, 0, 0); return { activeHour, From 634a4f2aec9538db4bb5517c77723cba6ac509af Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 19:50:56 +0100 Subject: [PATCH 13/16] Add logEngine instance and plugin to README Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: pr0uxx --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fbf4656..22c0b1d 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,8 @@ const azureStorageDestination = await AzureStorageDestination.create( } ); +const logEngine = LogEngine.getInstance(); + await logEngine.addPlugin({ 'create': async () => azureStorageDestination }); From 096c7a754bc86962e5d3f4d5bbca6669b0a803e7 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 20:11:29 +0100 Subject: [PATCH 14/16] Add block-count limit and batch flushing to Azure Storage Introduces maxBlocksPerBlob configuration option to rotate append blobs when reaching Azure's 50,000 block limit. Refactors append logic from sequential queueing to efficient batch flushing, grouping pending records within byte limits and supporting blob rotation via numeric suffixes within the same hour. Improves throughput by batching records queued during in-flight flushes into single append operations. --- README.md | 2 +- .../plugins/AzureStorageDestination.ts | 8 +- src/plugins/AzureStorageDestination.ts | 234 ++++++++++++++---- tests/plugins/AzureStorageDestination.test.ts | 50 ++++ 4 files changed, 239 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index fbf4656..f6c75e9 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ await logEngine.addPlugin({ }); ``` -The destination creates containers when necessary and uses separate hourly blobs for operational and audit streams. Append Blob records are capped at 4 MiB by default, the stable limit across Azure Storage service versions; set `maxAppendBlockBytes` only after confirming the service version and account capabilities used by the host application. +The destination creates containers when necessary and uses separate hourly blobs for operational and audit streams. Records queued while a batch is in flight are combined into a single append-blob write, and each write is capped at 4 MiB by default, the stable limit across Azure Storage service versions; set `maxAppendBlockBytes` only after confirming the service version and account capabilities used by the host application. Because Azure limits an append blob to 50,000 blocks, the destination rotates to a suffixed blob (e.g. `2025010203.operational.2.log`) within the same hour once `maxBlocksPerBlob` (default `50000`) is reached, so high-volume streams never exhaust a blob before the hourly rotation. Configure blob retention with an Azure Storage lifecycle-management policy scoped to the destination container or prefix. The plugin deliberately has only data-plane responsibilities and does not create, replace, or delete storage-account lifecycle rules. diff --git a/src/interfaces/plugins/AzureStorageDestination.ts b/src/interfaces/plugins/AzureStorageDestination.ts index 2a1102f..886311a 100644 --- a/src/interfaces/plugins/AzureStorageDestination.ts +++ b/src/interfaces/plugins/AzureStorageDestination.ts @@ -29,17 +29,23 @@ export interface AzureBlobContainerLike { export interface AzureStorageDestinationOptions extends LoggingPluginConfigurationOptions { /** Maximum number of UTF-8 bytes permitted in one append-blob write. */ 'maxAppendBlockBytes'?: number & tags.Minimum<1>; + /** Maximum number of blocks written to a single append blob before rotating to a suffixed blob. */ + 'maxBlocksPerBlob'?: number & tags.Minimum<1> & tags.Maximum<50_000>; } /** Fully resolved options used internally by the Azure Storage destination. */ export interface ResolvedAzureStorageDestinationOptions extends LoggingPluginConfigurationOptions { /** Maximum number of UTF-8 bytes permitted in one append-blob write. */ 'maxAppendBlockBytes': number & tags.Minimum<1>; + /** Maximum number of blocks written to a single append blob before rotating to a suffixed blob. */ + 'maxBlocksPerBlob': number & tags.Minimum<1> & tags.Maximum<50_000>; } /** Static default options for Azure Storage append-blob logging. */ export const DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS: ResolvedAzureStorageDestinationOptions = { ...DEFAULT_LOGGING_PLUGIN_CONFIGURATION_OPTIONS, // The stable limit supported by all Append Blob service API versions. - 'maxAppendBlockBytes': 4 * 1024 * 1024 + 'maxAppendBlockBytes': 4 * 1024 * 1024, + // Azure's hard append-blob limit; rotating at this point uses the full available capacity. + 'maxBlocksPerBlob': 50_000 }; diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts index b36ce5e..b2b5744 100644 --- a/src/plugins/AzureStorageDestination.ts +++ b/src/plugins/AzureStorageDestination.ts @@ -5,11 +5,24 @@ import { SerializableAuditLog } from '#/classes/SerializableAuditLog.js'; import { SerializableOperationalLog } from '#/classes/SerializableOperationalLog.js'; import { assertGuardEquals } from 'typia'; +/** A queued log record awaiting a batched append, along with its caller-facing settlement. */ +interface PendingLogRecord { + 'content': string; + 'reject': (error: unknown) => void; + 'resolve': () => void; +} + interface LogTypeCollection { 'activeBlob': AzureAppendBlobClientLike | undefined; - 'appendQueue': Promise; 'activeBlobDate': number | undefined; + /** Suffix applied to the blob name when rotating within the same hour due to the block-count limit. */ + 'blobSuffix': number; 'blobContainer': AzureBlobContainerLike | undefined; + /** Number of append blocks already written to the active blob. */ + 'blockCount': number; + /** In-flight batch flush; new records arriving while set are picked up by the next flush. */ + 'flushPromise': Promise | undefined; + 'pendingRecords': PendingLogRecord[]; } /** @@ -33,16 +46,22 @@ export class AzureStorageDestination extends LoggingPlugin { this.#operationalCollection = { 'activeBlob': void 0, - 'appendQueue': Promise.resolve(), 'activeBlobDate': void 0, - 'blobContainer': operationalLogContainer + 'blobContainer': operationalLogContainer, + 'blobSuffix': 0, + 'blockCount': 0, + 'flushPromise': void 0, + 'pendingRecords': [] }; this.#auditCollection = { 'activeBlob': void 0, - 'appendQueue': Promise.resolve(), 'activeBlobDate': void 0, - 'blobContainer': auditLogContainer + 'blobContainer': auditLogContainer, + 'blobSuffix': 0, + 'blockCount': 0, + 'flushPromise': void 0, + 'pendingRecords': [] }; } @@ -165,8 +184,8 @@ export class AzureStorageDestination extends LoggingPlugin { this.#isDisposed = true; void Promise.all([ - this.#operationalCollection?.appendQueue, - this.#auditCollection?.appendQueue + this.#operationalCollection?.flushPromise, + this.#auditCollection?.flushPromise ]).finally(() => { if (this.#operationalCollection) { this.#operationalCollection.activeBlob = void 0; @@ -200,51 +219,171 @@ export class AzureStorageDestination extends LoggingPlugin { `the configured maximum is ${ this.#appliedOptions.maxAppendBlockBytes } bytes.`); } - // Queue the append operation to ensure sequential writes. - const appendOperation = collection.appendQueue - .then(async () => { - // Check if a new blob needs to be created based on the current time. - const timeResult = AzureStorageDestination.#shouldCreateNewBlob(collection.activeBlobDate); - - return timeResult.shouldCreate - ? { - 'activeBlob': await this.#createNewBlob(type, timeResult.activeHour), - 'activeBlobDate': timeResult.activeHour - } - : void 0; - }) - .then((newActiveBlob) => { - if (newActiveBlob) { - collection.activeBlob = newActiveBlob.activeBlob; - - collection.activeBlobDate = newActiveBlob.activeBlobDate; - } + // Queue the record; it is combined with any records batched into the same flush. + return new Promise((resolve, reject) => { + collection.pendingRecords.push({ + content, + reject, + resolve + }); + + this.#scheduleFlush(collection, type); + }); + } + + /** + * Starts a batch flush for a collection when none is already in flight. + * @param collection Operational or audit collection to flush. + * @param type Discriminator identifying which collection is being flushed. + */ + #scheduleFlush(collection: LogTypeCollection, type: 'audit' | 'operational'): void { + if (collection.flushPromise) { + return; + } + + collection.flushPromise = this.#flushPending(collection, type).finally(() => { + collection.flushPromise = void 0; + + // Records may have queued up while this flush was in flight; batch them next. + if (collection.pendingRecords.length > 0) { + this.#scheduleFlush(collection, type); + } + }); + } + + /** + * Drains currently queued records, grouping them into append calls that respect the byte limit. + * @param collection Operational or audit collection to flush. + * @param type Discriminator identifying which collection is being flushed. + */ + async #flushPending(collection: LogTypeCollection, type: 'audit' | 'operational'): Promise { + const batch = collection.pendingRecords.splice(0, collection.pendingRecords.length); - // Ensure the active blob is initialized before appending. - if (!collection.activeBlob) { - throw new Error('Active blob is not initialized.'); + if (batch.length === 0) { + return; + } + + const groups = AzureStorageDestination.#groupByByteLimit(batch, this.#appliedOptions.maxAppendBlockBytes); + + for (const group of groups) { + try { + await this.#writeGroup(collection, type, group); + + for (const record of group) { + record.resolve(); + } + } catch (error: unknown) { + for (const record of group) { + record.reject(error); } + } + } + } - // Append the content to the active blob and discard the result. - return collection.activeBlob - .appendBlock(content, Buffer.byteLength(content)) - .then(() => void 0); - }); + /** + * Writes a single group of records as one append-blob block, rotating the blob first if required. + * @param collection Operational or audit collection to write to. + * @param type Discriminator identifying which collection is being written. + * @param group Records to combine into a single append-blob write. + */ + async #writeGroup(collection: LogTypeCollection, type: 'audit' | 'operational', group: PendingLogRecord[]): Promise { + await this.#ensureActiveBlob(collection, type); + + // Ensure the active blob is initialized before appending. + if (!collection.activeBlob) { + throw new Error('Active blob is not initialized.'); + } + + const content = group.map((record) => record.content).join(''); + + await collection.activeBlob.appendBlock(content, Buffer.byteLength(content)); + + collection.blockCount += 1; + } + + /** + * Rotates the active blob when the hour has changed or the block-count limit has been reached. + * @param collection Operational or audit collection to evaluate. + * @param type Discriminator identifying which collection is being rotated. + */ + async #ensureActiveBlob(collection: LogTypeCollection, type: 'audit' | 'operational'): Promise { + const activeHour = new Date().setMinutes(0, 0, 0); + + const isNewHour = collection.activeBlobDate === void 0 || activeHour > collection.activeBlobDate; + + const isBlockLimitReached = collection.blockCount >= this.#appliedOptions.maxBlocksPerBlob; + + if (!isNewHour && !isBlockLimitReached) { + return; + } - // Keep the queue usable after an individual append failure. - collection.appendQueue = appendOperation.catch(() => void 0); + // Reset the suffix on a new hour; otherwise advance it to rotate within the same hour. + const blobSuffix = isNewHour ? 0 : collection.blobSuffix + 1; - return appendOperation; + const activeBlob = await this.#createNewBlob(type, activeHour, blobSuffix); + + // eslint-disable-next-line require-atomic-updates -- #ensureActiveBlob only runs within a single-flight flush per collection. + collection.activeBlob = activeBlob; + + // eslint-disable-next-line require-atomic-updates -- #ensureActiveBlob only runs within a single-flight flush per collection. + collection.activeBlobDate = activeHour; + + // eslint-disable-next-line require-atomic-updates -- #ensureActiveBlob only runs within a single-flight flush per collection. + collection.blobSuffix = blobSuffix; + + // eslint-disable-next-line require-atomic-updates -- #ensureActiveBlob only runs within a single-flight flush per collection. + collection.blockCount = 0; } - async #createNewBlob(type: 'audit' | 'operational', activeHour: number): Promise { + /** + * Groups queued records into batches that each fit within the configured byte limit. + * @param batch Queued records awaiting a flush. + * @param maxBytes Maximum number of UTF-8 bytes permitted in one append-blob write. + * @returns Ordered groups, each of which can be safely combined into a single append-blob write. + */ + static #groupByByteLimit(batch: PendingLogRecord[], maxBytes: number): PendingLogRecord[][] { + const groups: PendingLogRecord[][] = []; + + let currentGroup: PendingLogRecord[] = []; + + let currentSize = 0; + + for (const record of batch) { + const recordSize = Buffer.byteLength(record.content); + + if (currentGroup.length > 0 && currentSize + recordSize > maxBytes) { + groups.push(currentGroup); + + currentGroup = []; + + currentSize = 0; + } + + currentGroup.push(record); + + currentSize += recordSize; + } + + if (currentGroup.length > 0) { + groups.push(currentGroup); + } + + return groups; + } + + async #createNewBlob(type: 'audit' | 'operational', activeHour: number, suffix: number): Promise { const activeHourDate = new Date(activeHour); - // Format the blob name based on the current hour - let blobName = activeHourDate.toISOString(); + const isoValue = activeHourDate.toISOString(); + + // Format the blob name based on the current hour. + const datePrefix = `${ isoValue.slice(0, 10) }${ isoValue.slice(11, 13) }`.replaceAll('-', ''); + + // Append a numeric suffix (e.g. .2, .3) when rotating within the same hour due to the block-count limit. + const suffixSegment = suffix > 0 ? `.${ suffix + 1 }` : ''; // Construct the final blob name in the format YYYYMMDDHH.audit.log or YYYYMMDDHH.operational.log. - blobName = `${ blobName.slice(0, 10) }${ blobName.slice(11, 13) }.${ type }.log`.replaceAll('-', ''); + const blobName = `${ datePrefix }.${ type }${ suffixSegment }.log`; const collection = type === 'audit' ? this.#auditCollection : this.#operationalCollection; @@ -288,24 +427,13 @@ export class AzureStorageDestination extends LoggingPlugin { ...DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS, ...configuration, 'maxAppendBlockBytes': configuration.maxAppendBlockBytes ?? DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS.maxAppendBlockBytes, + 'maxBlocksPerBlob': configuration.maxBlocksPerBlob ?? DEFAULT_AZURE_STORAGE_DESTINATION_OPTIONS.maxBlocksPerBlob, 'getShouldWriteAuditLogs': resolvedOptions.getShouldWriteAuditLogs, 'getShouldWriteDebugInfo': resolvedOptions.getShouldWriteDebugInfo, 'getShouldWriteOperationalLogs': resolvedOptions.getShouldWriteOperationalLogs }; } - static #shouldCreateNewBlob(currentActiveDate?: number): { - 'shouldCreate': boolean, - 'activeHour': number; - } { - const activeHour = new Date().setMinutes(0, 0, 0); - - return { - activeHour, - 'shouldCreate': currentActiveDate === void 0 || activeHour > currentActiveDate - }; - } - #writeDebugLog(message: string, ...optionalParams: unknown[]): void { if ( this.#appliedOptions.getShouldWriteDebugInfo && diff --git a/tests/plugins/AzureStorageDestination.test.ts b/tests/plugins/AzureStorageDestination.test.ts index bb2f6d7..2582f43 100644 --- a/tests/plugins/AzureStorageDestination.test.ts +++ b/tests/plugins/AzureStorageDestination.test.ts @@ -120,6 +120,56 @@ describe('AzureStorageDestination', () => { expect(operationalContainer.appendBlock).toHaveBeenCalledTimes(3); }); + it('should rotate to a suffixed blob within the same hour once the block-count limit is reached', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-02T03:04:05.678Z')); + + const operationalContainer = createContainer(); + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'maxBlocksPerBlob': 1 + }); + + await destination.log(operational); + await destination.log(operational); + await destination.log(operational); + + expect(operationalContainer.blobNames).toEqual([ + '2025010203.operational.log', + '2025010203.operational.2.log', + '2025010203.operational.3.log' + ]); + expect(operationalContainer.appendBlock).toHaveBeenCalledTimes(3); + }); + + it('should batch records queued while a flush is in flight into a single append-blob block', async () => { + let finishAppend: (() => void) | undefined; + + const operationalContainer = createContainer(); + + operationalContainer.appendBlock.mockImplementationOnce(() => new Promise((resolve) => { + finishAppend = resolve; + })); + + const destination = await AzureStorageDestination.create(operationalContainer.container); + + const firstLog = destination.log(operational); + + await vi.waitFor(() => expect(operationalContainer.appendBlock).toHaveBeenCalledOnce()); + + const secondLog = destination.log(operational); + const thirdLog = destination.log(operational); + + finishAppend?.(); + + await Promise.all([firstLog, secondLog, thirdLog]); + + expect(operationalContainer.appendBlock).toHaveBeenCalledTimes(2); + + const [, [batchedContent]] = operationalContainer.appendBlock.mock.calls as [[string, number], [string, number]]; + + expect(batchedContent.match(/"message":"operational entry"/gu)).toHaveLength(2); + }); + it('should skip disabled streams without appending records', async () => { const operationalContainer = createContainer(); const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { From 4b1c87968b62247a957a26ab5465563e535cbfb6 Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 20:14:53 +0100 Subject: [PATCH 15/16] Remove unused function --- src/plugins/AzureStorageDestination.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts index 146e96a..b2b5744 100644 --- a/src/plugins/AzureStorageDestination.ts +++ b/src/plugins/AzureStorageDestination.ts @@ -434,18 +434,6 @@ export class AzureStorageDestination extends LoggingPlugin { }; } - static #shouldCreateNewBlob(currentActiveDate?: number): { - 'shouldCreate': boolean, - 'activeHour': number; - } { - const activeHour = new Date().setUTCMinutes(0, 0, 0); - - return { - activeHour, - 'shouldCreate': currentActiveDate === void 0 || activeHour > currentActiveDate - }; - } - #writeDebugLog(message: string, ...optionalParams: unknown[]): void { if ( this.#appliedOptions.getShouldWriteDebugInfo && From 32488eba56e9fd6c71567080e1fe792c823e710b Mon Sep 17 00:00:00 2001 From: pr0uxx Date: Wed, 16 Sep 2026 21:01:14 +0100 Subject: [PATCH 16/16] Fix Azure blob rotation race handling Recover committed block counts when reopening existing append blobs, and use Azure's reported count after appends so concurrent writers stay in sync. This also skips over blobs that are already full instead of reusing them, with tests covering reopen, rotation, and count tracking. --- .../plugins/AzureStorageDestination.ts | 11 ++- src/plugins/AzureStorageDestination.ts | 69 ++++++++++++++-- tests/plugins/AzureStorageDestination.test.ts | 82 ++++++++++++++++++- 3 files changed, 148 insertions(+), 14 deletions(-) diff --git a/src/interfaces/plugins/AzureStorageDestination.ts b/src/interfaces/plugins/AzureStorageDestination.ts index 886311a..a4440fc 100644 --- a/src/interfaces/plugins/AzureStorageDestination.ts +++ b/src/interfaces/plugins/AzureStorageDestination.ts @@ -8,9 +8,14 @@ import type { tags } from 'typia'; */ export interface AzureAppendBlobClientLike { /** Appends a block of content to the append blob. */ - 'appendBlock': (content: string, contentLength: number) => Promise; - /** Creates the append blob if it does not already exist. */ - 'createIfNotExists': () => Promise<{ 'errorCode'?: string; }>; + 'appendBlock': (content: string, contentLength: number) => Promise<{ 'blobCommittedBlockCount'?: number; }>; + /** Creates the append blob if it does not already exist; `succeeded` is false when it already existed. */ + 'createIfNotExists': () => Promise<{ + 'errorCode'?: string; + 'succeeded'?: boolean; + }>; + /** Reads current blob properties, used to recover the true committed block count of a reopened blob. */ + 'getProperties': () => Promise<{ 'blobCommittedBlockCount'?: number; }>; } /** diff --git a/src/plugins/AzureStorageDestination.ts b/src/plugins/AzureStorageDestination.ts index b2b5744..afe4576 100644 --- a/src/plugins/AzureStorageDestination.ts +++ b/src/plugins/AzureStorageDestination.ts @@ -30,6 +30,9 @@ interface LogTypeCollection { * Utilizes Azure Blob Storage append blobs for log storage. */ export class AzureStorageDestination extends LoggingPlugin { + /** Upper bound on suffix advances while searching for a non-full blob, guarding against an unbounded loop. */ + static readonly #maxBlobRotationAttempts = 1000; + readonly #operationalCollection: LogTypeCollection | undefined = void 0; readonly #auditCollection: LogTypeCollection | undefined = void 0; readonly #appliedOptions: ResolvedAzureStorageDestinationOptions; @@ -296,9 +299,11 @@ export class AzureStorageDestination extends LoggingPlugin { const content = group.map((record) => record.content).join(''); - await collection.activeBlob.appendBlock(content, Buffer.byteLength(content)); + const response = await collection.activeBlob.appendBlock(content, Buffer.byteLength(content)); - collection.blockCount += 1; + // Prefer Azure's authoritative count so concurrent writers to the same blob stay in sync. + // eslint-disable-next-line require-atomic-updates -- #writeGroup only runs within a single-flight flush per collection. + collection.blockCount = response.blobCommittedBlockCount ?? collection.blockCount + 1; } /** @@ -318,12 +323,27 @@ export class AzureStorageDestination extends LoggingPlugin { } // Reset the suffix on a new hour; otherwise advance it to rotate within the same hour. - const blobSuffix = isNewHour ? 0 : collection.blobSuffix + 1; + let blobSuffix = isNewHour ? 0 : collection.blobSuffix + 1; + + let openedBlob = await this.#openBlob(type, activeHour, blobSuffix); + + let rotationAttempts = 0; - const activeBlob = await this.#createNewBlob(type, activeHour, blobSuffix); + // A reopened blob (from a prior process, or shared with another instance) may already be at capacity; skip past it. + while (openedBlob.committedBlockCount >= this.#appliedOptions.maxBlocksPerBlob) { + rotationAttempts += 1; + + if (rotationAttempts > AzureStorageDestination.#maxBlobRotationAttempts) { + throw new Error(`Failed to find an available ${ type } blob after ${ AzureStorageDestination.#maxBlobRotationAttempts } rotation attempts.`); + } + + blobSuffix += 1; + + openedBlob = await this.#openBlob(type, activeHour, blobSuffix); + } // eslint-disable-next-line require-atomic-updates -- #ensureActiveBlob only runs within a single-flight flush per collection. - collection.activeBlob = activeBlob; + collection.activeBlob = openedBlob.blobClient; // eslint-disable-next-line require-atomic-updates -- #ensureActiveBlob only runs within a single-flight flush per collection. collection.activeBlobDate = activeHour; @@ -332,7 +352,7 @@ export class AzureStorageDestination extends LoggingPlugin { collection.blobSuffix = blobSuffix; // eslint-disable-next-line require-atomic-updates -- #ensureActiveBlob only runs within a single-flight flush per collection. - collection.blockCount = 0; + collection.blockCount = openedBlob.committedBlockCount; } /** @@ -371,7 +391,18 @@ export class AzureStorageDestination extends LoggingPlugin { return groups; } - async #createNewBlob(type: 'audit' | 'operational', activeHour: number, suffix: number): Promise { + /** + * Creates or reopens the append blob for a given hour and suffix, recovering its true committed block + * count when it already existed so callers can detect a blob that is already at capacity. + * @param type Discriminator identifying which collection is being opened. + * @param activeHour Millisecond timestamp of the top of the hour the blob belongs to. + * @param suffix Rotation suffix applied when opening a blob other than the first one for the hour. + * @returns The opened blob client, if the collection's container is available, alongside its true committed block count. + */ + async #openBlob(type: 'audit' | 'operational', activeHour: number, suffix: number): Promise<{ + 'blobClient': AzureAppendBlobClientLike | undefined; + 'committedBlockCount': number; + }> { const activeHourDate = new Date(activeHour); const isoValue = activeHourDate.toISOString(); @@ -391,7 +422,10 @@ export class AzureStorageDestination extends LoggingPlugin { // Todo - log internal don't throw this.#writeDebugLog(`Failed to access the ${ type } log collection.`); - return void 0; + return { + 'blobClient': void 0, + 'committedBlockCount': 0 + }; } const blobClient = collection.blobContainer.getAppendBlobClient(blobName); @@ -402,7 +436,24 @@ export class AzureStorageDestination extends LoggingPlugin { this.#writeDebugLog('Failed to create new blob:', result.errorCode); } - return blobClient; + // A blob that already existed may have been written by a prior process or another instance; recover its true block count. + if (result.succeeded === false) { + try { + const properties = await blobClient.getProperties(); + + return { + blobClient, + 'committedBlockCount': properties.blobCommittedBlockCount ?? 0 + }; + } catch (error: unknown) { + this.#writeDebugLog('Failed to read existing blob properties:', error); + } + } + + return { + blobClient, + 'committedBlockCount': 0 + }; } static #isContainerClient(value: unknown): value is AzureBlobContainerLike { diff --git a/tests/plugins/AzureStorageDestination.test.ts b/tests/plugins/AzureStorageDestination.test.ts index 2582f43..d56a632 100644 --- a/tests/plugins/AzureStorageDestination.test.ts +++ b/tests/plugins/AzureStorageDestination.test.ts @@ -31,9 +31,11 @@ const audit: AuditLog = { interface BlobContainerTestDouble { 'appendBlock': ReturnType; + 'blobCreateIfNotExists': ReturnType; 'blobNames': string[]; 'createIfNotExists': ReturnType; 'container': ContainerClient; + 'getProperties': ReturnType; } function createContainer(): BlobContainerTestDouble { @@ -41,9 +43,14 @@ function createContainer(): BlobContainerTestDouble { const blobNames: string[] = []; + const blobCreateIfNotExists = vi.fn(() => Promise.resolve({ 'succeeded': true })); + + const getProperties = vi.fn(() => Promise.resolve({})); + const appendBlob = { 'appendBlock': appendBlock, - 'createIfNotExists': vi.fn(() => Promise.resolve({ 'succeeded': true })) + 'createIfNotExists': blobCreateIfNotExists, + 'getProperties': getProperties } as unknown as AppendBlobClient; const createIfNotExists = vi.fn(() => Promise.resolve({ 'succeeded': true })); @@ -59,9 +66,11 @@ function createContainer(): BlobContainerTestDouble { return { appendBlock, + blobCreateIfNotExists, blobNames, createIfNotExists, - container + container, + getProperties }; } @@ -141,6 +150,75 @@ describe('AzureStorageDestination', () => { expect(operationalContainer.appendBlock).toHaveBeenCalledTimes(3); }); + it('should recover the true block count from an existing blob\'s properties when reopening it', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-02T03:04:05.678Z')); + + const operationalContainer = createContainer(); + + operationalContainer.blobCreateIfNotExists.mockResolvedValueOnce({ 'succeeded': false }); + operationalContainer.getProperties.mockResolvedValueOnce({ 'blobCommittedBlockCount': 49_999 }); + + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'maxBlocksPerBlob': 50_000 + }); + + await destination.log(operational); + await destination.log(operational); + + expect(operationalContainer.blobNames).toEqual([ + '2025010203.operational.log', + '2025010203.operational.2.log' + ]); + }); + + it('should advance past a reopened blob that has already reached the block limit', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-02T03:04:05.678Z')); + + const operationalContainer = createContainer(); + + operationalContainer.blobCreateIfNotExists + .mockResolvedValueOnce({ 'succeeded': false }) + .mockResolvedValueOnce({ 'succeeded': false }); + operationalContainer.getProperties + .mockResolvedValueOnce({ 'blobCommittedBlockCount': 1 }) + .mockResolvedValueOnce({ 'blobCommittedBlockCount': 0 }); + + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'maxBlocksPerBlob': 1 + }); + + await destination.log(operational); + + expect(operationalContainer.blobNames).toEqual([ + '2025010203.operational.log', + '2025010203.operational.2.log' + ]); + expect(operationalContainer.appendBlock).toHaveBeenCalledTimes(1); + }); + + it('should track the block count reported by Azure so concurrent writers to the same blob stay in sync', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-02T03:04:05.678Z')); + + const operationalContainer = createContainer(); + + operationalContainer.appendBlock.mockResolvedValueOnce({ 'blobCommittedBlockCount': 41 }); + + const destination = await AzureStorageDestination.create(operationalContainer.container, void 0, { + 'maxBlocksPerBlob': 41 + }); + + await destination.log(operational); + await destination.log(operational); + + expect(operationalContainer.blobNames).toEqual([ + '2025010203.operational.log', + '2025010203.operational.2.log' + ]); + }); + it('should batch records queued while a flush is in flight into a single append-blob block', async () => { let finishAppend: (() => void) | undefined;