From 93395694bdf8f1cc307436980e42030b4a595788 Mon Sep 17 00:00:00 2001 From: hesprs <190185753+hesprs@users.noreply.github.com> Date: Tue, 29 Sep 2026 23:53:25 +0800 Subject: [PATCH] chore(benchmark): add S3 benchmark --- CHANGELOG.md | 2 +- docs/.vitepress/config.ts | 1 + docs/.vitepress/i18n.ts | 1 + docs/src/pages/en/deep-dive/benchmarking.md | 12 +--- docs/src/pages/en/usage/benchmark.md | 31 +++++++++-- packages/encryption/src/setting.ts | 1 - scripts/benchmark/speed-test.ts | 33 ----------- scripts/benchmark/testing.md | 62 +++++++++++++++++++++ 8 files changed, 91 insertions(+), 52 deletions(-) delete mode 100644 scripts/benchmark/speed-test.ts create mode 100644 scripts/benchmark/testing.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b960b356..c19b6989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to this project will be documented in this file. ### Core -- Unified sync strategy, inclusion rules, and exclusion rules into an ordered lost of sync strategies, providing granular control of syncing decisions. +- Unified sync strategy, inclusion rules, and exclusion rules into an ordered list of sync strategies, providing granular control of syncing decisions. - Supported brace expansion syntax in sync strategy Glob rules. - Replaced remote migration wizard with a confirmation pop-up reminding manual migration since the set of files to upload can no longer be reliably determined on a single device. - Fixed local operation failure caused by moving a file onto an existing file. diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index e9af6508..1f405055 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -123,6 +123,7 @@ const localeConfig = configGenerator((t) => { { link: `${deepDive}/architecture`, text: t('architecture') }, { link: `${deepDive}/sync`, text: t('sync') }, { link: `${deepDive}/extensibility`, text: t('extensibility') }, + { link: `${deepDive}/benchmarking`, text: t('benchmarking') }, { items: [ { link: `${deepDive}/file-system`, text: t('fileSystem') }, diff --git a/docs/.vitepress/i18n.ts b/docs/.vitepress/i18n.ts index 18573415..46c04cf9 100644 --- a/docs/.vitepress/i18n.ts +++ b/docs/.vitepress/i18n.ts @@ -6,6 +6,7 @@ const en = { architecture: 'Architecture', asymmetricStorage: 'Asymmetric Storage', benchmark: 'Benchmark', + benchmarking: 'Benchmarking', claims: 'Claims', code: 'en-US', contributing: 'Contributing', diff --git a/docs/src/pages/en/deep-dive/benchmarking.md b/docs/src/pages/en/deep-dive/benchmarking.md index e55c7cfd..bbc6bb3b 100644 --- a/docs/src/pages/en/deep-dive/benchmarking.md +++ b/docs/src/pages/en/deep-dive/benchmarking.md @@ -51,7 +51,7 @@ Run [Correctness Validator](#correctness-validator) on both original and replica ## Benchmarking Utilities -The benchmarking utilities are located in `scripts/benchmark/` in this repo. It is a CLI application majorly interacting with Obsidian CLI with five modules: Test Vault Bootstrapper, Operation Executor, Correctness Validator, Speed Tester, and Ping Tester. +The benchmarking utilities are located in `scripts/benchmark/` in this repo. It is a CLI application majorly interacting with Obsidian CLI with five modules: Test Vault Bootstrapper, Operation Executor, Correctness Validator, and Ping Tester. ### Test Vault Bootstrapper @@ -91,16 +91,6 @@ The validator is used to validate the sync correctness after the set of operatio After validation, it outputs the total count of errors, and each error with its reason. -### Speed Tester - -File: `scripts/benchmark/speed-test.ts` - -The speed tester tests both upload and download speeds. It needs the benchmarker to edit `scripts/benchmark/speed-test.ts` to include necessary request endpoint and authorization headers. - -When started, it launches sequential `PUT` and `GET` requests to testing uploading and downloading a generated 50 MiB test file. After tests, it launches `DELETE` request to the same URL to clean up the test file. - -It computes the average upload and download speeds via 50MiB / recorded time. - ### Ping Tester File: `scripts/benchmark/ping-test.ts` diff --git a/docs/src/pages/en/usage/benchmark.md b/docs/src/pages/en/usage/benchmark.md index e7a33dd7..c2b051c5 100644 --- a/docs/src/pages/en/usage/benchmark.md +++ b/docs/src/pages/en/usage/benchmark.md @@ -8,9 +8,9 @@ Startup speed measures the time by which the plugin delays the cold start of Obs | | Sync Engine[^1] | Remotely Save | Nextcloud Sync | | ------------ | --------------- | ------------- | -------------- | -| Version | 3.0.0 | 0.5.25 | 0.7.38 | -| Startup Time | 25.4 ms | 251 ms | **18.6 ms** | -| Plugin Size | **145 KB** | 4050 KB | 515 KB | +| Version | 3.2.0 | 0.5.25 | 0.7.38 | +| Startup Time | 27.2 ms | 251 ms | **18.6 ms** | +| Plugin Size | **155 KB** | 4050 KB | 515 KB | [^1]: The startup speed of Sync Engine is measured with a module loaded. @@ -21,10 +21,10 @@ Startup speed measures the time by which the plugin delays the cold start of Obs - **CPU single core score**: around 1700 - **Backend**: Self-hosted Nextcloud WebDAV[^2] - **Average ping**: 400 ms -- **Average upload speed**: 2.6 MiB/s -- **Average download speed**: 4.8 MiB/s +- **Average upload speed**: 3 MiB/s +- **Average download speed**: 5 MiB/s -[^2]: During local testing, Nextcloud connection is delayed and bandwidth is limited to simulate middle-to-low network quality. +[^2]: During local testing, the connection is delayed and bandwidth is limited to simulate middle-to-low network quality. | Item | Sync Engine | Remotely Save | Nextcloud Sync | | ----------------------------------------- | --------------------------------------------------------- | ----------------------------------------- | --------------------------------------------------------------------- | @@ -38,3 +38,22 @@ Startup speed measures the time by which the plugin delays the cold start of Obs In terms of syncing performance, Sync Engine is visibly faster than Remotely Save in terms of full upload and download. In the more realistic daily sync test, Remotely Save fails catastrophically and Sync Engine is around **100x faster** than Remotely Save. Although claimed to be optimized for Nextcloud, Nextcloud Sync still shows visible disadvantage in terms of performance compared with Sync Engine. More importantly, it **fails to sync 98 original files or changes to the replica**, and many errors appeared during the testing process. + +## S3 Performance + +- **Obsidian version**: 1.13.7 +- **Operating system**: NixOS 26.11 +- **CPU single core score**: around 1700 +- **Backend**: Self-hosted RustFs[^2] +- **Average ping**: 400 ms +- **Average upload speed**: 3 MiB/s +- **Average download speed**: 5 MiB/s + +| Item | Sync Engine | +| ----------------------------------------- | ------------------------------------------------------- | +| Version | 3.2.0 + `s3` module 0.1.10 | +| 2000 files upload | **3.74 min** | +| 2000 files download | **2.37 min** | +| Daily simulation
(Original, 10 rounds) | Min: **0.35 s**
Median: **0.6 s**
Max: **1.08 s** | +| Daily simulation
(Replica, 10 rounds) | Min: **1.3 s**
Median: **1.58 s**
Max: **6.24 s** | +| Correctness validation | 0 errors | diff --git a/packages/encryption/src/setting.ts b/packages/encryption/src/setting.ts index 21792d84..fe56a083 100644 --- a/packages/encryption/src/setting.ts +++ b/packages/encryption/src/setting.ts @@ -1,7 +1,6 @@ import type { EncryptionSettings } from '@'; import type { CallableOrObjectTree, - Context, Fragment, LabelDefinition, MaybePromise, diff --git a/scripts/benchmark/speed-test.ts b/scripts/benchmark/speed-test.ts deleted file mode 100644 index 3a2da713..00000000 --- a/scripts/benchmark/speed-test.ts +++ /dev/null @@ -1,33 +0,0 @@ -// oxlint-disable no-console unicorn/require-module-specifiers -const url = ''; -const headers: Record = {}; - -const SIZE_BYTES = 50 * 1024 * 1024; -const chunks: Array> = []; -for (let remaining = SIZE_BYTES; remaining > 0; remaining -= 65_536) { - const chunk = new Uint8Array(Math.min(65_536, remaining)); - crypto.getRandomValues(chunk); - chunks.push(chunk); -} -const payload = new Blob(chunks); - -const postStart = performance.now(); -const postRes = await fetch(url, { body: payload, headers, method: 'PUT' }); -if (!postRes.ok) throw new Error(`PUT ${postRes.status}`); -await postRes.arrayBuffer(); -const uploadSpeed = SIZE_BYTES / ((performance.now() - postStart) / 1000); - -const getStart = performance.now(); -const getRes = await fetch(url, { headers, method: 'GET' }); -if (!getRes.ok) throw new Error(`GET ${getRes.status}`); -const downloaded = await getRes.arrayBuffer(); -const downloadSpeed = downloaded.byteLength / ((performance.now() - getStart) / 1000); - -const delRes = await fetch(url, { headers, method: 'DELETE' }); -if (!delRes.ok && delRes.status !== 204 && delRes.status !== 404) - throw new Error(`DELETE ${delRes.status}`); - -console.log(`Upload: ${(uploadSpeed / 1_048_576).toFixed(2)} MiB/s`); -console.log(`Download: ${(downloadSpeed / 1_048_576).toFixed(2)} MiB/s`); - -export {}; diff --git a/scripts/benchmark/testing.md b/scripts/benchmark/testing.md new file mode 100644 index 00000000..94258ad4 --- /dev/null +++ b/scripts/benchmark/testing.md @@ -0,0 +1,62 @@ +# Local Testing + +This document specifies how to test services locally and simulate real network conditions. + +## WebDAV + +```sh +rclone serve webdav /home/hesprs/Desktop/DAV --addr :5000 +``` + +- Server URL: `http://localhost:5005` +- Any username and password + +## S3 + +Run RustFS in one terminal: + +```sh +mkdir $HOME/Desktop/S3 + +RUSTFS_ADDRESS=":5005" \ +RUSTFS_VOLUMES="$HOME/Desktop/S3" \ +RUSTFS_ACCESS_KEY="syncengine2026" \ +RUSTFS_SECRET_KEY="rustfs-test-secret" \ +rustfs +``` + +In a second terminal, create the bucket: + +```sh +export AWS_ACCESS_KEY_ID="syncengine2026" +export AWS_SECRET_ACCESS_KEY="rustfs-test-secret" +export AWS_DEFAULT_REGION="us-east-1" +awscli2 s3api create-bucket --bucket obsidian-sync --endpoint-url http://127.0.0.1:5005 +``` + +- Endpoint: `http://127.0.0.1:5005` +- Region: `us-east-1` +- Bucket: `obsidian-sync` +- Access key ID: `syncengine2026` +- Path-style URLs +- Secret access key: `rustfs-test-secret` + +## Artificial Latency and Bandwidth + +- 100ms latency per direction +- 3MiB/s inbound (client uploads) +- 5MiB/s outbound (client downloads) + +```sh +sudo tc qdisc add dev lo root handle 1: prio +sudo tc qdisc add dev lo parent 1:1 handle 10: netem delay 100ms rate 24576kbit +sudo tc qdisc add dev lo parent 1:2 handle 20: netem delay 100ms rate 40960kbit +sudo tc filter add dev lo protocol ip parent 1: prio 1 u32 match ip dport 5005 0xffff flowid 1:1 +sudo tc filter add dev lo protocol ip parent 1: prio 2 u32 match ip sport 5005 0xffff flowid 1:2 +``` + +Remove all shaping: + +```sh +sudo tc qdisc del dev lo root +```