Skip to content

Commit faa5f0e

Browse files
ruyadornoruyadorno
committed
add registries benchmarks
1 parent 98052a3 commit faa5f0e

File tree

9 files changed

+411
-4
lines changed

9 files changed

+411
-4
lines changed

‎.github/workflows/benchmark.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88
default: '["next", "astro", "vue", "svelte"]'
99
variations:
1010
description: 'The benchmark variations to run'
11-
default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules"]'
11+
default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-local-cache", "registry-proxied"]'
1212
binaries:
1313
description: 'The binaries to run the benchmarks on'
1414
default: '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"'
@@ -34,7 +34,7 @@ jobs:
3434
strategy:
3535
matrix:
3636
fixture: ${{ fromJson(inputs.fixtures || '["next", "astro", "vue", "svelte"]') }}
37-
variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules"]') }}
37+
variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-local-cache", "registry-proxied"]') }}
3838
include:
3939
- variation: "run"
4040
fixture: "run"
@@ -115,7 +115,9 @@ jobs:
115115
"lockfile",
116116
"lockfile+node_modules",
117117
"node_modules",
118-
"run"
118+
"run",
119+
"registry-local-cache",
120+
"registry-proxied"
119121
];
120122
121123
// Helper functions for statistical calculations

‎registry-configs/verdaccio/config.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
storage: ./storage
2+
auth:
3+
htpasswd:
4+
file: ./htpasswd
5+
uplinks:
6+
npmjs:
7+
url: https://registry.npmjs.org/
8+
packages:
9+
'@*/*':
10+
access: $all
11+
publish: $authenticated
12+
proxy: npmjs
13+
'**':
14+
access: $all
15+
publish: $authenticated
16+
proxy: npmjs
17+
server:
18+
keepAliveTimeout: 60
19+
bodyParser:
20+
limit: 100mb
21+
logs:
22+
- type: stdout
23+
format: pretty
24+
level: warn

‎registry-configs/vsr/vlt.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"registry": {
3+
"upstream": "https://registry.npmjs.org/",
4+
"cache": "./cache",
5+
"port": 1337
6+
}
7+
}

‎scripts/benchmark.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ case "$2" in
4747
run)
4848
bash ../../scripts/variations/run.sh "../../scripts" "../../results" "run" "run"
4949
;;
50+
registry-local-cache)
51+
bash ../../scripts/variations/registry-local-cache.sh "../../scripts" "../../results" "$1" "$2"
52+
;;
53+
registry-proxied)
54+
bash ../../scripts/variations/registry-proxied.sh "../../scripts" "../../results" "$1" "$2"
55+
;;
5056
*)
5157
echo "Error: Unknown install variation '$2'"
5258
exit 1

‎scripts/process-results.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ echo "Processing results..."
5353

5454
# Process variations results
5555
for fixture in next astro svelte vue; do
56-
for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules clean lockfile lockfile+node_modules node_modules run; do
56+
for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules clean lockfile lockfile+node_modules node_modules run registry-local-cache registry-proxied; do
5757
if [ -f "results/results-$fixture-$variation/benchmarks.json" ]; then
5858
print_summary "results/results-$fixture-$variation/benchmarks.json" "$fixture" "$variation"
5959
cp "results/results-$fixture-$variation/benchmarks.json" "results/$DATE/$fixture-$variation.json"

‎scripts/registry-helpers.sh‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/bin/bash
2+
# Registry helper functions
3+
4+
# Registry ports
5+
VSR_PORT=1337
6+
VERDACCIO_PORT=4873
7+
8+
# Start registry function
9+
start_registry() {
10+
local registry_type=$1
11+
local output_folder=$2
12+
13+
# Kill any existing registry processes
14+
stop_registry "$registry_type"
15+
sleep 2
16+
17+
if [ "$registry_type" = "vsr" ]; then
18+
echo "Starting VSR registry on port $VSR_PORT..."
19+
cd "$(dirname "$0")/../registry-configs/vsr"
20+
vsr --port $VSR_PORT --config ./vlt.json > "$output_folder/vsr.log" 2>&1 &
21+
echo $! > /tmp/vsr.pid
22+
cd - > /dev/null
23+
24+
# Wait for VSR to start
25+
local retries=0
26+
while ! curl -s "http://localhost:$VSR_PORT/-/ping" > /dev/null 2>&1; do
27+
sleep 1
28+
retries=$((retries + 1))
29+
if [ $retries -gt 10 ]; then
30+
echo "Error: VSR failed to start"
31+
return 1
32+
fi
33+
done
34+
35+
echo "http://localhost:$VSR_PORT"
36+
else
37+
echo "Starting Verdaccio registry on port $VERDACCIO_PORT..."
38+
cd "$(dirname "$0")/../registry-configs/verdaccio"
39+
verdaccio --config ./config.yaml > "$output_folder/verdaccio.log" 2>&1 &
40+
echo $! > /tmp/verdaccio.pid
41+
cd - > /dev/null
42+
43+
# Wait for Verdaccio to start
44+
local retries=0
45+
while ! curl -s "http://localhost:$VERDACCIO_PORT/-/ping" > /dev/null 2>&1; do
46+
sleep 1
47+
retries=$((retries + 1))
48+
if [ $retries -gt 10 ]; then
49+
echo "Error: Verdaccio failed to start"
50+
return 1
51+
fi
52+
done
53+
54+
echo "http://localhost:$VERDACCIO_PORT"
55+
fi
56+
}
57+
58+
# Stop registry function
59+
stop_registry() {
60+
local registry_type=$1
61+
62+
if [ "$registry_type" = "vsr" ]; then
63+
if [ -f /tmp/vsr.pid ]; then
64+
kill $(cat /tmp/vsr.pid) 2>/dev/null || true
65+
rm -f /tmp/vsr.pid
66+
fi
67+
pkill -f "vsr --port $VSR_PORT" || true
68+
else
69+
if [ -f /tmp/verdaccio.pid ]; then
70+
kill $(cat /tmp/verdaccio.pid) 2>/dev/null || true
71+
rm -f /tmp/verdaccio.pid
72+
fi
73+
pkill -f "verdaccio --config" || true
74+
fi
75+
}
76+
77+
# Clean registry cache function
78+
clean_registry_cache() {
79+
local registry_type=$1
80+
81+
if [ "$registry_type" = "vsr" ]; then
82+
rm -rf "$(dirname "$0")/../registry-configs/vsr/cache" || true
83+
else
84+
rm -rf "$(dirname "$0")/../registry-configs/verdaccio/storage" || true
85+
fi
86+
}
87+
88+
# Stop all registries
89+
stop_all_registries() {
90+
stop_registry "vsr"
91+
stop_registry "verdaccio"
92+
}
93+
94+
# Main execution logic
95+
if [ $# -eq 0 ]; then
96+
echo "Usage: $0 <command> [args]"
97+
echo "Commands:"
98+
echo " start <vsr|verdaccio> <output_folder> - Start a registry"
99+
echo " stop <vsr|verdaccio> - Stop a registry"
100+
echo " stop_all - Stop all registries"
101+
echo " clean <vsr|verdaccio> - Clean registry cache"
102+
exit 1
103+
fi
104+
105+
case "$1" in
106+
start)
107+
if [ $# -lt 3 ]; then
108+
echo "Usage: $0 start <vsr|verdaccio> <output_folder>"
109+
exit 1
110+
fi
111+
start_registry "$2" "$3"
112+
;;
113+
stop)
114+
if [ $# -lt 2 ]; then
115+
echo "Usage: $0 stop <vsr|verdaccio>"
116+
exit 1
117+
fi
118+
stop_registry "$2"
119+
;;
120+
stop_all)
121+
stop_all_registries
122+
;;
123+
clean)
124+
if [ $# -lt 2 ]; then
125+
echo "Usage: $0 clean <vsr|verdaccio>"
126+
exit 1
127+
fi
128+
clean_registry_cache "$2"
129+
;;
130+
*)
131+
echo "Unknown command: $1"
132+
exit 1
133+
;;
134+
esac

‎scripts/setup.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ echo "hyperfine: $HYPERFINE_VERSION"
3434
echo "Installing package managers and tools..."
3535
npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest
3636

37+
# Install alternative registries
38+
echo "Installing alternative registries..."
39+
npm install -g @vltpkg/vsr@latest verdaccio@latest
40+
3741
# Configure Package Managers
3842
echo "Configuring package managers..."
3943
corepack enable yarn pnpm
@@ -83,4 +87,49 @@ echo "{
8387
\"node\": \"$NODE_VERSION\"
8488
}" > ./results/versions.json
8589

90+
# Create registry configuration directories
91+
echo "Creating registry configuration directories..."
92+
mkdir -p ./registry-configs/vsr
93+
mkdir -p ./registry-configs/verdaccio
94+
95+
# Create Verdaccio configuration
96+
cat > ./registry-configs/verdaccio/config.yaml << 'EOF'
97+
storage: ./storage
98+
auth:
99+
htpasswd:
100+
file: ./htpasswd
101+
uplinks:
102+
npmjs:
103+
url: https://registry.npmjs.org/
104+
packages:
105+
'@*/*':
106+
access: $all
107+
publish: $authenticated
108+
proxy: npmjs
109+
'**':
110+
access: $all
111+
publish: $authenticated
112+
proxy: npmjs
113+
server:
114+
keepAliveTimeout: 60
115+
bodyParser:
116+
limit: 100mb
117+
logs:
118+
- type: stdout
119+
format: pretty
120+
level: warn
121+
EOF
122+
123+
# Create VSR configuration
124+
cat > ./registry-configs/vsr/vlt.json << 'EOF'
125+
{
126+
"registry": {
127+
"upstream": "https://registry.npmjs.org/",
128+
"cache": "./cache",
129+
"port": 1337
130+
}
131+
}
132+
EOF
133+
134+
echo "Registry configurations created successfully!"
86135
echo "Setup completed successfully!"

‎scripts/variations/registry-local-cache.sh‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/bin/bash
2+
# Exit on error
3+
set -Eeuxo pipefail
4+
5+
# Load common variables
6+
source "$1/variations/common.sh"
7+
8+
# Update command strings to use registry configuration
9+
update_commands_for_registry() {
10+
BENCH_COMMAND_NPM="npm install --registry=$BENCH_REGISTRY_URL --no-audit --no-fund --silent >> $BENCH_OUTPUT_FOLDER/npm-output-\${HYPERFINE_ITERATION}.log 2>&1"
11+
BENCH_COMMAND_YARN="corepack yarn@1 install --registry $BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/yarn-output-\${HYPERFINE_ITERATION}.log 2>&1"
12+
BENCH_COMMAND_BERRY="echo \"$BENCH_COMMAND_BERRY_PRE\" > .yarnrc.yml; echo \"npmRegistryServer: $BENCH_REGISTRY_URL\" >> .yarnrc.yml; corepack yarn@latest install > $BENCH_OUTPUT_FOLDER/berry-output-\${HYPERFINE_ITERATION}.log 2>&1"
13+
BENCH_COMMAND_PNPM="corepack pnpm@latest install --registry=$BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/pnpm-output-\${HYPERFINE_ITERATION}.log 2>&1"
14+
BENCH_COMMAND_VLT="vlt install --registry=$BENCH_REGISTRY_URL --view=silent > $BENCH_OUTPUT_FOLDER/vlt-output-\${HYPERFINE_ITERATION}.log 2>&1"
15+
# Bun and Deno don't support custom registries in the same way
16+
BENCH_COMMAND_BUN=""
17+
BENCH_COMMAND_DENO=""
18+
BENCH_INCLUDE_BUN=""
19+
BENCH_INCLUDE_DENO=""
20+
}
21+
22+
# Pre-warm registry function
23+
warm_registry() {
24+
local registry_type=$1
25+
26+
echo "Pre-warming $registry_type registry..."
27+
28+
# Save current directory
29+
local current_dir=$(pwd)
30+
31+
# Use npm to install packages and warm the cache
32+
npm install --registry="$BENCH_REGISTRY_URL" --silent > "$BENCH_OUTPUT_FOLDER/warm-$registry_type.log" 2>&1 || true
33+
34+
# Clean up after warming
35+
bash "$BENCH_SCRIPTS/clean-helpers.sh" clean_node_modules clean_lockfiles
36+
37+
cd "$current_dir"
38+
}
39+
40+
# Function to run benchmarks for a specific registry
41+
run_registry_benchmark() {
42+
local registry_type=$1
43+
44+
# Stop any existing registries
45+
bash "$BENCH_SCRIPTS/registry-helpers.sh" stop_all
46+
47+
# Clean the registry cache first
48+
bash "$BENCH_SCRIPTS/registry-helpers.sh" clean "$registry_type"
49+
50+
# Start the registry
51+
BENCH_REGISTRY_URL=$(bash "$BENCH_SCRIPTS/registry-helpers.sh" start "$registry_type" "$BENCH_OUTPUT_FOLDER")
52+
export BENCH_REGISTRY_URL
53+
54+
echo "Registry started at: $BENCH_REGISTRY_URL"
55+
56+
# Update commands with registry URL
57+
update_commands_for_registry
58+
59+
# Pre-warm the registry
60+
warm_registry "$registry_type"
61+
62+
# Run the benchmark suite
63+
echo "Running benchmarks with $registry_type registry (pre-warmed)..."
64+
hyperfine --ignore-failure \
65+
--time-unit=millisecond \
66+
--export-json="$BENCH_OUTPUT_FOLDER/benchmarks-$registry_type.json" \
67+
--warmup="$BENCH_WARMUP" \
68+
--runs="$BENCH_RUNS" \
69+
--prepare="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \
70+
--conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \
71+
--cleanup="bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \
72+
${BENCH_INCLUDE_NPM:+--command-name="npm-$registry_type" "$BENCH_COMMAND_NPM"} \
73+
${BENCH_INCLUDE_YARN:+--command-name="yarn-$registry_type" "$BENCH_COMMAND_YARN"} \
74+
${BENCH_INCLUDE_BERRY:+--command-name="berry-$registry_type" "$BENCH_COMMAND_BERRY"} \
75+
${BENCH_INCLUDE_PNPM:+--command-name="pnpm-$registry_type" "$BENCH_COMMAND_PNPM"} \
76+
${BENCH_INCLUDE_VLT:+--command-name="vlt-$registry_type" "$BENCH_COMMAND_VLT"}
77+
78+
# Stop the registry
79+
bash "$BENCH_SCRIPTS/registry-helpers.sh" stop "$registry_type"
80+
}
81+
82+
# Run benchmarks for both registries
83+
run_registry_benchmark "vsr"
84+
run_registry_benchmark "verdaccio"
85+
86+
# Merge results
87+
echo "Merging benchmark results..."
88+
jq -s '.[0] * {results: (.[0].results + .[1].results)}' \
89+
"$BENCH_OUTPUT_FOLDER/benchmarks-vsr.json" \
90+
"$BENCH_OUTPUT_FOLDER/benchmarks-verdaccio.json" \
91+
> "$BENCH_OUTPUT_FOLDER/benchmarks.json"
92+
93+
collect_package_count

0 commit comments

Comments
 (0)