Initial import of streamline

This commit is contained in:
2026-11-16 17:26:44 -08:00
commit 8e4aec1855
1221 changed files with 274256 additions and 0 deletions
+74
View File
@@ -0,0 +1,74 @@
/**
* TOS S3 upload load test — drives the real `tos.ts` module (same code path the
* control plane uses) and reports latency percentiles + error breakdown.
*
* bun scripts/tos_loadtest.ts [--conc 16] [--total 400] [--size 64k] [--secs 0]
*/
import { uploadImageToTos, uploadEvidenceToTos, isTosConfigured, evidenceContentType } from "../control-server/src/tos";
function arg(name: string, dflt: string): string {
const i = process.argv.indexOf(`--${name}`);
return i > 0 && process.argv[i + 1] ? process.argv[i + 1]! : dflt;
}
function bytes(spec: string): number {
const m = /^(\d+)([kKmM]?)$/.exec(spec);
if (!m) throw new Error(`bad size: ${spec}`);
return Number(m[1]) * (m[2]?.toLowerCase() === "k" ? 1024 : m[2]?.toLowerCase() === "m" ? 1024 * 1024 : 1);
}
const CONC = Number(arg("conc", "16"));
const TOTAL = Number(arg("total", "400"));
const SIZE = bytes(arg("size", "64k"));
const SECS = Number(arg("secs", "0")); // >0 → run for a duration instead of a fixed count
if (!isTosConfigured()) { console.error("TOS not configured (need TOS_ACCESS_KEY + TOS_SECRET_KEY)"); process.exit(1); }
const payload = new Uint8Array(SIZE);
for (let i = 0; i < SIZE; i++) payload[i] = i & 0xff;
const lat: number[] = [];
const errors = new Map<string, number>();
let done = 0, ok = 0, fail = 0;
const started = Date.now();
const deadline = SECS > 0 ? started + SECS * 1000 : Infinity;
function more(): boolean {
return SECS > 0 ? Date.now() < deadline : done < TOTAL;
}
async function worker(id: number): Promise<void> {
for (let n = 0; more(); n++) {
done++;
const t0 = performance.now();
try {
await uploadImageToTos(`_loadtest/w${id}_${n}.bin`, payload, "application/octet-stream");
lat.push(performance.now() - t0);
ok++;
} catch (err) {
fail++;
const msg = (err as Error).message.slice(0, 120);
errors.set(msg, (errors.get(msg) ?? 0) + 1);
}
}
}
function pct(sorted: number[], p: number): number {
if (!sorted.length) return NaN;
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]!;
}
await Promise.all(Array.from({ length: CONC }, (_, i) => worker(i)));
const wall = (Date.now() - started) / 1000;
const s = [...lat].sort((a, b) => a - b);
const f = (n: number) => (Number.isNaN(n) ? "-" : n.toFixed(1));
console.log(`\n=== TOS S3 PUT load test ===`);
console.log(`conc=${CONC} size=${(SIZE / 1024).toFixed(0)}KB requests=${ok + fail} wall=${wall.toFixed(1)}s`);
console.log(`ok=${ok} fail=${fail} error_rate=${((fail / (ok + fail)) * 100).toFixed(2)}%`);
console.log(`throughput=${((ok + fail) / wall).toFixed(1)} req/s ${(((ok * SIZE) / 1024 / 1024) / wall).toFixed(1)} MB/s`);
console.log(`latency ms: min=${f(s[0]!)} p50=${f(pct(s, 50))} p90=${f(pct(s, 90))} p99=${f(pct(s, 99))} max=${f(s[s.length - 1]!)}`);
if (errors.size) {
console.log(`\nerrors:`);
for (const [msg, n] of [...errors].sort((a, b) => b[1] - a[1])) console.log(` ${n}x ${msg}`);
}
process.exit(fail > 0 ? 1 : 0);