@@ -1,50 +1,63 @@
-#!/bin/bash
-
-set -u
-
-: $ZED_SERVER_URL
-: $ZED_CLIENT_SECRET_TOKEN
-
-# Compile the tests first
-mkdir -p target
-cargo test --release --lib --package collab --no-run
-if [[ $? != 0 ]]; then
- echo "Build failed"
- exit 1
-fi
-
-LOG_FILE=target/randomized-tests.log
-MIN_PLAN=target/test-plan.min.json
-export SAVE_PLAN=target/test-plan.json
-export OPERATIONS=200
-export ITERATIONS=100000
-export SEED=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
-
-echo "Starting seed: ${SEED}"
-
-cargo test --release --lib --package collab random 2>&1 > $LOG_FILE
-if [[ $? == 0 ]]; then
- echo "Tests passed"
- exit 0
-fi
-
-failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN)
-
-# If the tests failed, find the failing seed in the logs
-commit=$(git rev-parse HEAD)
-failing_plan=$(cat $MIN_PLAN)
-request="{
- \"seed\": \"${failing_seed}\",
- \"commit\": \"${commit}\",
- \"token\": \"${ZED_CLIENT_SECRET_TOKEN}\",
- \"plan\": ${failing_plan}
-}"
-
-echo "Reporting test failure."
-echo $request
-
-curl \
- -X POST \
- -H "Content-Type: application/json" \
- -d "${request}" \
- "${ZED_SERVER_URL}/api/randomized_test_failure"
+#!/usr/bin/env node --redirect-warnings=/dev/null
+
+const fs = require('fs')
+const {randomBytes} = require('crypto')
+const {execFileSync} = require('child_process')
+const {minimizeTestPlan, buildTests, runTests} = require('./randomized-test-minimize');
+
+const {ZED_SERVER_URL, ZED_CLIENT_SECRET_TOKEN} = process.env
+if (!ZED_SERVER_URL) throw new Error('Missing env var `ZED_SERVER_URL`')
+if (!ZED_CLIENT_SECRET_TOKEN) throw new Error('Missing env var `ZED_CLIENT_SECRET_TOKEN`')
+
+main()
+
+async function main() {
+ buildTests()
+
+ const seed = randomU64();
+ const commit = execFileSync(
+ 'git',
+ ['rev-parse', 'HEAD'],
+ {encoding: 'utf8'}
+ ).trim()
+
+ console.log("commit:", commit)
+ console.log("starting seed:", seed)
+
+ const planPath = 'target/test-plan.json'
+ const minPlanPath = 'target/test-plan.min.json'
+ const failingSeed = runTests({
+ SEED: seed,
+ SAVE_PLAN: planPath,
+ ITERATIONS: 50000,
+ OPERATIONS: 200,
+ })
+
+ if (!failingSeed) {
+ console.log("tests passed")
+ return
+ }
+
+ console.log("found failure at seed", failingSeed)
+ const minimizedSeed = minimizeTestPlan(planPath, minPlanPath)
+ const minimizedPlan = JSON.parse(fs.readFileSync(minPlanPath, 'utf8'))
+
+ const url = `${ZED_SERVER_URL}/api/randomized_test_failure`
+ const body = {
+ seed: minimizedSeed,
+ token: ZED_CLIENT_SECRET_TOKEN,
+ plan: minimizedPlan,
+ commit: commit,
+ }
+ await fetch(url, {
+ method: 'POST',
+ headers: {"Content-Type": "application/json"},
+ body: JSON.stringify(body)
+ })
+}
+
+function randomU64() {
+ const bytes = randomBytes(8)
+ const hexString = bytes.reduce(((string, byte) => string + byte.toString(16)), '')
+ return BigInt('0x' + hexString).toString(10)
+}
@@ -4,85 +4,109 @@ const fs = require('fs')
const path = require('path')
const {spawnSync} = require('child_process')
-if (process.argv.length < 4) {
- process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
- process.exit(1)
-}
-
-const inputPlanPath = process.argv[2]
-const outputPlanPath = process.argv[3]
-const startIndex = parseInt(process.argv[4]) || 0
-
-const tempPlanPath = inputPlanPath + '.try'
-
const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
-
-fs.copyFileSync(inputPlanPath, outputPlanPath)
-let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
-
-process.stderr.write("minimizing failing test plan...\n")
-for (let ix = startIndex; ix < testPlan.length; ix++) {
- // Skip 'MutateClients' entries, since they themselves are not single operations.
- if (testPlan[ix].MutateClients) {
- continue
+const CARGO_TEST_ARGS = [
+ '--release',
+ '--lib',
+ '--package', 'collab',
+ 'random_collaboration',
+]
+
+if (require.main === module) {
+ if (process.argv.length < 4) {
+ process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
+ process.exit(1)
}
- // Remove a row from the test plan
- const newTestPlan = testPlan.slice()
- newTestPlan.splice(ix, 1)
- fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+ minimizeTestPlan(
+ process.argv[2],
+ process.argv[3],
+ parseInt(process.argv[4]) || 0
+ );
+}
+
+function minimizeTestPlan(
+ inputPlanPath,
+ outputPlanPath,
+ startIndex = 0
+) {
+ const tempPlanPath = inputPlanPath + '.try'
+
+ fs.copyFileSync(inputPlanPath, outputPlanPath)
+ let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+
+ process.stderr.write("minimizing failing test plan...\n")
+ for (let ix = startIndex; ix < testPlan.length; ix++) {
+ // Skip 'MutateClients' entries, since they themselves are not single operations.
+ if (testPlan[ix].MutateClients) {
+ continue
+ }
- process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+ // Remove a row from the test plan
+ const newTestPlan = testPlan.slice()
+ newTestPlan.splice(ix, 1)
+ fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+
+ process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+ const failingSeed = runTests({
+ SEED: '0',
+ LOAD_PLAN: tempPlanPath,
+ SAVE_PLAN: tempPlanPath,
+ ITERATIONS: '500'
+ })
+
+ // If the test failed, keep the test plan with the removed row. Reload the test
+ // plan from the JSON file, since the test itself will remove any operations
+ // which are no longer valid before saving the test plan.
+ if (failingSeed != null) {
+ process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
+ fs.copyFileSync(tempPlanPath, outputPlanPath)
+ testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+ ix--
+ } else {
+ process.stderr.write(` - keep.\n`)
+ }
+ }
- const failingSeed = runTestsForPlan(tempPlanPath, 500)
+ fs.unlinkSync(tempPlanPath)
+
+ // Re-run the final minimized plan to get the correct failing seed.
+ // This is a workaround for the fact that the execution order can
+ // slightly change when replaying a test plan after it has been
+ // saved and loaded.
+ const failingSeed = runTests({
+ SEED: '0',
+ ITERATIONS: '5000',
+ LOAD_PLAN: outputPlanPath,
+ })
+
+ process.stderr.write(`final test plan: ${outputPlanPath}\n`)
+ process.stderr.write(`final seed: ${failingSeed}\n`)
+ return failingSeed
+}
- // If the test failed, keep the test plan with the removed row. Reload the test
- // plan from the JSON file, since the test itself will remove any operations
- // which are no longer valid before saving the test plan.
- if (failingSeed != null) {
- process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
- fs.copyFileSync(tempPlanPath, outputPlanPath)
- testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
- ix--
- } else {
- process.stderr.write(` - keep.\n`)
+function buildTests() {
+ const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
+ stdio: 'inherit',
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ }
+ });
+ if (status !== 0) {
+ throw new Error('build failed')
}
}
-fs.unlinkSync(tempPlanPath)
-
-// Re-run the final minimized plan to get the correct failing seed.
-// This is a workaround for the fact that the execution order can
-// slightly change when replaying a test plan after it has been
-// saved and loaded.
-const failingSeed = runTestsForPlan(outputPlanPath, 5000)
-
-process.stderr.write(`final test plan: ${outputPlanPath}\n`)
-process.stderr.write(`final seed: ${failingSeed}\n`)
-console.log(failingSeed)
-
-function runTestsForPlan(path, iterations) {
- const {status, stdout, stderr} = spawnSync(
- 'cargo',
- [
- 'test',
- '--release',
- '--lib',
- '--package', 'collab',
- 'random_collaboration'
- ],
- {
- stdio: 'pipe',
- encoding: 'utf8',
- env: {
- ...process.env,
- 'SEED': 0,
- 'LOAD_PLAN': path,
- 'SAVE_PLAN': path,
- 'ITERATIONS': String(iterations),
- }
+function runTests(env) {
+ const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], {
+ stdio: 'pipe',
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ ...env,
}
- );
+ });
if (status !== 0) {
FAILING_SEED_REGEX.lastIndex = 0
@@ -102,3 +126,7 @@ function runTestsForPlan(path, iterations) {
function serializeTestPlan(plan) {
return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
}
+
+exports.buildTests = buildTests
+exports.runTests = runTests
+exports.minimizeTestPlan = minimizeTestPlan