@@ -14,6 +14,7 @@ if [[ $? != 0 ]]; then
fi
LOG_FILE=target/randomized-tests.log
+MIN_PLAN=target/test-plan.min.json
export SAVE_PLAN=target/test-plan.json
export OPERATIONS=200
export ITERATIONS=100000
@@ -27,10 +28,11 @@ if [[ $? == 0 ]]; then
exit 0
fi
+failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN)
+
# If the tests failed, find the failing seed in the logs
commit=$(git rev-parse HEAD)
-failing_seed=$(grep "failing seed" $LOG_FILE | tail -n1 | cut -d: -f2 | xargs)
-failing_plan=$(cat $SAVE_PLAN)
+failing_plan=$(cat $MIN_PLAN)
request="{
\"seed\": \"${failing_seed}\",
\"commit\": \"${commit}\",
@@ -0,0 +1,104 @@
+#!/usr/bin/env node --redirect-warnings=/dev/null
+
+const fs = require('fs')
+const path = require('path')
+const {spawnSync} = require('child_process')
+
+if (process.argv.length < 4) {
+ process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
+ process.exit(1)
+}
+
+const inputPlanPath = process.argv[2]
+const outputPlanPath = process.argv[3]
+const startIndex = parseInt(process.argv[4]) || 0
+
+const tempPlanPath = inputPlanPath + '.try'
+
+const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
+
+fs.copyFileSync(inputPlanPath, outputPlanPath)
+let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+
+process.stderr.write("minimizing failing test plan...\n")
+for (let ix = startIndex; ix < testPlan.length; ix++) {
+ // Skip 'MutateClients' entries, since they themselves are not single operations.
+ if (testPlan[ix].MutateClients) {
+ continue
+ }
+
+ // Remove a row from the test plan
+ const newTestPlan = testPlan.slice()
+ newTestPlan.splice(ix, 1)
+ fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+
+ process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+
+ const failingSeed = runTestsForPlan(tempPlanPath, 500)
+
+ // If the test failed, keep the test plan with the removed row. Reload the test
+ // plan from the JSON file, since the test itself will remove any operations
+ // which are no longer valid before saving the test plan.
+ if (failingSeed != null) {
+ process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
+ fs.copyFileSync(tempPlanPath, outputPlanPath)
+ testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+ ix--
+ } else {
+ process.stderr.write(` - keep.\n`)
+ }
+}
+
+fs.unlinkSync(tempPlanPath)
+
+// Re-run the final minimized plan to get the correct failing seed.
+// This is a workaround for the fact that the execution order can
+// slightly change when replaying a test plan after it has been
+// saved and loaded.
+const failingSeed = runTestsForPlan(outputPlanPath, 5000)
+
+process.stderr.write(`final test plan: ${outputPlanPath}\n`)
+process.stderr.write(`final seed: ${failingSeed}\n`)
+console.log(failingSeed)
+
+function runTestsForPlan(path, iterations) {
+ const {status, stdout, stderr} = spawnSync(
+ 'cargo',
+ [
+ 'test',
+ '--release',
+ '--lib',
+ '--package', 'collab',
+ 'random_collaboration'
+ ],
+ {
+ stdio: 'pipe',
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ 'SEED': 0,
+ 'LOAD_PLAN': path,
+ 'SAVE_PLAN': path,
+ 'ITERATIONS': String(iterations),
+ }
+ }
+ );
+
+ if (status !== 0) {
+ FAILING_SEED_REGEX.lastIndex = 0
+ const match = FAILING_SEED_REGEX.exec(stdout)
+ if (!match) {
+ process.stderr.write("test failed, but no failing seed found:\n")
+ process.stderr.write(stdout)
+ process.stderr.write('\n')
+ process.exit(1)
+ }
+ return match[1]
+ } else {
+ return null
+ }
+}
+
+function serializeTestPlan(plan) {
+ return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
+}