1#!/usr/bin/env node --redirect-warnings=/dev/null
2
3const fs = require('fs')
4const path = require('path')
5const {spawnSync} = require('child_process')
6
7const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
8const CARGO_TEST_ARGS = [
9 '--release',
10 '--lib',
11 '--package', 'collab',
12 'random_collaboration',
13]
14
15if (require.main === module) {
16 if (process.argv.length < 4) {
17 process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
18 process.exit(1)
19 }
20
21 minimizeTestPlan(
22 process.argv[2],
23 process.argv[3],
24 parseInt(process.argv[4]) || 0
25 );
26}
27
28function minimizeTestPlan(
29 inputPlanPath,
30 outputPlanPath,
31 startIndex = 0
32) {
33 const tempPlanPath = inputPlanPath + '.try'
34
35 fs.copyFileSync(inputPlanPath, outputPlanPath)
36 let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
37
38 process.stderr.write("minimizing failing test plan...\n")
39 for (let ix = startIndex; ix < testPlan.length; ix++) {
40 // Skip 'MutateClients' entries, since they themselves are not single operations.
41 if (testPlan[ix].MutateClients) {
42 continue
43 }
44
45 // Remove a row from the test plan
46 const newTestPlan = testPlan.slice()
47 newTestPlan.splice(ix, 1)
48 fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
49
50 process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
51 const failingSeed = runTests({
52 SEED: '0',
53 LOAD_PLAN: tempPlanPath,
54 SAVE_PLAN: tempPlanPath,
55 ITERATIONS: '500'
56 })
57
58 // If the test failed, keep the test plan with the removed row. Reload the test
59 // plan from the JSON file, since the test itself will remove any operations
60 // which are no longer valid before saving the test plan.
61 if (failingSeed != null) {
62 process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
63 fs.copyFileSync(tempPlanPath, outputPlanPath)
64 testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
65 ix--
66 } else {
67 process.stderr.write(` - keep.\n`)
68 }
69 }
70
71 fs.unlinkSync(tempPlanPath)
72
73 // Re-run the final minimized plan to get the correct failing seed.
74 // This is a workaround for the fact that the execution order can
75 // slightly change when replaying a test plan after it has been
76 // saved and loaded.
77 const failingSeed = runTests({
78 SEED: '0',
79 ITERATIONS: '5000',
80 LOAD_PLAN: outputPlanPath,
81 })
82
83 process.stderr.write(`final test plan: ${outputPlanPath}\n`)
84 process.stderr.write(`final seed: ${failingSeed}\n`)
85 return failingSeed
86}
87
88function buildTests() {
89 const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
90 stdio: 'inherit',
91 encoding: 'utf8',
92 env: {
93 ...process.env,
94 }
95 });
96 if (status !== 0) {
97 throw new Error('build failed')
98 }
99}
100
101function runTests(env) {
102 const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], {
103 stdio: 'pipe',
104 encoding: 'utf8',
105 env: {
106 ...process.env,
107 ...env,
108 }
109 });
110
111 if (status !== 0) {
112 FAILING_SEED_REGEX.lastIndex = 0
113 const match = FAILING_SEED_REGEX.exec(stdout)
114 if (!match) {
115 process.stderr.write("test failed, but no failing seed found:\n")
116 process.stderr.write(stdout)
117 process.stderr.write('\n')
118 process.exit(1)
119 }
120 return match[1]
121 } else {
122 return null
123 }
124}
125
126function serializeTestPlan(plan) {
127 return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
128}
129
130exports.buildTests = buildTests
131exports.runTests = runTests
132exports.minimizeTestPlan = minimizeTestPlan