1#!/usr/bin/env node --redirect-warnings=/dev/null
2
3const fs = require('fs')
4const path = require('path')
5const {spawnSync} = require('child_process')
6
7const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
8const CARGO_TEST_ARGS = [
9 '--release',
10 '--lib',
11 '--package', 'collab',
12]
13
14if (require.main === module) {
15 if (process.argv.length < 4) {
16 process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
17 process.exit(1)
18 }
19
20 minimizeTestPlan(
21 process.argv[2],
22 process.argv[3],
23 parseInt(process.argv[4]) || 0
24 );
25}
26
27function minimizeTestPlan(
28 inputPlanPath,
29 outputPlanPath,
30 startIndex = 0
31) {
32 const tempPlanPath = inputPlanPath + '.try'
33
34 fs.copyFileSync(inputPlanPath, outputPlanPath)
35 let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
36
37 process.stderr.write("minimizing failing test plan...\n")
38 for (let ix = startIndex; ix < testPlan.length; ix++) {
39 // Skip 'MutateClients' entries, since they themselves are not single operations.
40 if (testPlan[ix].MutateClients) {
41 continue
42 }
43
44 // Remove a row from the test plan
45 const newTestPlan = testPlan.slice()
46 newTestPlan.splice(ix, 1)
47 fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
48
49 process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
50 const failingSeed = runTests({
51 SEED: '0',
52 LOAD_PLAN: tempPlanPath,
53 SAVE_PLAN: tempPlanPath,
54 ITERATIONS: '500'
55 })
56
57 // If the test failed, keep the test plan with the removed row. Reload the test
58 // plan from the JSON file, since the test itself will remove any operations
59 // which are no longer valid before saving the test plan.
60 if (failingSeed != null) {
61 process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
62 fs.copyFileSync(tempPlanPath, outputPlanPath)
63 testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
64 ix--
65 } else {
66 process.stderr.write(` - keep.\n`)
67 }
68 }
69
70 fs.unlinkSync(tempPlanPath)
71
72 // Re-run the final minimized plan to get the correct failing seed.
73 // This is a workaround for the fact that the execution order can
74 // slightly change when replaying a test plan after it has been
75 // saved and loaded.
76 const failingSeed = runTests({
77 SEED: '0',
78 ITERATIONS: '5000',
79 LOAD_PLAN: outputPlanPath,
80 })
81
82 process.stderr.write(`final test plan: ${outputPlanPath}\n`)
83 process.stderr.write(`final seed: ${failingSeed}\n`)
84 return failingSeed
85}
86
87function buildTests() {
88 const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
89 stdio: 'inherit',
90 encoding: 'utf8',
91 env: {
92 ...process.env,
93 }
94 });
95 if (status !== 0) {
96 throw new Error('build failed')
97 }
98}
99
100function runTests(env) {
101 const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS, 'random_project_collaboration'], {
102 stdio: 'pipe',
103 encoding: 'utf8',
104 env: {
105 ...process.env,
106 ...env,
107 }
108 });
109
110 if (status !== 0) {
111 FAILING_SEED_REGEX.lastIndex = 0
112 const match = FAILING_SEED_REGEX.exec(stdout)
113 if (!match) {
114 process.stderr.write("test failed, but no failing seed found:\n")
115 process.stderr.write(stdout)
116 process.stderr.write('\n')
117 process.exit(1)
118 }
119 return match[1]
120 } else {
121 return null
122 }
123}
124
125function serializeTestPlan(plan) {
126 return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
127}
128
129exports.buildTests = buildTests
130exports.runTests = runTests
131exports.minimizeTestPlan = minimizeTestPlan