1use std::sync::Arc;
2
3use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
4use edit_prediction::metrics::count_tree_sitter_errors;
5use fs::FakeFs;
6use gpui::{AppContext as _, TestAppContext};
7use language::{Buffer, BufferSnapshot, LanguageRegistry};
8use languages::init as init_languages;
9use node_runtime::NodeRuntime;
10use settings::SettingsStore;
11
12struct ParsedCase {
13 label: String,
14 bytes: usize,
15 error_count: usize,
16 snapshot: BufferSnapshot,
17}
18
19fn replace_nth_occurrences(
20 source: &mut String,
21 needle: &str,
22 replacement: &str,
23 every: usize,
24 max_replacements: usize,
25) {
26 let mut rebuilt = String::with_capacity(source.len());
27 let mut cursor = 0;
28 let mut seen = 0;
29 let mut replaced = 0;
30
31 while let Some(relative_index) = source[cursor..].find(needle) {
32 let start = cursor + relative_index;
33 let end = start + needle.len();
34 rebuilt.push_str(&source[cursor..start]);
35
36 if seen % every == 0 && replaced < max_replacements {
37 rebuilt.push_str(replacement);
38 replaced += 1;
39 } else {
40 rebuilt.push_str(needle);
41 }
42
43 seen += 1;
44 cursor = end;
45 }
46
47 rebuilt.push_str(&source[cursor..]);
48 *source = rebuilt;
49}
50
51fn rust_source(function_count: usize) -> String {
52 let mut source = String::from(
53 "pub struct Counter {\n value: usize,\n}\n\nimpl Counter {\n pub fn new() -> Self {\n Self { value: 0 }\n }\n}\n\n",
54 );
55 for index in 0..function_count {
56 source.push_str(&format!(
57 "pub fn compute_value_{index}(input: usize) -> usize {{\n let mut total = input;\n for offset in 0..32 {{\n total += offset + {index};\n }}\n if total % 2 == 0 {{\n total / 2\n }} else {{\n total * 3 + 1\n }}\n}}\n\n"
58 ));
59 }
60 source
61}
62
63fn rust_source_with_errors(function_count: usize) -> String {
64 let mut source = rust_source(function_count);
65 replace_nth_occurrences(
66 &mut source,
67 " if total % 2 == 0 {\n",
68 " if total % 2 == 0 \n",
69 17,
70 48,
71 );
72 source
73}
74
75fn python_source(function_count: usize) -> String {
76 let mut source = String::from(
77 "class Counter:\n def __init__(self) -> None:\n self.value = 0\n\n\n",
78 );
79 for index in 0..function_count {
80 source.push_str(&format!(
81 "def compute_value_{index}(input_value: int) -> int:\n total = input_value\n for offset in range(32):\n total += offset + {index}\n if total % 2 == 0:\n return total // 2\n return total * 3 + 1\n\n"
82 ));
83 }
84 source
85}
86
87fn python_source_with_errors(function_count: usize) -> String {
88 let mut source = python_source(function_count);
89 replace_nth_occurrences(
90 &mut source,
91 " if total % 2 == 0:\n",
92 " if total % 2 == 0\n",
93 19,
94 48,
95 );
96 source
97}
98
99fn go_source(function_count: usize) -> String {
100 let mut source = String::from(
101 "package bench\n\ntype Counter struct {\n\tvalue int\n}\n\nfunc NewCounter() Counter {\n\treturn Counter{value: 0}\n}\n\n",
102 );
103 for index in 0..function_count {
104 source.push_str(&format!(
105 "func ComputeValue{index}(inputValue int) int {{\n\ttotal := inputValue\n\tfor offset := 0; offset < 32; offset++ {{\n\t\ttotal += offset + {index}\n\t}}\n\tif total%2 == 0 {{\n\t\treturn total / 2\n\t}}\n\treturn total*3 + 1\n}}\n\n"
106 ));
107 }
108 source
109}
110
111fn go_source_with_errors(function_count: usize) -> String {
112 let mut source = go_source(function_count);
113 replace_nth_occurrences(
114 &mut source,
115 "\tfor offset := 0; offset < 32; offset++ {\n",
116 "\tfor offset := 0; offset < 32; offset++ \n",
117 17,
118 48,
119 );
120 source
121}
122
123fn typescript_source(function_count: usize) -> String {
124 let mut source = String::from(
125 "export type Counter = { value: number };\n\nexport function newCounter(): Counter {\n return { value: 0 };\n}\n\n",
126 );
127 for index in 0..function_count {
128 source.push_str(&format!(
129 "export function computeValue{index}(inputValue: number): number {{\n let total = inputValue;\n for (let offset = 0; offset < 32; offset += 1) {{\n total += offset + {index};\n }}\n return total % 2 === 0 ? total / 2 : total * 3 + 1;\n}}\n\n"
130 ));
131 }
132 source
133}
134
135fn typescript_source_with_errors(function_count: usize) -> String {
136 let mut source = typescript_source(function_count);
137 replace_nth_occurrences(
138 &mut source,
139 " return total % 2 === 0 ? total / 2 : total * 3 + 1;\n",
140 " return total % 2 === 0 ? total / 2 : ;\n",
141 17,
142 64,
143 );
144 source
145}
146
147fn tsx_source(component_count: usize) -> String {
148 let mut source = String::from(
149 "type ItemProps = { index: number; label: string };\n\nfunction Item({ index, label }: ItemProps) {\n return <li data-index={index}>{label}</li>;\n}\n\nexport function App() {\n return <section><ul>{[0, 1, 2].map((value) => <Item key={value} index={value} label={`item-${value}`} />)}</ul></section>;\n}\n\n",
150 );
151 for index in 0..component_count {
152 source.push_str(&format!(
153 "export function Widget{index}(): JSX.Element {{\n const items = Array.from({{ length: 16 }}, (_, value) => value + {index});\n return (\n <div className=\"widget-{index}\">\n <h2>Widget {index}</h2>\n <ul>\n {{items.map((value) => (\n <Item key={{value}} index={{value}} label={{`widget-{index}-${{value}}`}} />\n ))}}\n </ul>\n </div>\n );\n}}\n\n"
154 ));
155 }
156 source
157}
158
159fn tsx_source_with_errors(component_count: usize) -> String {
160 let mut source = tsx_source(component_count);
161 replace_nth_occurrences(
162 &mut source,
163 " const items = Array.from({ length: 16 }, (_, value) => value + ",
164 " const items = Array.from({ length: 16 }, (_, value) => ); // ",
165 11,
166 32,
167 );
168 source
169}
170
171fn json_source(object_count: usize) -> String {
172 let mut source = String::from("{\n \"items\": [\n");
173 for index in 0..object_count {
174 let suffix = if index + 1 == object_count { "" } else { "," };
175 source.push_str(&format!(
176 " {{\n \"id\": {index},\n \"name\": \"item-{index}\",\n \"enabled\": true,\n \"tags\": [\"alpha\", \"beta\", \"gamma\"],\n \"metrics\": {{ \"count\": {}, \"ratio\": {} }}\n }}{suffix}\n",
177 index * 3 + 1,
178 index as f64 / 10.0,
179 ));
180 }
181 source.push_str(" ]\n}\n");
182 source
183}
184
185fn json_source_with_errors(object_count: usize) -> String {
186 let mut source = json_source(object_count);
187 replace_nth_occurrences(
188 &mut source,
189 " \"enabled\": true,\n",
190 " \"enabled\": ,\n",
191 23,
192 64,
193 );
194 source
195}
196
197fn yaml_source(document_count: usize) -> String {
198 let mut source = String::new();
199 for index in 0..document_count {
200 source.push_str(&format!(
201 "- id: {index}\n name: item-{index}\n enabled: true\n tags:\n - alpha\n - beta\n - gamma\n metrics:\n count: {}\n ratio: {}\n",
202 index * 3 + 1,
203 index as f64 / 10.0,
204 ));
205 }
206 source
207}
208
209fn yaml_source_with_errors(document_count: usize) -> String {
210 let mut source = yaml_source(document_count);
211 replace_nth_occurrences(&mut source, " count: ", " count ", 23, 64);
212 source
213}
214
215fn css_source(rule_count: usize) -> String {
216 let mut source = String::new();
217 for index in 0..rule_count {
218 source.push_str(&format!(
219 ".widget-{index} {{\n display: grid;\n grid-template-columns: repeat(4, minmax(0, 1fr));\n gap: 12px;\n padding: 8px;\n color: rgb({}, {}, {});\n}}\n\n.widget-{index} > .item-{index} {{\n border: 1px solid rgba(0, 0, 0, 0.15);\n background: linear-gradient(90deg, #fff, #eef);\n}}\n\n",
220 (index * 17) % 255,
221 (index * 31) % 255,
222 (index * 47) % 255,
223 ));
224 }
225 source
226}
227
228fn css_source_with_errors(rule_count: usize) -> String {
229 let mut source = css_source(rule_count);
230 replace_nth_occurrences(&mut source, " gap: 12px;\n", " gap 12px;\n", 29, 64);
231 source
232}
233
234fn build_case(
235 context: &mut TestAppContext,
236 languages: &Arc<LanguageRegistry>,
237 language_name: &'static str,
238 variant_name: &'static str,
239 source: String,
240 expect_errors: bool,
241) -> ParsedCase {
242 let language_task = context.background_spawn({
243 let languages = languages.clone();
244 async move { languages.language_for_name(language_name).await }
245 });
246 while !language_task.is_ready() {
247 context.run_until_parked();
248 }
249 let language = futures::executor::block_on(language_task)
250 .unwrap_or_else(|error| panic!("failed to load {language_name}: {error}"));
251
252 let buffer = context.new(|cx| Buffer::local(source, cx).with_language(language, cx));
253 context.run_until_parked();
254 while buffer.read_with(context, |buffer, _| buffer.is_parsing()) {
255 context.run_until_parked();
256 }
257
258 let snapshot = buffer.read_with(context, |buffer, _| buffer.snapshot());
259 let full_range = 0..snapshot.text.len();
260 let error_count = count_tree_sitter_errors(snapshot.syntax_layers());
261 if expect_errors {
262 assert!(
263 error_count > 0,
264 "expected tree-sitter errors for {language_name}/{variant_name}",
265 );
266 } else {
267 assert_eq!(
268 error_count, 0,
269 "expected no tree-sitter errors for {language_name}/{variant_name}",
270 );
271 }
272
273 let label = format!(
274 "{}/{}_{}kb_{}e",
275 language_name.to_lowercase(),
276 variant_name,
277 full_range.end / 1024,
278 error_count,
279 );
280 ParsedCase {
281 label,
282 bytes: full_range.end,
283 error_count,
284 snapshot,
285 }
286}
287
288fn parsed_cases() -> Vec<ParsedCase> {
289 let mut context = TestAppContext::single();
290 context.update(|cx| {
291 let settings_store = SettingsStore::test(cx);
292 cx.set_global(settings_store);
293 });
294
295 let languages = Arc::new(LanguageRegistry::new(context.executor()));
296 let fs = FakeFs::new(context.executor());
297 let node_runtime = NodeRuntime::unavailable();
298 context.update(|cx| init_languages(languages.clone(), fs, node_runtime, cx));
299
300 vec![
301 build_case(
302 &mut context,
303 &languages,
304 "Rust",
305 "valid",
306 rust_source(900),
307 false,
308 ),
309 build_case(
310 &mut context,
311 &languages,
312 "Rust",
313 "error_heavy",
314 rust_source_with_errors(900),
315 true,
316 ),
317 build_case(
318 &mut context,
319 &languages,
320 "Python",
321 "valid",
322 python_source(1100),
323 false,
324 ),
325 build_case(
326 &mut context,
327 &languages,
328 "Python",
329 "error_heavy",
330 python_source_with_errors(1100),
331 true,
332 ),
333 build_case(
334 &mut context,
335 &languages,
336 "Go",
337 "valid",
338 go_source(1000),
339 false,
340 ),
341 build_case(
342 &mut context,
343 &languages,
344 "Go",
345 "error_heavy",
346 go_source_with_errors(1000),
347 true,
348 ),
349 build_case(
350 &mut context,
351 &languages,
352 "TypeScript",
353 "valid",
354 typescript_source(1000),
355 false,
356 ),
357 build_case(
358 &mut context,
359 &languages,
360 "TypeScript",
361 "error_heavy",
362 typescript_source_with_errors(1000),
363 true,
364 ),
365 build_case(
366 &mut context,
367 &languages,
368 "TSX",
369 "valid",
370 tsx_source(350),
371 false,
372 ),
373 build_case(
374 &mut context,
375 &languages,
376 "TSX",
377 "error_heavy",
378 tsx_source_with_errors(350),
379 true,
380 ),
381 build_case(
382 &mut context,
383 &languages,
384 "JSON",
385 "valid",
386 json_source(2200),
387 false,
388 ),
389 build_case(
390 &mut context,
391 &languages,
392 "JSON",
393 "error_heavy",
394 json_source_with_errors(2200),
395 true,
396 ),
397 build_case(
398 &mut context,
399 &languages,
400 "YAML",
401 "valid",
402 yaml_source(2200),
403 false,
404 ),
405 build_case(
406 &mut context,
407 &languages,
408 "YAML",
409 "error_heavy",
410 yaml_source_with_errors(2200),
411 true,
412 ),
413 build_case(
414 &mut context,
415 &languages,
416 "CSS",
417 "valid",
418 css_source(2400),
419 false,
420 ),
421 build_case(
422 &mut context,
423 &languages,
424 "CSS",
425 "error_heavy",
426 css_source_with_errors(2400),
427 true,
428 ),
429 ]
430}
431
432fn ts_error_count_benchmark(c: &mut Criterion) {
433 let cases = parsed_cases();
434 let mut group = c.benchmark_group("ts_error_count/full_file");
435
436 for case in &cases {
437 group.bench_with_input(
438 BenchmarkId::from_parameter(&case.label),
439 case,
440 |bench, case| {
441 bench.iter(|| {
442 black_box(case.bytes);
443 black_box(case.error_count);
444 black_box(count_tree_sitter_errors(case.snapshot.syntax_layers()))
445 });
446 },
447 );
448 }
449
450 group.finish();
451}
452
453criterion_group!(benches, ts_error_count_benchmark);
454criterion_main!(benches);