1import birdie
2import gleam/int
3import gleam/list
4import gleam/string
5import gleeunit
6import nibble.{Expected}
7import nibble/lexer.{Span, Token}
8import node
9import parser
10import quasi_lexer
11
12pub fn main() -> Nil {
13 gleeunit.main()
14}
15
16pub fn simple_quasi_lexer_test() {
17 quasi_lexer.chars()
18 |> quasi_lexer.run(on: "let x1 = e1")
19 |> list.index_map(fn(token, index) {
20 let Token(span, lexeme, value) = token
21 assert lexeme == value
22
23 let Span(row_start, col_start, row_end, col_end) = span
24 assert row_start == row_end && row_start == 1
25 assert col_start == index + 1
26 assert col_end == col_start + 1
27 })
28}
29
30pub fn quasi_lexer_off_by_one_test() {
31 let input = "let x1 =\n e1"
32 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
33
34 let snap =
35 tokens
36 |> list.index_map(fn(token, index) {
37 let Token(Span(rs, cs, re, ce), lexeme, _) = token
38 "Token "
39 <> int.to_string(index)
40 <> ": '"
41 <> lexeme
42 <> "' at Span(row_start: "
43 <> int.to_string(rs)
44 <> ", col_start: "
45 <> int.to_string(cs)
46 <> ", row_end: "
47 <> int.to_string(re)
48 <> ", col_end: "
49 <> int.to_string(ce)
50 <> ")\n"
51 })
52 |> list.fold("", fn(acc, line) { acc <> line })
53
54 birdie.snap(snap, title: "Quasi lexer spans with multiline input")
55}
56
57pub fn parse_let_successfully_test() {
58 let input = "let"
59 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
60 let parser = parser.exact_string("let", node.Let)
61 let assert Ok(_) = nibble.run(tokens, parser)
62}
63
64pub fn parse_let_failing_test() {
65 let input = "lt"
66 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
67 let parser = parser.exact_string("let", node.Let)
68 let assert Error(error) = nibble.run(tokens, parser)
69 let assert [nibble.DeadEnd(Span(_, cs, _, _), Expected(msg, got: got), [])] =
70 error
71
72 let snap =
73 "Msg: "
74 <> msg
75 <> "\n"
76 <> "Got: "
77 <> got
78 <> "\n"
79 <> "At column: "
80 <> int.to_string(cs)
81 <> "\n"
82
83 birdie.snap(snap, title: "Should fail to parse 'lt' as node.Let")
84}
85
86pub fn parse_unix_line_ending_test() {
87 let input = "\n"
88 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
89 let parser = parser.end_of_line()
90 let assert Ok(node.EndOfLine) = nibble.run(tokens, parser)
91}
92
93pub fn parse_windows_line_ending_test() {
94 let input = "\r\n"
95 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
96 let parser = parser.end_of_line()
97 let assert Ok(node.EndOfLine) = nibble.run(tokens, parser)
98}
99
100pub fn parse_line_ending_fails_on_lone_carriage_return_test() {
101 let input = "\r"
102 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
103 let parser = parser.end_of_line()
104 let assert Error(error) = nibble.run(tokens, parser)
105
106 let snap =
107 "Input: \\r (lone carriage return)\n"
108 <> "Number of errors: "
109 <> int.to_string(list.length(error))
110 <> "\n"
111 <> {
112 error
113 |> list.index_map(fn(dead_end, idx) {
114 let nibble.DeadEnd(Span(rs, cs, re, ce), reason, context) = dead_end
115 "Error "
116 <> int.to_string(idx + 1)
117 <> ":\n"
118 <> " Reason: "
119 <> string.inspect(reason)
120 <> "\n"
121 <> " Span: (row_start: "
122 <> int.to_string(rs)
123 <> ", col_start: "
124 <> int.to_string(cs)
125 <> ", row_end: "
126 <> int.to_string(re)
127 <> ", col_end: "
128 <> int.to_string(ce)
129 <> ")\n"
130 <> " Context: "
131 <> string.inspect(context)
132 <> "\n"
133 })
134 |> string.join("")
135 }
136
137 birdie.snap(snap, title: "Line ending should reject lone carriage return")
138}
139
140pub fn parse_line_ending_fails_on_other_chars_test() {
141 let input = "x"
142 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
143 let parser = parser.end_of_line()
144 let assert Error(_) = nibble.run(tokens, parser)
145}
146
147pub fn parse_line_ending_after_content_test() {
148 let input = "let\n"
149 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
150 let parser = {
151 use _ <- nibble.do(parser.let_keyword())
152 parser.end_of_line()
153 }
154 let assert Ok(node.EndOfLine) = nibble.run(tokens, parser)
155}
156
157pub fn parse_multiple_line_endings_test() {
158 let input = "\n\r\n\n"
159 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
160 let parser = {
161 use eol1 <- nibble.do(parser.end_of_line())
162 use eol2 <- nibble.do(parser.end_of_line())
163 use eol3 <- nibble.do(parser.end_of_line())
164 nibble.return([eol1, eol2, eol3])
165 }
166 let assert Ok(nodes) = nibble.run(tokens, parser)
167
168 let snap =
169 "Input: \\n\\r\\n\\n (Unix, Windows, Unix line endings)\n"
170 <> "Parsed "
171 <> int.to_string(list.length(nodes))
172 <> " line endings:\n"
173 <> {
174 nodes
175 |> list.index_map(fn(n, idx) {
176 " " <> int.to_string(idx + 1) <> ". " <> string.inspect(n) <> "\n"
177 })
178 |> string.join("")
179 }
180
181 birdie.snap(
182 snap,
183 title: "Multiple line endings should all parse as EndOfLine",
184 )
185}
186
187// Tests for printable parser (%x20-7F)
188
189pub fn parse_printable_space_test() {
190 let input = " "
191 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
192 let parser = parser.printable()
193 let assert Ok(node.Printable(" ")) = nibble.run(tokens, parser)
194}
195
196pub fn parse_printable_ascii_test() {
197 let input = "a"
198 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
199 let parser = parser.printable()
200 let assert Ok(node.Printable("a")) = nibble.run(tokens, parser)
201}
202
203pub fn parse_printable_tilde_test() {
204 let input = "~"
205 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
206 let parser = parser.printable()
207 let assert Ok(node.Printable("~")) = nibble.run(tokens, parser)
208}
209
210pub fn parse_printable_rejects_tab_test() {
211 let input = "\t"
212 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
213 let parser = parser.printable()
214 let assert Error(_) = nibble.run(tokens, parser)
215}
216
217pub fn parse_printable_rejects_newline_test() {
218 let input = "\n"
219 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
220 let parser = parser.printable()
221 let assert Error(_) = nibble.run(tokens, parser)
222}
223
224pub fn parse_printable_rejects_non_ascii_test() {
225 let input = "é"
226 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
227 let parser = parser.printable()
228 let assert Error(_) = nibble.run(tokens, parser)
229}
230
231// Tests for valid-non-ascii parser
232
233pub fn parse_valid_non_ascii_latin_test() {
234 let input = "é"
235 // é is U+00E9, in range 0x80-0xD7FF
236 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
237 let parser = parser.valid_non_ascii()
238 let assert Ok(node.ValidNonAscii("é")) = nibble.run(tokens, parser)
239}
240
241pub fn parse_valid_non_ascii_emoji_test() {
242 let input = "🎉"
243 // 🎉 is U+1F389, in range 0x10000-0x1FFFD
244 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
245 let parser = parser.valid_non_ascii()
246 let assert Ok(node.ValidNonAscii("🎉")) = nibble.run(tokens, parser)
247}
248
249pub fn parse_valid_non_ascii_chinese_test() {
250 let input = "中"
251 // 中 is U+4E2D, in range 0xE000-0xFFFD
252 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
253 let parser = parser.valid_non_ascii()
254 let assert Ok(node.ValidNonAscii("中")) = nibble.run(tokens, parser)
255}
256
257pub fn parse_valid_non_ascii_rejects_ascii_test() {
258 let input = "a"
259 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
260 let parser = parser.valid_non_ascii()
261 let assert Error(_) = nibble.run(tokens, parser)
262}
263
264pub fn parse_valid_non_ascii_rejects_tab_test() {
265 let input = "\t"
266 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
267 let parser = parser.valid_non_ascii()
268 let assert Error(_) = nibble.run(tokens, parser)
269}
270
271// Tests for not-end-of-line parser
272
273pub fn parse_not_end_of_line_printable_test() {
274 let input = "a"
275 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
276 let parser = parser.not_end_of_line()
277 let assert Ok(node.NotEndOfLine(node.Printable("a"))) =
278 nibble.run(tokens, parser)
279}
280
281pub fn parse_not_end_of_line_space_test() {
282 let input = " "
283 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
284 let parser = parser.not_end_of_line()
285 let assert Ok(node.NotEndOfLine(node.Printable(" "))) =
286 nibble.run(tokens, parser)
287}
288
289pub fn parse_not_end_of_line_tab_test() {
290 let input = "\t"
291 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
292 let parser = parser.not_end_of_line()
293 let assert Ok(node.NotEndOfLine(node.Tab)) = nibble.run(tokens, parser)
294}
295
296pub fn parse_not_end_of_line_valid_non_ascii_test() {
297 let input = "λ"
298 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
299 let parser = parser.not_end_of_line()
300 let assert Ok(node.NotEndOfLine(node.ValidNonAscii("λ"))) =
301 nibble.run(tokens, parser)
302}
303
304pub fn parse_not_end_of_line_rejects_newline_test() {
305 let input = "\n"
306 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
307 let parser = parser.not_end_of_line()
308 let assert Error(_) = nibble.run(tokens, parser)
309}
310
311pub fn parse_not_end_of_line_rejects_carriage_return_test() {
312 let input = "\r"
313 let tokens = quasi_lexer.chars() |> quasi_lexer.run(on: input)
314 let parser = parser.not_end_of_line()
315 let assert Error(_) = nibble.run(tokens, parser)
316}