Line data Source code
1 : //! Reading the content of `@vis` tags: the joined text of a tag and a tolerant JSON scanner.
2 : //!
3 : //! The content lines of a tag are joined with line breaks into one text, offsets map back to the script.
4 : //! The JSON of `vegalite` tags is scanned token by token, also when it is incomplete or malformed, and
5 : //! every scalar knows its slot: a key, the value of a key or an element of an array. Highlighting,
6 : //! completion and the warnings of the analyzer find their way through the JSON with this scanner,
7 : //! `serde_json` builds the values.
8 :
9 : use dashql_parser::span::TextSpan;
10 :
11 : /// The content of a tag: its lines joined with line breaks, mapped back to the script text
12 : pub(crate) struct TagText {
13 : /// The joined content lines
14 : pub text: String,
15 : /// The script offset of every content line and its offset in the joined text
16 : lines: Vec<(usize, usize)>,
17 : /// The whole tag block, the location of spans in a tag without content
18 : block: TextSpan,
19 : }
20 :
21 : impl TagText {
22 387 : pub fn new(script: &str, lines: &[TextSpan], block: TextSpan) -> Self {
23 387 : let mut text = String::new();
24 387 : let mut offsets = Vec::with_capacity(lines.len());
25 601 : for (i, line) in lines.iter().enumerate() {
26 601 : if i > 0 {
27 216 : text.push('\n');
28 526 : }
29 601 : offsets.push((line.offset as usize, text.len()));
30 601 : text.push_str(&script[line.offset as usize..line.end() as usize]);
31 : }
32 387 : Self {
33 387 : text,
34 387 : lines: offsets,
35 387 : block,
36 387 : }
37 387 : }
38 :
39 : /// Does the tag have content lines?
40 21 : pub fn is_empty(&self) -> bool {
41 21 : self.lines.is_empty()
42 21 : }
43 :
44 : /// The script span of a range of the joined text. A range across lines includes the comment markers
45 : /// and gutters between them.
46 517 : pub fn span(&self, begin: usize, end: usize) -> TextSpan {
47 517 : if self.lines.is_empty() {
48 0 : return self.block;
49 517 : }
50 517 : let (begin, end) = (self.to_script(begin), self.to_script(end));
51 517 : TextSpan::new(begin as u32, (end - begin) as u32)
52 517 : }
53 :
54 : /// The joined offset of a script offset. An offset before the content of a line, e.g. in its
55 : /// indentation, maps to the begin of the line.
56 23 : pub fn to_joined(&self, offset: usize) -> usize {
57 23 : let mut joined = 0;
58 27 : for (i, &(script, at)) in self.lines.iter().enumerate() {
59 27 : let len = self.line_len(i);
60 27 : if offset < script {
61 1 : return at;
62 26 : }
63 26 : if offset <= script + len {
64 22 : return at + offset - script;
65 4 : }
66 4 : joined = at + len;
67 : }
68 0 : joined
69 23 : }
70 :
71 : /// The script offset of a joined offset, the tag must have content lines
72 1034 : fn to_script(&self, joined: usize) -> usize {
73 1312 : let line = self.lines.partition_point(|&(_, at)| at <= joined).saturating_sub(1);
74 1034 : let (script, at) = self.lines[line];
75 1034 : script + joined - at
76 1034 : }
77 :
78 27 : fn line_len(&self, i: usize) -> usize {
79 27 : let end = self.lines.get(i + 1).map_or(self.text.len(), |&(_, at)| at - 1);
80 27 : end - self.lines[i].1
81 27 : }
82 : }
83 :
84 : /// The kind of a JSON token
85 : #[derive(Clone, Copy, Debug, PartialEq, Eq)]
86 : pub(crate) enum JsonTokenKind {
87 : ObjectBegin,
88 : ObjectEnd,
89 : ArrayBegin,
90 : ArrayEnd,
91 : Colon,
92 : Comma,
93 : /// A string, not closed if the text or the line ends before the closing quote
94 : String { closed: bool },
95 : /// A number, e.g. `-1.5e3`
96 : Number,
97 : /// `true`, `false` or `null`
98 : Literal,
99 : /// Any other text up to the next delimiter, e.g. a word that is still typed
100 : Word,
101 : }
102 :
103 : /// The slot of a scalar in its container
104 : #[derive(Clone, Debug, PartialEq, Eq)]
105 : pub(crate) enum JsonSlot {
106 : /// A key of an object
107 : Key,
108 : /// The value of a key
109 : Value(String),
110 : /// An element of an array
111 : Element,
112 : }
113 :
114 : /// An element of the path to a token
115 : #[derive(Clone, Debug, PartialEq, Eq)]
116 : pub(crate) enum PathItem {
117 : Key(String),
118 : Index,
119 : }
120 :
121 : /// A token of JSON text
122 : #[derive(Clone, Debug, PartialEq, Eq)]
123 : pub(crate) struct JsonToken {
124 : pub kind: JsonTokenKind,
125 : /// The byte range of the token
126 : pub begin: usize,
127 : pub end: usize,
128 : /// The number of containers around the token, the brackets of a container are outside of it
129 : pub depth: usize,
130 : /// The slot of a scalar (string, number, literal or word), None for brackets and punctuation and
131 : /// for a scalar where no value fits, e.g. a second value after a key
132 : pub slot: Option<JsonSlot>,
133 : }
134 :
135 : impl JsonToken {
136 : /// The value of a closed string
137 139 : pub fn string(&self, text: &[u8]) -> Option<String> {
138 139 : match self.kind {
139 131 : JsonTokenKind::String { closed: true } => serde_json::from_slice(&text[self.begin..self.end]).ok(),
140 8 : _ => None,
141 : }
142 139 : }
143 : }
144 :
145 : /// The state of an open container
146 : enum Frame {
147 : Object { key: Option<String>, state: ObjectState },
148 : Array { expects_value: bool },
149 : }
150 :
151 : #[derive(Clone, Copy, PartialEq, Eq)]
152 : enum ObjectState {
153 : ExpectKey,
154 : ExpectColon,
155 : ExpectValue,
156 : AfterValue,
157 : }
158 :
159 : /// The bytes that end a number, a literal or a word
160 : const DELIMITERS: &[u8] = b"{}[],:\" \t\r\n";
161 :
162 : /// Scans JSON text into tokens and tracks the containers. Nothing is an error: unbalanced brackets,
163 : /// missing commas and unclosed strings are scanned as they come.
164 : pub(crate) struct JsonScanner<'a> {
165 : text: &'a [u8],
166 : at: usize,
167 : stack: Vec<Frame>,
168 : }
169 :
170 : impl<'a> JsonScanner<'a> {
171 43 : pub fn new(text: &'a [u8]) -> Self {
172 43 : Self {
173 43 : text,
174 43 : at: 0,
175 43 : stack: Vec::new(),
176 43 : }
177 43 : }
178 :
179 : /// The slot of the next scalar, None outside of containers or where no value fits
180 197 : pub fn slot(&self) -> Option<JsonSlot> {
181 197 : match self.stack.last()? {
182 131 : Frame::Object { state: ObjectState::ExpectKey, .. } => Some(JsonSlot::Key),
183 55 : Frame::Object { state: ObjectState::ExpectValue, key } => Some(JsonSlot::Value(key.clone().unwrap_or_default())),
184 7 : Frame::Array { expects_value: true } => Some(JsonSlot::Element),
185 3 : _ => None,
186 : }
187 197 : }
188 :
189 : /// The path to the innermost open container
190 27 : pub fn path(&self) -> Vec<PathItem> {
191 27 : let outer = self.stack.split_last().map_or(&[][..], |(_, outer)| outer);
192 27 : outer
193 27 : .iter()
194 44 : .map(|frame| match frame {
195 39 : Frame::Object { key, .. } => PathItem::Key(key.clone().unwrap_or_default()),
196 5 : Frame::Array { .. } => PathItem::Index,
197 44 : })
198 27 : .collect()
199 27 : }
200 :
201 : /// A value ended, the container moves on
202 81 : fn value_done(&mut self) {
203 81 : match self.stack.last_mut() {
204 65 : Some(Frame::Object { state, .. }) => *state = ObjectState::AfterValue,
205 10 : Some(Frame::Array { expects_value }) => *expects_value = false,
206 6 : None => {}
207 : }
208 81 : }
209 :
210 : /// A scalar ended: a key or a value
211 190 : fn scalar_done(&mut self, token: &JsonToken) {
212 190 : let text = self.text;
213 190 : match self.stack.last_mut() {
214 183 : Some(Frame::Object { key, state }) if *state == ObjectState::ExpectKey => {
215 128 : *key = Some(match token.kind {
216 126 : JsonTokenKind::String { .. } => token.string(text).unwrap_or_default(),
217 2 : _ => String::from_utf8_lossy(&text[token.begin..token.end]).into_owned(),
218 : });
219 128 : *state = ObjectState::ExpectColon;
220 : }
221 62 : _ => self.value_done(),
222 : }
223 190 : }
224 : }
225 :
226 : impl Iterator for JsonScanner<'_> {
227 : type Item = JsonToken;
228 :
229 484 : fn next(&mut self) -> Option<JsonToken> {
230 484 : let text = self.text;
231 692 : while self.at < text.len() && text[self.at].is_ascii_whitespace() {
232 208 : self.at += 1;
233 208 : }
234 484 : let begin = self.at;
235 484 : let c = *text.get(begin)?;
236 470 : let token = |kind, end, depth, slot| JsonToken { kind, begin, end, depth, slot };
237 470 : let depth = self.stack.len();
238 470 : match c {
239 : b'{' | b'[' => {
240 108 : self.at += 1;
241 108 : let (kind, frame) = if c == b'{' {
242 96 : (JsonTokenKind::ObjectBegin, Frame::Object { key: None, state: ObjectState::ExpectKey })
243 : } else {
244 12 : (JsonTokenKind::ArrayBegin, Frame::Array { expects_value: true })
245 : };
246 108 : self.stack.push(frame);
247 108 : Some(token(kind, self.at, depth, None))
248 : }
249 : b'}' | b']' => {
250 19 : self.at += 1;
251 19 : self.stack.pop();
252 19 : self.value_done();
253 19 : let kind = if c == b'}' { JsonTokenKind::ObjectEnd } else { JsonTokenKind::ArrayEnd };
254 19 : Some(token(kind, self.at, self.stack.len(), None))
255 : }
256 : b',' => {
257 39 : self.at += 1;
258 39 : match self.stack.last_mut() {
259 35 : Some(Frame::Object { key, state }) => {
260 35 : *key = None;
261 35 : *state = ObjectState::ExpectKey;
262 35 : }
263 4 : Some(Frame::Array { expects_value }) => *expects_value = true,
264 0 : None => {}
265 : }
266 39 : Some(token(JsonTokenKind::Comma, self.at, depth, None))
267 : }
268 : b':' => {
269 114 : self.at += 1;
270 114 : if let Some(Frame::Object { state, .. }) = self.stack.last_mut() {
271 114 : *state = ObjectState::ExpectValue;
272 114 : }
273 114 : Some(token(JsonTokenKind::Colon, self.at, depth, None))
274 : }
275 : b'"' => {
276 : // JSON strings contain no line breaks, an unclosed string ends with its line
277 182 : let mut i = begin + 1;
278 182 : let mut closed = false;
279 906 : while i < text.len() && text[i] != b'\n' {
280 887 : match text[i] {
281 : b'"' => {
282 163 : closed = true;
283 163 : i += 1;
284 163 : break;
285 : }
286 1 : b'\\' if text.get(i + 1).is_some_and(|&c| c != b'\n') => i += 2,
287 723 : _ => i += 1,
288 : }
289 : }
290 182 : self.at = i;
291 182 : let string = token(JsonTokenKind::String { closed }, i, depth, self.slot());
292 182 : self.scalar_done(&string);
293 182 : Some(string)
294 : }
295 : _ => {
296 8 : let mut i = begin;
297 34 : while i < text.len() && !DELIMITERS.contains(&text[i]) {
298 26 : i += 1;
299 26 : }
300 8 : self.at = i;
301 8 : let word = &text[begin..i];
302 8 : let kind = if matches!(word, b"true" | b"false" | b"null") {
303 3 : JsonTokenKind::Literal
304 5 : } else if (c == b'-' || c.is_ascii_digit())
305 9 : && word.iter().all(|&c| c.is_ascii_digit() || matches!(c, b'.' | b'e' | b'E' | b'+' | b'-'))
306 : {
307 2 : JsonTokenKind::Number
308 : } else {
309 3 : JsonTokenKind::Word
310 : };
311 8 : let scalar = token(kind, i, depth, self.slot());
312 8 : self.scalar_done(&scalar);
313 8 : Some(scalar)
314 : }
315 : }
316 484 : }
317 : }
318 :
319 : #[cfg(test)]
320 : mod tests {
321 : use super::*;
322 :
323 3 : fn tokens(json: &str) -> Vec<(&str, JsonTokenKind, usize, Option<JsonSlot>)> {
324 26 : JsonScanner::new(json.as_bytes()).map(|t| (&json[t.begin..t.end], t.kind, t.depth, t.slot)).collect()
325 3 : }
326 :
327 : #[test]
328 1 : fn scans_tokens_with_their_slots() {
329 : use JsonSlot::*;
330 : use JsonTokenKind::*;
331 2 : let value = |key: &str| Some(Value(key.to_string()));
332 1 : assert_eq!(
333 1 : tokens("{\"mark\": \"bar\", \"x\": [1.5, true], \"y\": null}"),
334 1 : vec![
335 1 : ("{", ObjectBegin, 0, None),
336 1 : ("\"mark\"", String { closed: true }, 1, Some(Key)),
337 1 : (":", Colon, 1, None),
338 1 : ("\"bar\"", String { closed: true }, 1, value("mark")),
339 1 : (",", Comma, 1, None),
340 1 : ("\"x\"", String { closed: true }, 1, Some(Key)),
341 1 : (":", Colon, 1, None),
342 1 : ("[", ArrayBegin, 1, None),
343 1 : ("1.5", Number, 2, Some(Element)),
344 1 : (",", Comma, 2, None),
345 1 : ("true", Literal, 2, Some(Element)),
346 1 : ("]", ArrayEnd, 1, None),
347 1 : (",", Comma, 1, None),
348 1 : ("\"y\"", String { closed: true }, 1, Some(Key)),
349 1 : (":", Colon, 1, None),
350 1 : ("null", Literal, 1, value("y")),
351 1 : ("}", ObjectEnd, 0, None),
352 : ]
353 : );
354 1 : }
355 :
356 : #[test]
357 1 : fn scans_malformed_json() {
358 : use JsonSlot::*;
359 : use JsonTokenKind::*;
360 : // An unclosed string ends with its line, an escaped quote does not close a string. Without a
361 : // comma, the next value still belongs to the key before.
362 1 : assert_eq!(
363 1 : tokens("{\"a\": \"b\n\"c\\\"\": 12px"),
364 1 : vec![
365 1 : ("{", ObjectBegin, 0, None),
366 1 : ("\"a\"", String { closed: true }, 1, Some(Key)),
367 1 : (":", Colon, 1, None),
368 1 : ("\"b", String { closed: false }, 1, Some(Value("a".into()))),
369 1 : ("\"c\\\"\"", String { closed: true }, 1, None),
370 1 : (":", Colon, 1, None),
371 1 : ("12px", Word, 1, Some(Value("a".into()))),
372 : ]
373 : );
374 : // Unbalanced brackets
375 1 : assert_eq!(tokens("}]"), vec![("}", ObjectEnd, 0, None), ("]", ArrayEnd, 0, None)]);
376 1 : }
377 :
378 : #[test]
379 1 : fn tracks_the_path() {
380 1 : let mut scanner = JsonScanner::new(b"{\"layer\": [{\"encoding\": {\"x\": {");
381 1 : scanner.by_ref().for_each(drop);
382 3 : let key = |k: &str| PathItem::Key(k.to_string());
383 1 : assert_eq!(scanner.path(), vec![key("layer"), PathItem::Index, key("encoding"), key("x")]);
384 1 : assert_eq!(scanner.slot(), Some(JsonSlot::Key));
385 1 : }
386 :
387 : #[test]
388 1 : fn maps_the_tag_text_to_the_script() {
389 1 : let script = "-- @vis vegalite {\n-- \"mark\": 1\n-- }";
390 3 : let line = |text: &str| TextSpan::new(script.find(text).unwrap() as u32, text.len() as u32);
391 1 : let tag = TagText::new(script, &[line("vegalite {"), line("\"mark\": 1"), line("}")], TextSpan::new(0, 0));
392 1 : assert_eq!(tag.text, "vegalite {\n\"mark\": 1\n}");
393 2 : let span = |begin: usize, end: usize| {
394 2 : let span = tag.span(begin, end);
395 2 : &script[span.offset as usize..span.end() as usize]
396 2 : };
397 1 : assert_eq!(span(11, 17), "\"mark\"");
398 1 : assert_eq!(span(9, tag.text.len()), "{\n-- \"mark\": 1\n-- }");
399 1 : assert_eq!(tag.to_joined(script.find("\"mark\"").unwrap()), 11);
400 1 : assert_eq!(tag.to_joined(script.find("-- ").unwrap()), 11);
401 1 : }
402 : }
|
