Line data Source code
1 : //! The syntax spans of comments: comment tags are highlighted like code, everything else is a comment.
2 : //!
3 : //! Every tag, `@vis`, `@board`, `@flow`, `@param` and the `@end` that older scripts close a region
4 : //! of line comments with, is a keyword, so the tags of an annotation read alike. The JSON of `@vis vegalite` is split
5 : //! into its tokens: keys are identifiers, strings, numbers and booleans are literals. In a `@vis` tag, the kind is a
6 : //! keyword and `{{field}}` placeholders of Markdown templates are identifiers. The keys of the args of `@board` and
7 : //! `@flow` (and of `@vis board` and `@vis flow` of older scripts) are identifiers. The rest of the comment stays a
8 : //! comment, e.g. the value of a `@param`.
9 : //!
10 : //! The JSON objects of `vegalite` tags are found with the tokens, editors fold them.
11 :
12 : use crate::vis_text::{JsonScanner, JsonSlot, JsonTokenKind, TagText};
13 : use dashql_buffers::parser::{CommentBlockType, ScannerTokenType};
14 : use dashql_parser::comment_blocks::{COMMENT_TAG_END, COMMENT_TAG_PREFIX};
15 : use dashql_parser::parser::ParsedScript;
16 : use dashql_parser::span::TextSpan;
17 :
18 : /// The tag of visualizations
19 : const VIS_TAG: &[u8] = b"vis";
20 : /// The tag of the dashboard placement
21 : const BOARD_TAG: &[u8] = b"board";
22 : /// The tag of the position on the dataflow canvas
23 : const FLOW_TAG: &[u8] = b"flow";
24 :
25 : /// The tokens of the content lines of a `board` or `markdown` tag
26 : struct Lines<'a> {
27 : script: &'a [u8],
28 : out: &'a mut Vec<(TextSpan, ScannerTokenType)>,
29 : }
30 :
31 : impl Lines<'_> {
32 61 : fn push(&mut self, begin: usize, end: usize, token: ScannerTokenType) {
33 61 : if end > begin {
34 61 : self.out.push((TextSpan::new(begin as u32, (end - begin) as u32), token));
35 61 : }
36 61 : }
37 :
38 : /// The `key=value` args of a board placement
39 7 : fn args(&mut self, begin: usize, end: usize) {
40 7 : let text = self.script;
41 7 : let mut i = begin;
42 20 : while i < end {
43 20 : while i < end && text[i].is_ascii_whitespace() {
44 7 : i += 1;
45 7 : }
46 13 : let arg_begin = i;
47 106 : while i < end && !text[i].is_ascii_whitespace() {
48 93 : i += 1;
49 93 : }
50 71 : if let Some(eq) = text[arg_begin..i].iter().position(|&c| c == b'=') {
51 13 : self.push(arg_begin, arg_begin + eq, ScannerTokenType::IDENTIFIER);
52 13 : self.push(arg_begin + eq + 1, i, ScannerTokenType::LITERAL_INTEGER);
53 13 : }
54 : }
55 7 : }
56 :
57 : /// The `{{field}}` placeholders of a Markdown template
58 2 : fn placeholders(&mut self, begin: usize, end: usize) {
59 2 : let text = self.script;
60 2 : let mut i = begin;
61 9 : while i + 1 < end {
62 7 : if &text[i..i + 2] != b"{{" {
63 5 : i += 1;
64 5 : continue;
65 2 : }
66 4 : match text[i + 2..end].windows(2).position(|w| w == b"}}") {
67 2 : Some(close) => {
68 2 : let close = i + 2 + close + 2;
69 2 : self.push(i, close, ScannerTokenType::IDENTIFIER);
70 2 : i = close;
71 2 : }
72 0 : None => break,
73 : }
74 : }
75 2 : }
76 : }
77 :
78 : /// The tokens of the JSON of a tag, from `body` in the tag text, and the spans of its outermost objects
79 4 : fn json_tokens(tag: &TagText, body: usize, out: &mut Vec<(TextSpan, ScannerTokenType)>, mut objects: Option<&mut Vec<TextSpan>>) {
80 4 : let json = &tag.text.as_bytes()[body..];
81 4 : let mut object_begin = None;
82 35 : for token in JsonScanner::new(json) {
83 35 : let (begin, end) = (body + token.begin, body + token.end);
84 15 : let token_type = match token.kind {
85 5 : JsonTokenKind::ObjectBegin if token.depth == 0 => {
86 4 : object_begin = Some(begin);
87 4 : continue;
88 : }
89 4 : JsonTokenKind::ObjectEnd if token.depth == 0 => {
90 3 : if let (Some(begin), Some(objects)) = (object_begin.take(), objects.as_deref_mut()) {
91 3 : objects.push(tag.span(begin, end));
92 3 : }
93 3 : continue;
94 : }
95 13 : JsonTokenKind::String { .. } if token.slot == Some(JsonSlot::Key) => ScannerTokenType::IDENTIFIER,
96 5 : JsonTokenKind::String { .. } => ScannerTokenType::LITERAL_STRING,
97 2 : JsonTokenKind::Number if json[token.begin..token.end].iter().any(|c| matches!(c, b'.' | b'e' | b'E')) => {
98 1 : ScannerTokenType::LITERAL_FLOAT
99 : }
100 0 : JsonTokenKind::Number => ScannerTokenType::LITERAL_INTEGER,
101 1 : JsonTokenKind::Literal if &json[token.begin..token.end] == b"null" => ScannerTokenType::KEYWORD,
102 1 : JsonTokenKind::Literal => ScannerTokenType::LITERAL_BOOLEAN,
103 13 : _ => continue,
104 : };
105 15 : out.push((tag.span(begin, end), token_type));
106 : }
107 4 : }
108 :
109 : /// The `@end` of a line comment that closes a tagged region, as script offsets
110 20 : fn end_tag(script: &[u8], comment: TextSpan) -> Option<(usize, usize)> {
111 20 : let (begin, end) = (comment.offset as usize, comment.end() as usize);
112 20 : let text = &script[begin..end];
113 34 : let dashes = text.iter().take_while(|&&c| c == b'-').count();
114 20 : if dashes < 2 {
115 13 : return None;
116 7 : }
117 7 : let trimmed = text[dashes..].trim_ascii();
118 7 : let end_tag = [COMMENT_TAG_PREFIX.as_bytes(), COMMENT_TAG_END.as_bytes()].concat();
119 7 : if trimmed != end_tag.as_slice() {
120 3 : return None;
121 4 : }
122 4 : let at = begin + dashes + (text[dashes..].len() - text[dashes..].trim_ascii_start().len());
123 4 : Some((at, at + end_tag.len()))
124 20 : }
125 :
126 : /// The highlighted tokens of the comment tags, sorted by offset, and the JSON objects of the `vegalite` tags
127 106 : fn tag_tokens(parsed: &ParsedScript) -> (Vec<(TextSpan, ScannerTokenType)>, Vec<TextSpan>) {
128 106 : let script = parsed.scanned.text.as_bytes();
129 106 : let blocks = &parsed.comment_blocks;
130 106 : let mut out = Vec::new();
131 106 : let mut objects = Vec::new();
132 109 : for block in &blocks.blocks {
133 32 : if block.block_type != CommentBlockType::DASHQL_TAG {
134 12 : continue;
135 20 : }
136 20 : let tag = block.tag_span;
137 20 : let name = &script[tag.offset as usize..tag.end() as usize];
138 20 : let mut lines = Lines { script, out: &mut out };
139 : // The tag with its `@`, and the end line of a region of line comments
140 20 : lines.push(tag.offset as usize - 1, tag.end() as usize, ScannerTokenType::KEYWORD);
141 20 : let last = parsed.scanned.comments[(block.comments_begin + block.comments_count - 1) as usize];
142 20 : if let Some((begin, end)) = end_tag(script, last) {
143 4 : lines.push(begin, end, ScannerTokenType::KEYWORD);
144 16 : }
145 20 : let content = &blocks.lines[block.lines_begin as usize..(block.lines_begin + block.lines_count) as usize];
146 : // `@board` and `@flow` are args
147 20 : if name == BOARD_TAG || name == FLOW_TAG {
148 6 : if let Some(first) = content.first() {
149 6 : lines.args(first.offset as usize, first.end() as usize);
150 6 : }
151 6 : continue;
152 14 : }
153 14 : if name != VIS_TAG {
154 5 : continue;
155 9 : }
156 9 : let Some(first) = content.first() else { continue };
157 9 : let first_begin = first.offset as usize;
158 9 : let mut kind_end = first_begin;
159 72 : while kind_end < first.end() as usize && !script[kind_end].is_ascii_whitespace() {
160 63 : kind_end += 1;
161 63 : }
162 9 : lines.push(first_begin, kind_end, ScannerTokenType::KEYWORD);
163 9 : let kind = &script[first_begin..kind_end];
164 9 : if kind == b"vegalite" {
165 4 : let text = TagText::new(&parsed.scanned.text, content, block.text_span);
166 4 : json_tokens(&text, kind_end - first_begin, &mut out, Some(&mut objects));
167 4 : continue;
168 5 : }
169 5 : for (i, line) in content.iter().enumerate() {
170 5 : let begin = if i == 0 { kind_end } else { line.offset as usize };
171 5 : let end = line.end() as usize;
172 0 : match kind {
173 5 : b"board" | b"card" | b"flow" if i == 0 => lines.args(begin, end),
174 2 : b"markdown" => lines.placeholders(begin, end),
175 2 : _ => {}
176 : }
177 : }
178 : }
179 106 : out.sort_by_key(|(span, _)| span.offset);
180 106 : objects.sort_by_key(|span| span.offset);
181 106 : (out, objects)
182 106 : }
183 :
184 : /// The syntax spans of the comments: a comment is one comment span, comment tags are split into their
185 : /// tokens and the comment spans between them. Returns the spans and the JSON objects of the `vegalite` tags.
186 106 : pub fn comment_syntax_spans(parsed: &ParsedScript) -> (Vec<(TextSpan, ScannerTokenType)>, Vec<TextSpan>) {
187 106 : let (tokens, objects) = tag_tokens(parsed);
188 106 : let mut out = Vec::with_capacity(parsed.scanned.comments.len() + tokens.len() * 2);
189 106 : let mut next = 0;
190 112 : for &comment in &parsed.scanned.comments {
191 37 : let mut at = comment.offset as usize;
192 37 : let end = comment.end() as usize;
193 37 : while next < tokens.len() && (tokens[next].0.offset as usize) < at {
194 0 : next += 1;
195 0 : }
196 113 : while next < tokens.len() && (tokens[next].0.end() as usize) <= end {
197 76 : let (span, token) = tokens[next];
198 76 : if span.offset as usize > at {
199 76 : out.push((TextSpan::new(at as u32, span.offset - at as u32), ScannerTokenType::COMMENT));
200 76 : }
201 76 : out.push((span, token));
202 76 : at = span.end() as usize;
203 76 : next += 1;
204 : }
205 37 : if end > at {
206 30 : out.push((TextSpan::new(at as u32, (end - at) as u32), ScannerTokenType::COMMENT));
207 30 : }
208 : }
209 106 : (out, objects)
210 106 : }
211 :
212 : #[cfg(test)]
213 : mod tests {
214 : use super::*;
215 : use dashql_parser::parser::parse;
216 : use dashql_parser::scanner::scan;
217 : use std::sync::Arc;
218 :
219 6 : fn spans(text: &str) -> Vec<(&str, ScannerTokenType)> {
220 6 : let parsed = parse(Arc::new(scan(text, 0, 0)));
221 6 : comment_syntax_spans(&parsed)
222 6 : .0
223 6 : .into_iter()
224 52 : .map(|(span, token)| (&text[span.offset as usize..span.end() as usize], token))
225 6 : .collect()
226 6 : }
227 :
228 : #[test]
229 1 : fn highlights_vis_tags() {
230 : use ScannerTokenType as T;
231 1 : assert_eq!(spans("/* plain @vis */ select 1"), vec![("/* plain @vis */", T::COMMENT)]);
232 1 : assert_eq!(
233 1 : spans("/* @vis vegalite {\"mark\": \"bar\", \"x\": 1.5, \"y\": true} */ select 1"),
234 1 : vec![
235 1 : ("/* ", T::COMMENT),
236 1 : ("@vis", T::KEYWORD),
237 1 : (" ", T::COMMENT),
238 1 : ("vegalite", T::KEYWORD),
239 1 : (" {", T::COMMENT),
240 1 : ("\"mark\"", T::IDENTIFIER),
241 1 : (": ", T::COMMENT),
242 1 : ("\"bar\"", T::LITERAL_STRING),
243 1 : (", ", T::COMMENT),
244 1 : ("\"x\"", T::IDENTIFIER),
245 1 : (": ", T::COMMENT),
246 1 : ("1.5", T::LITERAL_FLOAT),
247 1 : (", ", T::COMMENT),
248 1 : ("\"y\"", T::IDENTIFIER),
249 1 : (": ", T::COMMENT),
250 1 : ("true", T::LITERAL_BOOLEAN),
251 1 : ("} */", T::COMMENT),
252 : ]
253 : );
254 1 : assert_eq!(
255 1 : spans("-- @vis board width=6\n-- @vis markdown **{{a}}**\n-- @end\nselect 1"),
256 1 : vec![
257 1 : ("-- ", T::COMMENT),
258 1 : ("@vis", T::KEYWORD),
259 1 : (" ", T::COMMENT),
260 1 : ("board", T::KEYWORD),
261 1 : (" ", T::COMMENT),
262 1 : ("width", T::IDENTIFIER),
263 1 : ("=", T::COMMENT),
264 1 : ("6", T::LITERAL_INTEGER),
265 1 : ("-- ", T::COMMENT),
266 1 : ("@vis", T::KEYWORD),
267 1 : (" ", T::COMMENT),
268 1 : ("markdown", T::KEYWORD),
269 1 : (" **", T::COMMENT),
270 1 : ("{{a}}", T::IDENTIFIER),
271 1 : ("**", T::COMMENT),
272 1 : ("-- ", T::COMMENT),
273 1 : ("@end", T::KEYWORD),
274 : ]
275 : );
276 1 : }
277 :
278 : #[test]
279 1 : fn highlights_every_tag_alike() {
280 : use ScannerTokenType as T;
281 : // The content of a tag without tokens stays a comment, the keys of the args of a board are identifiers
282 1 : assert_eq!(
283 1 : spans("/* @param Sales.\n @board width=6\n*/\nselect 1"),
284 1 : vec![
285 1 : ("/* ", T::COMMENT),
286 1 : ("@param", T::KEYWORD),
287 1 : (" Sales.\n ", T::COMMENT),
288 1 : ("@board", T::KEYWORD),
289 1 : (" ", T::COMMENT),
290 1 : ("width", T::IDENTIFIER),
291 1 : ("=", T::COMMENT),
292 1 : ("6", T::LITERAL_INTEGER),
293 1 : ("\n*/", T::COMMENT),
294 : ]
295 : );
296 1 : assert_eq!(
297 1 : spans("-- @param\n-- Sales.\n-- @end\nselect 1"),
298 1 : vec![
299 1 : ("-- ", T::COMMENT),
300 1 : ("@param", T::KEYWORD),
301 1 : ("-- Sales.", T::COMMENT),
302 1 : ("-- ", T::COMMENT),
303 1 : ("@end", T::KEYWORD),
304 : ]
305 : );
306 : // A region of line comments ends with the run, the end line is not needed
307 1 : assert_eq!(
308 1 : spans("-- @param Sales.\nselect 1"),
309 1 : vec![("-- ", T::COMMENT), ("@param", T::KEYWORD), (" Sales.", T::COMMENT)]
310 : );
311 1 : }
312 :
313 4 : fn vegalite_specs(text: &str) -> Vec<&str> {
314 4 : let parsed = parse(Arc::new(scan(text, 0, 0)));
315 4 : comment_syntax_spans(&parsed)
316 4 : .1
317 4 : .into_iter()
318 4 : .map(|span| &text[span.offset as usize..span.end() as usize])
319 4 : .collect()
320 4 : }
321 :
322 : #[test]
323 1 : fn finds_vegalite_specs() {
324 1 : assert_eq!(
325 1 : vegalite_specs("/* @vis vegalite\n {\n \"mark\": {\"type\": \"bar\"},\n \"title\": \"a } b\"\n }\n*/\nselect 1"),
326 1 : vec!["{\n \"mark\": {\"type\": \"bar\"},\n \"title\": \"a } b\"\n }"]
327 : );
328 1 : assert_eq!(vegalite_specs("-- @vis vegalite {\"mark\":\"bar\"}\n-- @end\nselect 1"), vec!["{\"mark\":\"bar\"}"]);
329 : // Only the JSON of vegalite tags, and only closed objects
330 1 : assert_eq!(vegalite_specs("/* @vis markdown {{a}} */ select 1; /* {\"mark\": 1} */ select 2"), Vec::<&str>::new());
331 1 : assert_eq!(vegalite_specs("/* @vis vegalite {\"mark\": \"bar\" */ select 1"), Vec::<&str>::new());
332 1 : }
333 : }
|
