Line data Source code
1 : //! The Markdown renderer spec: a row template with `{{field}}` placeholders compiled into a safe, renderer-neutral
2 : //! document tree.
3 : //!
4 : //! The fields are replaced by unique tokens, the template is parsed as CommonMark with pulldown-cmark, and the tokens
5 : //! are turned into field nodes again. Raw HTML is dropped, links and images with unsafe URLs or with fields in their
6 : //! URLs become plain spans. Adjacent text is merged into one text node, soft line breaks are text (`\n`).
7 :
8 : use dashql_core::analyzer::types::VisualizationSpec;
9 : use dashql_buffers::visualization::{
10 : MarkdownNode as MarkdownNodeBuffer, MarkdownNodeArgs, MarkdownNodeType, MarkdownSpec, MarkdownSpecArgs,
11 : };
12 : use flatbuffers::{FlatBufferBuilder, WIPOffset};
13 : use pulldown_cmark::{CodeBlockKind, Event, LinkType, Parser, Tag, TagEnd};
14 :
15 : /// A node of a compiled document. Empty strings are not set.
16 : #[derive(Clone, Debug, PartialEq)]
17 : pub struct MarkdownNode {
18 : /// The node type
19 : pub node_type: MarkdownNodeType,
20 : /// The text of a text node or the field name of a field node
21 : pub value: String,
22 : /// The URL of a link or image
23 : pub url: String,
24 : /// The title of a link or image, empty if it contains a field
25 : pub title: String,
26 : /// The language of a fenced code block, the first word of its info string
27 : pub language: String,
28 : /// The level of a heading
29 : pub level: u8,
30 : /// The number of the first item of an ordered list
31 : pub start: u32,
32 : /// Is the list ordered?
33 : pub ordered: bool,
34 : /// Is the list tight, its items not wrapped in paragraphs?
35 : pub tight: bool,
36 : /// The child nodes
37 : pub children: Vec<MarkdownNode>,
38 : }
39 :
40 : impl MarkdownNode {
41 261 : pub fn new(node_type: MarkdownNodeType) -> Self {
42 261 : Self {
43 261 : node_type,
44 261 : value: String::new(),
45 261 : url: String::new(),
46 261 : title: String::new(),
47 261 : language: String::new(),
48 261 : level: 0,
49 261 : start: 0,
50 261 : ordered: false,
51 261 : tight: false,
52 261 : children: Vec::new(),
53 261 : }
54 261 : }
55 :
56 116 : fn leaf(node_type: MarkdownNodeType, value: &str) -> Self {
57 116 : Self { value: value.to_string(), ..Self::new(node_type) }
58 116 : }
59 : }
60 :
61 : /// A compiled Markdown spec
62 : #[derive(Clone, Debug, PartialEq)]
63 : pub struct MarkdownDocument {
64 : /// The row template
65 : pub template: String,
66 : /// The distinct field names in order of appearance
67 : pub fields: Vec<String>,
68 : /// The document tree, a DOCUMENT node
69 : pub document: MarkdownNode,
70 : }
71 :
72 : /// A field of the template and the token that stands for it in the parsed Markdown
73 : struct MarkdownField {
74 : /// The field name between the braces, trimmed
75 : name: String,
76 : /// The unique token that replaces the placeholder
77 : token: String,
78 : }
79 :
80 : /// Replace `{{field}}` placeholders with tokens that do not occur in the template
81 36 : fn tokenize_fields(row_template: &str) -> (String, Vec<MarkdownField>) {
82 36 : let mut prefix = String::from("DASHQLMARKDOWNFIELD");
83 37 : while row_template.contains(&prefix) {
84 1 : prefix.push('X');
85 1 : }
86 :
87 36 : let mut markdown = String::with_capacity(row_template.len());
88 36 : let mut fields: Vec<MarkdownField> = Vec::new();
89 36 : let mut rest = row_template;
90 73 : while let Some(open) = rest.find("{{") {
91 38 : let Some(close) = rest[open + 2..].find("}}").map(|close| open + 2 + close) else {
92 1 : break;
93 : };
94 37 : markdown.push_str(&rest[..open]);
95 37 : let name = rest[open + 2..close].trim_matches([' ', '\n']);
96 37 : if name.is_empty() {
97 1 : markdown.push_str(&rest[open..close + 2]);
98 1 : } else {
99 36 : let index = fields.iter().position(|field| field.name == name).unwrap_or_else(|| {
100 29 : let token = format!("{prefix}{}TOKEN", fields.len());
101 29 : fields.push(MarkdownField { name: name.to_string(), token });
102 29 : fields.len() - 1
103 29 : });
104 36 : markdown.push_str(&fields[index].token);
105 : }
106 37 : rest = &rest[close + 2..];
107 : }
108 36 : markdown.push_str(rest);
109 36 : (markdown, fields)
110 36 : }
111 :
112 : /// Relative URLs and the schemes http, https and mailto are safe
113 22 : fn is_safe_url(url: &str) -> bool {
114 308 : let normalized: String = url.chars().filter(|&c| c > ' ' && c != '\x7f').map(|c| c.to_ascii_lowercase()).collect();
115 22 : let Some(colon) = normalized.find(':') else {
116 5 : return true;
117 : };
118 17 : if normalized.find(['/', '?', '#']).is_some_and(|delimiter| delimiter < colon) {
119 2 : return true;
120 15 : }
121 15 : matches!(&normalized[..colon], "http" | "https" | "mailto")
122 22 : }
123 :
124 : /// Builds the document tree from the pulldown-cmark events
125 : struct DocumentBuilder<'a> {
126 : /// The fields of the template
127 : fields: &'a [MarkdownField],
128 : /// The open nodes, the root first. A node is added to its parent when it is closed.
129 : stack: Vec<MarkdownNode>,
130 : /// Text that is not yet added, so that adjacent text events become one node
131 : text: String,
132 : /// Inside a raw HTML block, which is dropped
133 : in_html_block: bool,
134 : }
135 :
136 : impl<'a> DocumentBuilder<'a> {
137 35 : fn new(fields: &'a [MarkdownField]) -> Self {
138 35 : Self {
139 35 : fields,
140 35 : stack: vec![MarkdownNode::new(MarkdownNodeType::DOCUMENT)],
141 35 : text: String::new(),
142 35 : in_html_block: false,
143 35 : }
144 35 : }
145 :
146 35 : fn finish(mut self) -> MarkdownNode {
147 35 : self.flush_text();
148 35 : while self.stack.len() > 1 {
149 0 : self.close();
150 0 : }
151 35 : self.stack.pop().unwrap()
152 35 : }
153 :
154 30 : fn contains_field(&self, text: &str) -> bool {
155 30 : self.fields.iter().any(|field| text.contains(&field.token))
156 30 : }
157 :
158 226 : fn append(&mut self, node: MarkdownNode) {
159 226 : self.stack.last_mut().unwrap().children.push(node);
160 226 : }
161 :
162 105 : fn open(&mut self, node: MarkdownNode) {
163 105 : self.flush_text();
164 105 : self.stack.push(node);
165 105 : }
166 :
167 105 : fn close(&mut self) {
168 105 : self.flush_text();
169 105 : if self.stack.len() > 1 {
170 105 : let mut node = self.stack.pop().unwrap();
171 105 : if node.node_type == MarkdownNodeType::LIST {
172 : // pulldown-cmark does not report the tightness of a list, but it omits the paragraphs of tight items
173 8 : node.tight = !node
174 8 : .children
175 8 : .iter()
176 14 : .any(|item| item.children.iter().any(|child| child.node_type == MarkdownNodeType::PARAGRAPH));
177 97 : }
178 105 : self.append(node);
179 0 : }
180 105 : }
181 :
182 : /// Add the pending text, with field tokens replaced by field nodes
183 250 : fn flush_text(&mut self) {
184 250 : if !self.text.is_empty() {
185 90 : let text = std::mem::take(&mut self.text);
186 90 : self.append_text(&text);
187 160 : }
188 250 : }
189 :
190 93 : fn append_text(&mut self, text: &str) {
191 93 : let mut rest = text;
192 122 : while !rest.is_empty() {
193 102 : let next_field = self
194 102 : .fields
195 102 : .iter()
196 124 : .filter_map(|field| rest.find(&field.token).map(|at| (at, field)))
197 102 : .min_by_key(|(at, _)| *at);
198 102 : let Some((at, field)) = next_field else {
199 73 : self.append(MarkdownNode::leaf(MarkdownNodeType::TEXT, rest));
200 73 : break;
201 : };
202 29 : if at > 0 {
203 14 : self.append(MarkdownNode::leaf(MarkdownNodeType::TEXT, &rest[..at]));
204 19 : }
205 29 : self.append(MarkdownNode::leaf(MarkdownNodeType::FIELD, &field.name));
206 29 : rest = &rest[at + field.token.len()..];
207 : }
208 93 : }
209 :
210 : /// A link or an image, a span if the URL is unsafe or contains a field
211 16 : fn link(&self, node_type: MarkdownNodeType, link_type: LinkType, url: &str, title: &str) -> MarkdownNode {
212 16 : let url = if link_type == LinkType::Email { format!("mailto:{url}") } else { url.to_string() };
213 16 : let safe = is_safe_url(&url) && !self.contains_field(&url);
214 16 : let mut node = MarkdownNode::new(if safe { node_type } else { MarkdownNodeType::SPAN });
215 16 : if !self.contains_field(title) {
216 13 : node.title = title.to_string();
217 13 : }
218 16 : node.url = url;
219 16 : node
220 16 : }
221 :
222 104 : fn start(&mut self, tag: Tag) {
223 104 : let node = match tag {
224 35 : Tag::Paragraph => MarkdownNode::new(MarkdownNodeType::PARAGRAPH),
225 10 : Tag::Heading { level, .. } => {
226 10 : MarkdownNode { level: level as u8, ..MarkdownNode::new(MarkdownNodeType::HEADING) }
227 : }
228 4 : Tag::BlockQuote(_) => MarkdownNode::new(MarkdownNodeType::BLOCKQUOTE),
229 4 : Tag::CodeBlock(kind) => {
230 4 : let language = match kind {
231 2 : CodeBlockKind::Fenced(info) => info.split_whitespace().next().unwrap_or_default().to_string(),
232 2 : CodeBlockKind::Indented => String::new(),
233 : };
234 4 : MarkdownNode { language, ..MarkdownNode::new(MarkdownNodeType::CODE_BLOCK) }
235 : }
236 8 : Tag::List(start) => MarkdownNode {
237 8 : ordered: start.is_some(),
238 8 : start: start.map_or(0, |start| u32::try_from(start).unwrap_or(u32::MAX)),
239 8 : ..MarkdownNode::new(MarkdownNodeType::LIST)
240 : },
241 14 : Tag::Item => MarkdownNode::new(MarkdownNodeType::LIST_ITEM),
242 5 : Tag::Emphasis => MarkdownNode::new(MarkdownNodeType::EMPHASIS),
243 6 : Tag::Strong => MarkdownNode::new(MarkdownNodeType::STRONG),
244 14 : Tag::Link { link_type, dest_url, title, .. } => {
245 14 : self.link(MarkdownNodeType::LINK, link_type, &dest_url, &title)
246 : }
247 2 : Tag::Image { link_type, dest_url, title, .. } => {
248 2 : self.link(MarkdownNodeType::IMAGE, link_type, &dest_url, &title)
249 : }
250 : Tag::HtmlBlock => {
251 2 : self.in_html_block = true;
252 2 : return;
253 : }
254 : // The extensions are not enabled
255 0 : _ => MarkdownNode::new(MarkdownNodeType::SPAN),
256 : };
257 102 : self.open(node);
258 104 : }
259 :
260 327 : fn event(&mut self, event: Event) {
261 327 : if self.in_html_block {
262 6 : self.in_html_block = event != Event::End(TagEnd::HtmlBlock);
263 6 : return;
264 321 : }
265 321 : match event {
266 104 : Event::Start(tag) => self.start(tag),
267 102 : Event::End(_) => self.close(),
268 101 : Event::Text(text) => self.text.push_str(&text),
269 1 : Event::SoftBreak => self.text.push('\n'),
270 3 : Event::Code(code) => {
271 3 : self.open(MarkdownNode::new(MarkdownNodeType::INLINE_CODE));
272 3 : self.append_text(&code);
273 3 : self.close();
274 3 : }
275 3 : Event::HardBreak => {
276 3 : self.flush_text();
277 3 : self.append(MarkdownNode::new(MarkdownNodeType::HARD_BREAK));
278 3 : }
279 2 : Event::Rule => {
280 2 : self.flush_text();
281 2 : self.append(MarkdownNode::new(MarkdownNodeType::THEMATIC_BREAK));
282 2 : }
283 : // Inline HTML is dropped, the other events belong to extensions that are not enabled
284 5 : _ => {}
285 : }
286 327 : }
287 : }
288 :
289 : /// Compile a Markdown row template into a document
290 35 : pub fn compile_markdown_template(row_template: &str) -> MarkdownDocument {
291 35 : let (markdown, fields) = tokenize_fields(row_template);
292 : // CommonMark replaces NUL characters for security reasons, pulldown-cmark leaves that to the caller
293 35 : let markdown = markdown.replace('\0', "\u{fffd}");
294 35 : let mut builder = DocumentBuilder::new(&fields);
295 327 : for event in Parser::new(&markdown) {
296 327 : builder.event(event);
297 327 : }
298 35 : let document = builder.finish();
299 : MarkdownDocument {
300 35 : template: row_template.to_string(),
301 35 : fields: fields.into_iter().map(|field| field.name).collect(),
302 35 : document,
303 : }
304 35 : }
305 :
306 : /// Compile the Markdown spec of a visualization, `None` if it has none
307 17 : pub fn compile_markdown_spec(spec: &VisualizationSpec) -> Option<MarkdownDocument> {
308 17 : Some(compile_markdown_template(&spec.markdown.as_ref()?.row_template))
309 17 : }
310 :
311 : /// Pack a node, empty strings and vectors are omitted
312 25 : fn pack_markdown_node<'a>(
313 25 : builder: &mut FlatBufferBuilder<'a>,
314 25 : node: &MarkdownNode,
315 25 : ) -> WIPOffset<MarkdownNodeBuffer<'a>> {
316 100 : let mut string = |s: &str| if s.is_empty() { None } else { Some(builder.create_string(s)) };
317 25 : let value = string(&node.value);
318 25 : let url = string(&node.url);
319 25 : let title = string(&node.title);
320 25 : let language = string(&node.language);
321 25 : let children = if node.children.is_empty() {
322 9 : None
323 : } else {
324 18 : let children: Vec<_> = node.children.iter().map(|child| pack_markdown_node(builder, child)).collect();
325 16 : Some(builder.create_vector(&children))
326 : };
327 25 : MarkdownNodeBuffer::create(
328 25 : builder,
329 25 : &MarkdownNodeArgs {
330 25 : node_type: node.node_type,
331 25 : value,
332 25 : url,
333 25 : title,
334 25 : language,
335 25 : level: node.level,
336 25 : start: node.start,
337 25 : ordered: node.ordered,
338 25 : tight: node.tight,
339 25 : children,
340 25 : },
341 : )
342 25 : }
343 :
344 : /// Pack a compiled document
345 7 : pub fn pack_markdown_spec<'a>(
346 7 : builder: &mut FlatBufferBuilder<'a>,
347 7 : spec: &MarkdownDocument,
348 7 : ) -> WIPOffset<MarkdownSpec<'a>> {
349 7 : let template = if spec.template.is_empty() { None } else { Some(builder.create_string(&spec.template)) };
350 7 : let fields = if spec.fields.is_empty() {
351 0 : None
352 : } else {
353 7 : let fields: Vec<_> = spec.fields.iter().map(|f| builder.create_string(f)).collect();
354 7 : Some(builder.create_vector(&fields))
355 : };
356 7 : let document = Some(pack_markdown_node(builder, &spec.document));
357 7 : MarkdownSpec::create(builder, &MarkdownSpecArgs { template, fields, document })
358 7 : }
359 :
360 : #[cfg(test)]
361 : mod tests;
|
