Line data Source code
1 : //! Offset conversions of the script rope.
2 :
3 : use ropey::Rope;
4 :
5 : /// A position in the text in all offset units
6 : #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
7 : pub struct TextPosition {
8 : /// The UTF-8 byte offset
9 : pub text_bytes: usize,
10 : /// The code point offset
11 : pub utf8_codepoints: usize,
12 : /// The UTF-16 code unit offset
13 : pub utf16_code_units: usize,
14 : }
15 :
16 : /// Resolve a UTF-8 byte offset that lies on a code point boundary
17 6686 : pub fn resolve_byte_boundary(text: &Rope, byte_idx: usize) -> Option<TextPosition> {
18 6686 : if byte_idx > text.len_bytes() {
19 4 : return None;
20 6682 : }
21 6682 : let char_idx = text.byte_to_char(byte_idx);
22 6682 : if text.char_to_byte(char_idx) != byte_idx {
23 3209 : return None;
24 3473 : }
25 3473 : Some(TextPosition {
26 3473 : text_bytes: byte_idx,
27 3473 : utf8_codepoints: char_idx,
28 3473 : utf16_code_units: text.char_to_utf16_cu(char_idx),
29 3473 : })
30 6686 : }
31 :
32 : /// Resolve a UTF-16 code unit offset that does not split a surrogate pair
33 72 : pub fn resolve_utf16_boundary(text: &Rope, utf16_idx: usize) -> Option<TextPosition> {
34 72 : if utf16_idx > text.len_utf16_cu() {
35 1 : return None;
36 71 : }
37 71 : let char_idx = text.utf16_cu_to_char(utf16_idx);
38 71 : if text.char_to_utf16_cu(char_idx) != utf16_idx {
39 2 : return None;
40 69 : }
41 69 : Some(TextPosition {
42 69 : text_bytes: text.char_to_byte(char_idx),
43 69 : utf8_codepoints: char_idx,
44 69 : utf16_code_units: utf16_idx,
45 69 : })
46 72 : }
47 :
48 : /// Converts UTF-8 byte offsets of a text to UTF-16 code units.
49 : ///
50 : /// Projections convert many offsets that mostly ascend (tokens, comments, references). The
51 : /// converter remembers the last position and walks forward from it over a short distance, and looks
52 : /// the offset up in the rope otherwise, so that converting all spans of a script stays linear.
53 : pub struct Utf16Projector<'a> {
54 : /// The text of the rope, if it is at hand
55 : text: Option<&'a str>,
56 : /// The rope to look offsets up in
57 : rope: &'a Rope,
58 : /// The byte offset of the last projection
59 : last_byte: usize,
60 : /// The UTF-16 offset of the last projection
61 : last_utf16: usize,
62 : }
63 :
64 : /// The longest distance that is walked instead of looked up
65 : const MAX_WALK: usize = 256;
66 :
67 : impl<'a> Utf16Projector<'a> {
68 : /// Create a converter. `text` must be the text of `rope`, without it every offset is looked up.
69 31 : pub fn new(text: Option<&'a str>, rope: &'a Rope) -> Self {
70 31 : debug_assert!(text.is_none_or(|t| t.len() == rope.len_bytes()));
71 31 : Self { text, rope, last_byte: 0, last_utf16: 0 }
72 31 : }
73 :
74 : /// The UTF-16 offset of a byte offset, None if it is past the end or not on a code point boundary
75 6753 : pub fn project(&mut self, byte_idx: usize) -> Option<usize> {
76 6753 : let Some(text) = self.text else {
77 3265 : return resolve_byte_boundary(self.rope, byte_idx).map(|p| p.utf16_code_units);
78 : };
79 3488 : if byte_idx > text.len() || !text.is_char_boundary(byte_idx) {
80 1603 : return None;
81 1885 : }
82 1885 : if byte_idx >= self.last_byte && byte_idx - self.last_byte <= MAX_WALK {
83 1042 : let units: usize = text[self.last_byte..byte_idx].chars().map(char::len_utf16).sum();
84 1042 : self.last_utf16 += units;
85 1042 : } else {
86 843 : self.last_utf16 = self.rope.char_to_utf16_cu(self.rope.byte_to_char(byte_idx));
87 843 : }
88 1885 : self.last_byte = byte_idx;
89 1885 : Some(self.last_utf16)
90 6753 : }
91 : }
92 :
93 : #[cfg(test)]
94 : mod tests {
95 : use super::*;
96 :
97 : #[test]
98 1 : fn resolves_boundaries() {
99 : // a, é (2 bytes), 😀 (4 bytes, a surrogate pair), z
100 1 : let rope = Rope::from_str("a\u{e9}\u{1F600}z");
101 1 : assert_eq!(resolve_byte_boundary(&rope, 0).unwrap().utf16_code_units, 0);
102 1 : assert_eq!(resolve_byte_boundary(&rope, 1).unwrap().utf16_code_units, 1);
103 1 : assert!(resolve_byte_boundary(&rope, 2).is_none());
104 1 : assert_eq!(resolve_byte_boundary(&rope, 3).unwrap().utf16_code_units, 2);
105 1 : assert!(resolve_byte_boundary(&rope, 4).is_none());
106 1 : let end = resolve_byte_boundary(&rope, 7).unwrap();
107 1 : assert_eq!((end.utf8_codepoints, end.utf16_code_units), (3, 4));
108 1 : assert_eq!(resolve_byte_boundary(&rope, 8).unwrap().utf16_code_units, 5);
109 1 : assert!(resolve_byte_boundary(&rope, 9).is_none());
110 :
111 1 : assert_eq!(resolve_utf16_boundary(&rope, 2).unwrap().text_bytes, 3);
112 1 : assert!(resolve_utf16_boundary(&rope, 3).is_none());
113 1 : assert_eq!(resolve_utf16_boundary(&rope, 4).unwrap().text_bytes, 7);
114 1 : assert_eq!(resolve_utf16_boundary(&rope, 5).unwrap().text_bytes, 8);
115 1 : assert!(resolve_utf16_boundary(&rope, 6).is_none());
116 1 : }
117 :
118 : #[test]
119 1 : fn projects_like_lookups() {
120 1 : let text = "x\u{1F600}y\u{e9}".repeat(200);
121 1 : let rope = Rope::from_str(&text);
122 1 : let mut projector = Utf16Projector::new(Some(&text), &rope);
123 1 : let mut lookups = Utf16Projector::new(None, &rope);
124 3208 : for offset in (0..=text.len() + 1).chain((0..=text.len()).rev()).chain([3, 900, 5, 1200, 0]) {
125 3208 : let expected = resolve_byte_boundary(&rope, offset).map(|p| p.utf16_code_units);
126 3208 : assert_eq!(projector.project(offset), expected, "{offset}");
127 3208 : assert_eq!(lookups.project(offset), expected, "{offset}");
128 : }
129 1 : }
130 : }
|
