Line data Source code
1 : import type { HttpClient, HttpFetchResult } from '../../platform/http/http_client.js';
2 : import type { DataFileData, ScriptData } from '../../storage/storage_backend.js';
3 : import {
4 : STORAGE_SESSION_FILE,
5 : STORAGE_ATTACHMENTS_FOLDER,
6 : STORAGE_SCRIPTS_FOLDER,
7 : } from '../../storage/storage_backend.js';
8 : import type { SessionBundle } from '../../storage/session_bundle.js';
9 : import { assembleSessionBundle, readSessionBundleHead } from '../../storage/session_bundle.js';
10 : import { type MigrationFolder, OverlayFolder } from '../../storage/migration/migration_folder.js';
11 : import { LEGACY_SESSION_FILE } from '../../platform/events/legacy_links.js';
12 :
13 1 : const MAX_MANIFEST_BYTES = 1024 * 1024;
14 1 : const MAX_SCRIPT_BYTES = 8 * 1024 * 1024;
15 1 : const MAX_TOTAL_BYTES = 32 * 1024 * 1024;
16 :
17 : export interface HttpSessionLoadResult {
18 : /// The loaded session
19 : bundle: SessionBundle;
20 : /// The number of scripts that the session file lists
21 : listedScriptCount: number;
22 : /// The number of scripts that were loaded
23 : loadedScriptCount: number;
24 : /// Were files missing, unusable or over the size budget?
25 : incomplete: boolean;
26 : }
27 :
28 : export type HttpSessionLoadProgress =
29 : | { phase: 'preparing' }
30 : | { phase: 'manifest' }
31 : | { phase: 'listing'; sessionName: string; sessionId: string }
32 : | {
33 : phase: 'files';
34 : sessionName: string;
35 : sessionId: string;
36 : completedFileCount: number;
37 : totalFileCount: number;
38 : completedScriptCount: number;
39 : totalScriptCount: number;
40 : };
41 :
42 1 : export async function readSessionBundleFromHttp(
43 1 : manifestUrl: URL,
44 1 : httpClient: HttpClient,
45 1 : signal?: AbortSignal,
46 1 : onProgress?: (progress: HttpSessionLoadProgress) => void,
47 1 : ): Promise<HttpSessionLoadResult> {
48 1 : const manifestName = validateManifestUrl(manifestUrl);
49 1 : const baseUrl = new URL('./', manifestUrl);
50 1 : let totalBytes = 0;
51 :
52 1 : onProgress?.({ phase: 'manifest' });
53 1 : const manifestText = await fetchRequiredText(httpClient, manifestUrl, manifestName, MAX_MANIFEST_BYTES, signal);
54 1 : totalBytes += textBytes(manifestText);
55 : // Sessions published by older versions are migrated in memory
56 1 : const folder = new OverlayFolder(new HttpFolder(baseUrl, httpClient, MAX_SCRIPT_BYTES, signal, manifestName, manifestText));
57 1 : const head = await readSessionBundleHead(folder, 'Invalid remote session');
58 1 : const session = head.session;
59 1 : const sessionName = session.name?.trim() || session.metadata.originalFileName || 'Unnamed session';
60 1 : totalBytes += head.catalogs.flatMap(catalog => [catalog.relations, catalog.functions])
61 1 : .reduce((bytes, text) => bytes + (text == null ? 0 : textBytes(text)), 0);
62 :
63 1 : onProgress?.({ phase: 'listing', sessionName, sessionId: session.sessionId });
64 1 : const listedScripts = session.scripts;
65 1 : const listedAttachments = session.attachments;
66 1 : const listedScriptCount = listedScripts.length;
67 : // The manifest and the catalog files of the connections
68 1 : const headFileCount = 1 + 2 * session.connections.length;
69 1 : const totalFileCount = headFileCount + listedScriptCount + listedAttachments.length;
70 1 : let completedFileCount = headFileCount;
71 1 : let completedScriptCount = 0;
72 1 : const reportFileProgress = () => onProgress?.({
73 1 : phase: 'files',
74 1 : sessionName,
75 1 : sessionId: session.sessionId,
76 1 : completedFileCount,
77 1 : totalFileCount,
78 1 : completedScriptCount,
79 1 : totalScriptCount: listedScriptCount,
80 1 : });
81 1 : const trackFile = async <T>(load: Promise<T>, script: boolean = false): Promise<T> => {
82 1 : try {
83 1 : return await load;
84 1 : } finally {
85 1 : completedFileCount += 1;
86 1 : if (script) completedScriptCount += 1;
87 1 : reportFileProgress();
88 1 : }
89 1 : };
90 1 : reportFileProgress();
91 :
92 1 : const scripts: ScriptData[] = (await Promise.all(listedScripts.map(async script => {
93 1 : const sql = await trackFile(fetchBestEffortText(
94 1 : httpClient,
95 1 : sessionFileUrl(baseUrl, script.name),
96 1 : MAX_SCRIPT_BYTES,
97 1 : signal,
98 1 : ), true);
99 1 : return sql == null ? null : { name: script.name, sql };
100 1 : }))).filter((script): script is ScriptData => script != null);
101 :
102 1 : totalBytes += scripts.reduce((bytes, script) => bytes + textBytes(script.sql), 0);
103 1 : const data = (await Promise.all(listedAttachments.map(async file => {
104 1 : const bytes = await trackFile(fetchBestEffortBytes(httpClient,
105 1 : new URL(`${STORAGE_ATTACHMENTS_FOLDER}/${encodeURIComponent(file.name)}`, baseUrl), MAX_TOTAL_BYTES, signal));
106 1 : return bytes == null ? null : { name: file.name, bytes };
107 1 : }))).filter((file): file is DataFileData => file != null);
108 1 : totalBytes += data.reduce((size, file) => size + file.bytes.byteLength, 0);
109 : // Optional files are loaded only while the session remains within the total budget. The
110 : // manifest is the sole required file, so an oversized optional tree degrades to metadata only.
111 1 : const includeOptionalFiles = totalBytes <= MAX_TOTAL_BYTES;
112 :
113 : // Scripts that failed to load are dropped from the list
114 1 : const bundle = assembleSessionBundle({
115 1 : session: {
116 1 : ...session,
117 1 : metadata: {
118 1 : ...session.metadata,
119 1 : originType: 'HTTP',
120 1 : originalHttpUrl: manifestUrl.toString(),
121 1 : },
122 1 : },
123 1 : catalogs: includeOptionalFiles ? head.catalogs : [],
124 1 : }, includeOptionalFiles ? scripts : [], includeOptionalFiles ? data : []);
125 1 : const loadedScriptCount = bundle.scripts.length;
126 1 : return {
127 1 : bundle,
128 1 : listedScriptCount,
129 1 : loadedScriptCount,
130 1 : incomplete: !head.filesValid || !includeOptionalFiles || loadedScriptCount !== listedScriptCount
131 1 : || data.length !== listedAttachments.length,
132 1 : };
133 1 : }
134 :
135 : /// The top-level files of a hosted session, fetched when a reader asks for them.
136 : ///
137 : /// Each file is fetched at most once, a missing or unreadable file reads as null. The folder is
138 : /// read-only, the session migration changes it through an `OverlayFolder`.
139 1 : class HttpFolder implements MigrationFolder {
140 : /// The URL of the session folder
141 1 : private readonly baseUrl: URL;
142 : /// The client that fetches the files
143 1 : private readonly httpClient: HttpClient;
144 : /// The maximum size of a file
145 1 : private readonly maxBytes: number;
146 : /// The signal that aborts the fetches
147 1 : private readonly signal: AbortSignal | undefined;
148 : /// The fetched files by name, null for missing ones
149 1 : private readonly files: Map<string, Promise<string | null>>;
150 :
151 1 : constructor(baseUrl: URL, httpClient: HttpClient, maxBytes: number, signal: AbortSignal | undefined, manifestName: string, manifestText: string) {
152 1 : this.baseUrl = baseUrl;
153 1 : this.httpClient = httpClient;
154 1 : this.maxBytes = maxBytes;
155 1 : this.signal = signal;
156 1 : this.files = new Map([[manifestName, Promise.resolve(manifestText)]]);
157 1 : }
158 :
159 1 : async exists(name: string): Promise<boolean> {
160 1 : return (await this.readText(name)) != null;
161 1 : }
162 :
163 1 : readText(name: string): Promise<string | null> {
164 1 : let file = this.files.get(name);
165 1 : if (file === undefined) {
166 1 : file = fetchBestEffortText(this.httpClient, new URL(name, this.baseUrl), this.maxBytes, this.signal);
167 1 : this.files.set(name, file);
168 1 : }
169 1 : return file;
170 1 : }
171 :
172 0 : async writeText(name: string): Promise<void> {
173 0 : throw new Error(`Cannot write ${name} of a remote session`);
174 0 : }
175 :
176 0 : async rename(from: string): Promise<void> {
177 0 : throw new Error(`Cannot rename ${from} of a remote session`);
178 0 : }
179 :
180 0 : async remove(name: string): Promise<void> {
181 0 : throw new Error(`Cannot remove ${name} of a remote session`);
182 0 : }
183 1 : }
184 :
185 : /// Validate the URL of a hosted session, returns the name of the session metadata
186 1 : function validateManifestUrl(url: URL): string {
187 1 : const isPublicHttps = url.protocol === 'https:';
188 : const isDevelopmentLoopback = process.env.DASHQL_BUILD_MODE === 'development'
189 : && url.protocol === 'http:'
190 : && (url.hostname === 'localhost' || url.hostname === '127.0.0.1' || url.hostname === '[::1]');
191 1 : if ((!isPublicHttps && !isDevelopmentLoopback) || url.username || url.password || url.hash) {
192 0 : throw new Error('Session URL must be public HTTPS or development loopback HTTP without credentials or a fragment');
193 : }
194 : // Links of older versions point to the metadata under its older name
195 1 : const name = url.pathname.split('/').pop();
196 1 : if (name !== STORAGE_SESSION_FILE && name !== LEGACY_SESSION_FILE) {
197 1 : throw new Error(`Remote session URL must end in ${STORAGE_SESSION_FILE}`);
198 : }
199 1 : return name;
200 1 : }
201 :
202 :
203 1 : async function fetchBestEffortBytes(
204 1 : httpClient: HttpClient, url: URL, maxBytes: number, signal?: AbortSignal,
205 1 : ): Promise<Uint8Array | null> {
206 1 : try {
207 1 : const response = await httpClient.fetch(url, { method: 'GET', credentials: 'omit', signal });
208 1 : if (response.status < 200 || response.status >= 300) return null;
209 1 : const declaredSize = Number(response.headers.get('content-length'));
210 1 : if (Number.isFinite(declaredSize) && declaredSize > maxBytes) return null;
211 1 : const bytes = new Uint8Array(await response.arrayBuffer());
212 1 : return bytes.byteLength > maxBytes ? null : bytes;
213 1 : } catch (error) {
214 0 : if (signal?.aborted || (error as Error)?.name === 'AbortError') throw error;
215 0 : return null;
216 0 : }
217 1 : }
218 :
219 1 : function sessionFileUrl(baseUrl: URL, scriptName: string): URL {
220 1 : return new URL(
221 1 : `${STORAGE_SCRIPTS_FOLDER}/${encodeURIComponent(scriptName)}`,
222 1 : baseUrl,
223 1 : );
224 1 : }
225 :
226 1 : async function fetchRequiredText(
227 1 : httpClient: HttpClient,
228 1 : url: URL,
229 1 : label: string,
230 1 : maxBytes: number,
231 1 : signal?: AbortSignal,
232 1 : ): Promise<string> {
233 1 : const response = await httpClient.fetch(url, { method: 'GET', credentials: 'omit', signal });
234 1 : if (response.status < 200 || response.status >= 300) {
235 0 : throw new Error(`Failed to fetch remote session file ${label}: HTTP ${response.status}`);
236 : }
237 1 : return await readLimitedText(response, label, maxBytes);
238 1 : }
239 :
240 1 : async function fetchBestEffortText(
241 1 : httpClient: HttpClient,
242 1 : url: URL,
243 1 : maxBytes: number,
244 1 : signal?: AbortSignal,
245 1 : ): Promise<string | null> {
246 1 : try {
247 1 : const response = await httpClient.fetch(url, { method: 'GET', credentials: 'omit', signal });
248 1 : if (response.status < 200 || response.status >= 300) return null;
249 1 : return await readLimitedText(response, url.pathname, maxBytes);
250 1 : } catch (error) {
251 0 : if (signal?.aborted || (error as Error)?.name === 'AbortError') throw error;
252 0 : return null;
253 0 : }
254 1 : }
255 :
256 1 : async function readLimitedText(response: HttpFetchResult, label: string, maxBytes: number): Promise<string> {
257 1 : const declaredSize = Number(response.headers.get('content-length'));
258 1 : if (Number.isFinite(declaredSize) && declaredSize > maxBytes) {
259 0 : throw new Error(`Remote session file ${label} exceeds the maximum size`);
260 : }
261 1 : const text = await response.text();
262 1 : if (textBytes(text) > maxBytes) {
263 0 : throw new Error(`Remote session file ${label} exceeds the maximum size`);
264 : }
265 1 : return text;
266 1 : }
267 :
268 :
269 :
270 :
271 1 : function textBytes(value: string): number {
272 1 : return new TextEncoder().encode(value).byteLength;
273 1 : }
|
