Line data Source code
1 : import {
2 : STORAGE_ATTACHMENTS_FOLDER,
3 : STORAGE_CONNECTIONS_FOLDER,
4 : STORAGE_MANIFEST_FILE,
5 : STORAGE_SCRIPTS_FOLDER,
6 : STORAGE_SESSION_FILE,
7 : STORAGE_SESSIONS_FOLDER,
8 : STORAGE_SHELL_FOLDER,
9 : StorageBackendType,
10 : } from './storage_backend.js';
11 : import { OPFS_MANIFEST_LOCK, OPFS_SESSION_TREE_LOCK } from './opfs_storage_backend.js';
12 : import { type ManifestJson, readManifestJson } from './migration/storage_root.js';
13 : import { type SessionEntry, STORAGE_SESSION_TREE_FOLDER, flattenSessionTree, readSessionTree } from './session_tree.js';
14 : import { SHELL_FILES_DIRECTORY } from '../platform/hyperdb/hyperdb_opfs.js';
15 : import {
16 : HYPER_INSTANCE_LOCK_PREFIX,
17 : HYPER_INSTANCES_DIRECTORY,
18 : HYPER_LEGACY_INSTANCE_PREFIX,
19 : HYPER_LEGACY_LOGS_DIRECTORY,
20 : isUuid,
21 : } from '../platform/hyperdb/hyperdb_opfs_instance.js';
22 : import { PARQUET_EXPORT_DIRECTORY } from '../platform/hyperdb/hyperdb_parquet_export.js';
23 : import { QUERY_RESULT_CACHE_DIRECTORY } from './opfs_query_result_cache.js';
24 : import { OPFS_TESTS_FOLDER } from '../platform/opfs_root.js';
25 :
26 : /// An OPFS entry that the current app version doesn't use
27 : export interface OPFSCleanupCandidate {
28 : /// The path segments from the OPFS root
29 : path: string[];
30 : /// Is the entry a directory?
31 : directory: boolean;
32 : /// Why the entry can be deleted
33 : reason: string;
34 : /// The registration that would make the entry used again, checked again before the entry is deleted
35 : guard: { sessionId: string } | { connectionId: string } | null;
36 : /// The Web Lock that marks the entry as in use, the entry is only deleted while the lock is free
37 : lockName: string | null;
38 : }
39 :
40 : /// The outcome of a cleanup
41 : export interface OPFSCleanupResult {
42 : /// The number of deleted entries
43 : removed: number;
44 : /// The entries that were kept
45 : skipped: { path: string; reason: string }[];
46 : }
47 :
48 : /// The sessions that the session tree and the connections that the manifest registers, with lower-case ids
49 : interface Registrations {
50 : /// The sessions that live in OPFS
51 : opfsSessions: Set<string>;
52 : /// The sessions that live in a native folder, OPFS keeps a mirror of their attachments
53 : nativeSessions: Set<string>;
54 : /// The connections
55 : connections: Set<string>;
56 : }
57 :
58 : /// Read the registrations of the manifest and the session tree, null without a manifest
59 1 : function readRegistrations(manifest: ManifestJson | null, sessions: readonly SessionEntry[]): Registrations | null {
60 1 : if (manifest == null) return null;
61 1 : const registrations: Registrations = { opfsSessions: new Set(), nativeSessions: new Set(), connections: new Set() };
62 1 : for (const { path, storageType } of sessions) {
63 1 : (storageType === StorageBackendType.Native ? registrations.nativeSessions : registrations.opfsSessions).add(path.toLowerCase());
64 : }
65 1 : for (const entry of Array.isArray(manifest.connections) ? manifest.connections as unknown[] : []) {
66 1 : const { id } = (entry ?? {}) as { id?: unknown };
67 1 : if (typeof id === 'string') registrations.connections.add(id.toLowerCase());
68 1 : }
69 1 : return registrations;
70 1 : }
71 :
72 : /// Is the guarded registration in the manifest? Without a manifest every guarded entry counts as registered.
73 1 : function isRegistered(guard: OPFSCleanupCandidate['guard'], registrations: Registrations | null): boolean {
74 1 : if (guard == null) return false;
75 1 : if (registrations == null) return true;
76 1 : if ('sessionId' in guard) {
77 1 : const id = guard.sessionId.toLowerCase();
78 1 : return registrations.opfsSessions.has(id) || registrations.nativeSessions.has(id);
79 1 : }
80 1 : return registrations.connections.has(guard.connectionId.toLowerCase());
81 1 : }
82 :
83 : /// The entries of a session folder in OPFS. The folder of a native session only mirrors its attachments.
84 1 : const OPFS_SESSION_ENTRIES = new Map<string, boolean>([
85 1 : [STORAGE_SESSION_FILE, false],
86 1 : [STORAGE_SCRIPTS_FOLDER, true],
87 1 : [STORAGE_ATTACHMENTS_FOLDER, true],
88 1 : ]);
89 1 : const NATIVE_SESSION_ENTRIES = new Map<string, boolean>([[STORAGE_ATTACHMENTS_FOLDER, true]]);
90 :
91 1 : const UNUSED = 'Not used by this version of DashQL';
92 :
93 : /// Find the OPFS entries that the current app version doesn't use.
94 : ///
95 : /// The scan compares the OPFS root against the current layout: the manifest, the session tree, the
96 : /// registered sessions and connections, the shell folders, the instance folders of open app windows and the Parquet
97 : /// exports. Everything else was left by earlier app versions, by interrupted operations or by sessions
98 : /// and connections that are gone. The scan reads directory listings only, it never reads files besides
99 : /// the manifest and the folder files of the session tree and only descends into the folders it knows.
100 1 : export async function findOPFSCleanupCandidates(
101 1 : root: FileSystemDirectoryHandle,
102 1 : locks: LockManager | null = navigator.locks ?? null,
103 1 : ): Promise<OPFSCleanupCandidate[]> {
104 : // Without a manifest there is no telling which sessions and connections are registered, they are left alone
105 1 : const registrations = await readRootRegistrations(root);
106 : // Without Web Locks there is no telling which instance folders are alive, they are left alone
107 1 : const heldLocks = locks == null ? null : new Set(((await locks.query()).held ?? []).map(lock => lock.name));
108 1 : const candidates: OPFSCleanupCandidate[] = [];
109 1 : const add = (path: string[], directory: boolean, reason: string, guard: OPFSCleanupCandidate['guard'] = null, lockName: string | null = null) => {
110 1 : candidates.push({ path, directory, reason, guard, lockName });
111 1 : };
112 :
113 1 : for await (const [name, handle] of root.entries()) {
114 1 : const directory = handle.kind === 'directory';
115 1 : if (name === STORAGE_MANIFEST_FILE && !directory) continue;
116 1 : if (!directory) {
117 1 : add([name], false, UNUSED);
118 1 : continue;
119 1 : }
120 1 : const dir = handle as FileSystemDirectoryHandle;
121 1 : switch (name) {
122 1 : case STORAGE_SESSIONS_FOLDER:
123 1 : if (registrations != null) await scanSessions(dir, registrations, add);
124 1 : break;
125 1 : case STORAGE_CONNECTIONS_FOLDER:
126 1 : if (registrations == null) break;
127 1 : for await (const [id, child] of dir.entries()) {
128 1 : if (child.kind !== 'directory') add([name, id], false, UNUSED);
129 1 : else if (!registrations.connections.has(id.toLowerCase())) add([name, id], true, 'Connection that is not registered', { connectionId: id });
130 : }
131 1 : break;
132 1 : case HYPER_INSTANCES_DIRECTORY:
133 1 : if (heldLocks == null) break;
134 1 : for await (const [id, child] of dir.entries()) {
135 1 : const lockName = `${HYPER_INSTANCE_LOCK_PREFIX}${id}`;
136 1 : if (child.kind !== 'directory' || !isUuid(id)) add([name, id], child.kind === 'directory', UNUSED);
137 1 : else if (!heldLocks.has(lockName)) add([name, id], true, 'Database of a closed app window', null, lockName);
138 1 : }
139 1 : break;
140 1 : case PARQUET_EXPORT_DIRECTORY:
141 1 : for await (const [id, child] of dir.entries()) {
142 1 : add([name, id], child.kind === 'directory', 'Leftover of an interrupted Parquet export');
143 : }
144 1 : break;
145 1 : case QUERY_RESULT_CACHE_DIRECTORY:
146 : // The cache evicts its own files, also the leftovers of interrupted runs
147 1 : break;
148 1 : case STORAGE_SHELL_FOLDER:
149 1 : case SHELL_FILES_DIRECTORY:
150 1 : case STORAGE_SESSION_TREE_FOLDER:
151 1 : break;
152 1 : case HYPER_LEGACY_LOGS_DIRECTORY:
153 1 : add([name], true, 'Hyper logs of an earlier version');
154 1 : break;
155 1 : case OPFS_TESTS_FOLDER:
156 1 : add([name], true, 'Files of the browser tests');
157 1 : break;
158 1 : default:
159 1 : if (name.startsWith(HYPER_LEGACY_INSTANCE_PREFIX) && isUuid(name.slice(HYPER_LEGACY_INSTANCE_PREFIX.length))) {
160 1 : if (heldLocks != null && !heldLocks.has(name)) add([name], true, 'Database of an earlier version', null, name);
161 : } else {
162 1 : add([name], true, UNUSED);
163 : }
164 1 : }
165 1 : }
166 1 : return candidates.sort((a, b) => a.path.join('/').localeCompare(b.path.join('/'), undefined, { numeric: true }));
167 1 : }
168 :
169 : /// Scan the session folders against the registered sessions
170 1 : async function scanSessions(
171 1 : sessions: FileSystemDirectoryHandle,
172 1 : registrations: Registrations,
173 1 : add: (path: string[], directory: boolean, reason: string, guard?: OPFSCleanupCandidate['guard']) => void,
174 1 : ): Promise<void> {
175 1 : for await (const [id, handle] of sessions.entries()) {
176 1 : const path = [STORAGE_SESSIONS_FOLDER, id];
177 1 : if (handle.kind !== 'directory') {
178 0 : add(path, false, UNUSED);
179 0 : continue;
180 0 : }
181 1 : const key = id.toLowerCase();
182 1 : const native = registrations.nativeSessions.has(key);
183 1 : if (!native && !registrations.opfsSessions.has(key)) {
184 1 : add(path, true, 'Session that is not registered', { sessionId: id });
185 1 : continue;
186 1 : }
187 1 : const expected = native ? NATIVE_SESSION_ENTRIES : OPFS_SESSION_ENTRIES;
188 1 : for await (const [name, child] of (handle as FileSystemDirectoryHandle).entries()) {
189 1 : const directory = child.kind === 'directory';
190 1 : if (expected.get(name) !== directory) {
191 1 : add([...path, name], directory, native ? 'Left behind when the session moved to a folder' : 'Not part of the current session layout');
192 1 : } else if (name === STORAGE_SCRIPTS_FOLDER) {
193 : // Scripts are flat files, script folders are pages of an earlier version
194 1 : for await (const [script, entry] of (child as FileSystemDirectoryHandle).entries()) {
195 1 : if (entry.kind === 'directory') add([...path, name, script], true, 'Script folder of an earlier version');
196 : }
197 : }
198 1 : }
199 1 : }
200 1 : }
201 :
202 : /// Delete cleanup candidates.
203 : ///
204 : /// The deletion holds the manifest and session tree locks and reads the registrations again, a candidate whose session or
205 : /// connection was registered in the meantime is kept. A candidate with a lock is only deleted while
206 : /// its lock is free, and files that are still open are kept.
207 1 : export async function removeOPFSCleanupCandidates(
208 1 : root: FileSystemDirectoryHandle,
209 1 : candidates: readonly OPFSCleanupCandidate[],
210 1 : locks: LockManager | null = navigator.locks ?? null,
211 1 : ): Promise<OPFSCleanupResult> {
212 1 : const run = async (): Promise<OPFSCleanupResult> => {
213 1 : const registrations = await readRootRegistrations(root);
214 1 : const result: OPFSCleanupResult = { removed: 0, skipped: [] };
215 1 : for (const candidate of candidates) {
216 1 : const path = candidate.path.join('/');
217 1 : if (isRegistered(candidate.guard, registrations)) {
218 1 : result.skipped.push({ path, reason: 'registered again' });
219 1 : continue;
220 1 : }
221 1 : const failure = candidate.lockName != null && locks != null
222 1 : ? await locks.request(candidate.lockName, { ifAvailable: true }, lock => lock == null ? 'in use' : removeEntry(root, candidate))
223 1 : : await removeEntry(root, candidate);
224 1 : if (failure == null) ++result.removed;
225 1 : else result.skipped.push({ path, reason: failure });
226 1 : }
227 1 : return result;
228 1 : };
229 : // Sessions are registered in the session tree and connections in the manifest, the deletion holds both locks
230 1 : return locks != null
231 1 : ? await locks.request(OPFS_MANIFEST_LOCK, () => locks.request(OPFS_SESSION_TREE_LOCK, run))
232 1 : : await run();
233 1 : }
234 :
235 : /// Read the registrations of a storage root
236 1 : async function readRootRegistrations(root: FileSystemDirectoryHandle): Promise<Registrations | null> {
237 1 : return readRegistrations(await readManifestJson(root), flattenSessionTree(await readSessionTree(root)));
238 1 : }
239 :
240 : /// Remove an entry, returns why it was kept or null
241 1 : async function removeEntry(root: FileSystemDirectoryHandle, candidate: OPFSCleanupCandidate): Promise<string | null> {
242 1 : try {
243 1 : let parent = root;
244 1 : for (const name of candidate.path.slice(0, -1)) {
245 1 : parent = await parent.getDirectoryHandle(name, { create: false });
246 : }
247 1 : await parent.removeEntry(candidate.path[candidate.path.length - 1], { recursive: candidate.directory });
248 1 : return null;
249 1 : } catch (error) {
250 1 : const name = (error as DOMException)?.name;
251 1 : if (name === 'NotFoundError') return null;
252 : // Browsers report files that are still open with either modification error
253 1 : if (name === 'NoModificationAllowedError' || name === 'InvalidModificationError') return 'in use';
254 0 : return (error as Error)?.message ?? String(error);
255 1 : }
256 1 : }
|
