Line data Source code
1 : import { stringifyError } from '../logger/logger.js';
2 : import { ClientOptions, HttpClient, HttpFetchResult } from './http_client.js';
3 :
4 : /// The number of requests that the log keeps, older requests are dropped
5 : export const HTTP_REQUEST_LOG_LIMIT = 500;
6 : /// The bytes of a response body that the log keeps
7 1 : export const HTTP_RESPONSE_PREVIEW_BYTES = 16 * 1024;
8 : /// The bytes of a request body that the log keeps, curl commands need the whole body
9 1 : export const HTTP_REQUEST_BODY_BYTES = 1024 * 1024;
10 :
11 : /// The headers whose values are masked in the viewer, copy as curl keeps them
12 1 : const SENSITIVE_HEADERS = new Set([
13 1 : 'authorization',
14 1 : 'proxy-authorization',
15 1 : 'cookie',
16 1 : 'set-cookie',
17 1 : 'x-trino-extra-credential',
18 1 : ]);
19 :
20 : /// A body of a request or a response
21 : export type HttpBodyRecord =
22 : /// A text body, `byteLength` is the size of the whole body
23 : | { kind: 'text'; text: string; byteLength: number; truncated: boolean }
24 : /// A binary body
25 : | { kind: 'binary'; byteLength: number }
26 : /// A JSON body that the caller parsed, the log does not keep it
27 : | { kind: 'parsed' }
28 : /// A body that the log cannot read without consuming it
29 : | { kind: 'opaque' };
30 :
31 : /// A recorded http request
32 : export interface HttpRequestEntry {
33 : /// The id of the request
34 : requestId: number;
35 : /// The method
36 : method: string;
37 : /// The url
38 : url: string;
39 : /// The request headers
40 : requestHeaders: [string, string][];
41 : /// The request body
42 : requestBody: HttpBodyRecord | null;
43 : /// When the request was sent
44 : sentAt: Date;
45 : /// When the response headers arrived
46 : respondedAt: Date | null;
47 : /// When the response body was read
48 : bodyReceivedAt: Date | null;
49 : /// When the request failed
50 : failedAt: Date | null;
51 : /// The response status
52 : status: number | null;
53 : /// The response status text
54 : statusText: string | null;
55 : /// The response headers
56 : responseHeaders: [string, string][] | null;
57 : /// The start of the response body, read when the caller reads it, error bodies are read right away
58 : responseBody: HttpBodyRecord | null;
59 : /// The error of a failed request
60 : error: string | null;
61 : }
62 :
63 : /// The outgoing http requests of the app
64 1 : export class HttpRequestLog {
65 1 : private readonly entries = new Map<number, HttpRequestEntry>();
66 1 : private snapshot: readonly HttpRequestEntry[] = [];
67 1 : private readonly listeners = new Set<() => void>();
68 1 : private nextRequestId = 1;
69 1 : private readonly limit: number;
70 :
71 1 : constructor(limit: number = HTTP_REQUEST_LOG_LIMIT) {
72 1 : this.limit = limit;
73 1 : }
74 :
75 : /// The requests, oldest first
76 1 : readonly getSnapshot = (): readonly HttpRequestEntry[] => this.snapshot;
77 :
78 : /// Subscribe to changes
79 1 : subscribe = (listener: () => void): (() => void) => {
80 1 : this.listeners.add(listener);
81 1 : return () => this.listeners.delete(listener);
82 1 : };
83 :
84 : /// Record a sent request
85 1 : add(entry: Omit<HttpRequestEntry, 'requestId'>): number {
86 1 : const requestId = this.nextRequestId++;
87 1 : this.entries.set(requestId, { ...entry, requestId });
88 : // Maps iterate in insertion order, the first key is the oldest request
89 1 : while (this.entries.size > this.limit) {
90 1 : this.entries.delete(this.entries.keys().next().value!);
91 : }
92 1 : this.publish();
93 1 : return requestId;
94 1 : }
95 :
96 : /// Update a request, a dropped request is ignored
97 1 : update(requestId: number, patch: Partial<Omit<HttpRequestEntry, 'requestId'>>): void {
98 1 : const entry = this.entries.get(requestId);
99 1 : if (entry == null) return;
100 1 : this.entries.set(requestId, { ...entry, ...patch });
101 1 : this.publish();
102 1 : }
103 :
104 : /// Drop all requests
105 1 : clear(): void {
106 1 : this.entries.clear();
107 1 : this.publish();
108 1 : }
109 :
110 1 : private publish(): void {
111 1 : this.snapshot = [...this.entries.values()];
112 1 : for (const listener of this.listeners) listener();
113 1 : }
114 1 : }
115 :
116 : /// An http client that records its requests in a log
117 1 : export class RecordingHttpClient implements HttpClient {
118 1 : constructor(private readonly inner: HttpClient, private readonly log: HttpRequestLog) { }
119 :
120 1 : async fetch(input: URL | Request | string, init?: RequestInit & ClientOptions): Promise<HttpFetchResult> {
121 1 : const request = input instanceof Request ? input : null;
122 1 : const headers = new Headers(request?.headers);
123 1 : new Headers(init?.headers).forEach((value, name) => headers.set(name, value));
124 : // The content type that fetch derives from the body
125 1 : const bodyContentType = init?.body != null ? implicitContentType(init.body) : null;
126 1 : if (bodyContentType != null && !headers.has('content-type')) headers.set('content-type', bodyContentType);
127 1 : const requestId = this.log.add({
128 1 : method: (init?.method ?? request?.method ?? 'GET').toUpperCase(),
129 1 : url: request?.url ?? input.toString(),
130 1 : requestHeaders: [...headers.entries()],
131 1 : requestBody: init?.body != null
132 1 : ? recordRequestBody(init.body, headers.get('content-type'))
133 1 : : request?.body != null ? { kind: 'opaque' } : null,
134 1 : sentAt: new Date(),
135 1 : respondedAt: null,
136 1 : bodyReceivedAt: null,
137 1 : failedAt: null,
138 1 : status: null,
139 1 : statusText: null,
140 1 : responseHeaders: null,
141 1 : responseBody: null,
142 1 : error: null,
143 1 : });
144 1 : let response: HttpFetchResult;
145 1 : try {
146 1 : response = await this.inner.fetch(input, init);
147 1 : } catch (e: unknown) {
148 1 : this.log.update(requestId, { failedAt: new Date(), error: stringifyError(e) });
149 1 : throw e;
150 1 : }
151 1 : this.log.update(requestId, {
152 1 : respondedAt: new Date(),
153 1 : status: response.status,
154 1 : statusText: response.statusText,
155 1 : responseHeaders: [...response.headers.entries()],
156 1 : });
157 1 : return new RecordedHttpFetchResult(response, this.log, requestId);
158 1 : }
159 1 : }
160 :
161 : /// A response that records its body when the caller reads it
162 1 : class RecordedHttpFetchResult implements HttpFetchResult {
163 : /// The body of an error response, read right away since callers often throw without reading it
164 1 : private readonly bufferedBody: Promise<ArrayBuffer> | null = null;
165 :
166 1 : constructor(private readonly inner: HttpFetchResult, private readonly log: HttpRequestLog, private readonly requestId: number) {
167 1 : if (inner.status >= 400) {
168 1 : this.bufferedBody = this.record(inner.arrayBuffer(), buffer => recordBytes(buffer, this.contentType, HTTP_RESPONSE_PREVIEW_BYTES));
169 : // The caller sees a failed read when it reads the body
170 1 : this.bufferedBody.catch(() => { });
171 1 : }
172 1 : }
173 :
174 0 : get headers(): Headers { return this.inner.headers; }
175 1 : get status(): number { return this.inner.status; }
176 0 : get statusText(): string { return this.inner.statusText; }
177 1 : private get contentType(): string | null { return this.inner.headers.get('content-type'); }
178 :
179 1 : arrayBuffer(): Promise<ArrayBuffer> {
180 1 : return this.bufferedBody ?? this.record(this.inner.arrayBuffer(), buffer => recordBytes(buffer, this.contentType, HTTP_RESPONSE_PREVIEW_BYTES));
181 1 : }
182 :
183 1 : async text(): Promise<string> {
184 1 : if (this.bufferedBody != null) return new TextDecoder().decode(await this.bufferedBody);
185 1 : return this.record(this.inner.text(), text => recordText(text, HTTP_RESPONSE_PREVIEW_BYTES));
186 1 : }
187 :
188 1 : async json(): Promise<any> {
189 1 : if (this.bufferedBody != null) {
190 1 : const text = new TextDecoder().decode(await this.bufferedBody);
191 1 : return text == '' ? {} : JSON.parse(text);
192 1 : }
193 : // Parsed bodies are not stringified again, they can be large
194 0 : return this.record(this.inner.json(), () => ({ kind: 'parsed' }));
195 1 : }
196 :
197 : /// Record the body once it is read
198 1 : private record<T>(body: Promise<T>, toRecord: (body: T) => HttpBodyRecord): Promise<T> {
199 1 : return body.then(
200 1 : value => {
201 1 : this.log.update(this.requestId, { bodyReceivedAt: new Date(), responseBody: toRecord(value) });
202 1 : return value;
203 1 : },
204 0 : (e: unknown) => {
205 0 : this.log.update(this.requestId, { failedAt: new Date(), error: stringifyError(e) });
206 0 : throw e;
207 0 : },
208 1 : );
209 1 : }
210 1 : }
211 :
212 : /// Is a body of this content type text?
213 1 : function isTextContentType(contentType: string | null): boolean {
214 1 : if (contentType == null) return false;
215 1 : const mime = contentType.split(';')[0].trim().toLowerCase();
216 1 : return mime.startsWith('text/')
217 1 : || mime == 'application/json'
218 1 : || mime.endsWith('+json')
219 1 : || mime == 'application/xml'
220 1 : || mime.endsWith('+xml')
221 1 : || mime == 'application/x-www-form-urlencoded'
222 1 : || mime == 'application/javascript'
223 1 : || mime == 'application/graphql';
224 1 : }
225 :
226 : /// Record a text body, keep at most `limit` characters
227 1 : function recordText(text: string, limit: number): HttpBodyRecord {
228 1 : const byteLength = new TextEncoder().encode(text).byteLength;
229 1 : return { kind: 'text', text: text.length > limit ? text.slice(0, limit) : text, byteLength, truncated: text.length > limit };
230 1 : }
231 :
232 : /// Record a byte body, it is text if the content type says so
233 1 : function recordBytes(bytes: ArrayBuffer | ArrayBufferView, contentType: string | null, limit: number): HttpBodyRecord {
234 1 : const view = bytes instanceof ArrayBuffer
235 1 : ? new Uint8Array(bytes)
236 1 : : new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
237 : // An empty body without content type is no body
238 1 : if (!isTextContentType(contentType) && view.byteLength > 0) return { kind: 'binary', byteLength: view.byteLength };
239 1 : const truncated = view.byteLength > limit;
240 : // A cut through a multi-byte character decodes as a replacement character
241 1 : const text = new TextDecoder().decode(truncated ? view.subarray(0, limit) : view);
242 1 : return { kind: 'text', text, byteLength: view.byteLength, truncated };
243 1 : }
244 :
245 : /// Record a request body, a body is text unless it is binary data of a content type that is not text
246 1 : function recordRequestBody(body: BodyInit, contentType: string | null): HttpBodyRecord {
247 1 : if (typeof body == 'string') return recordText(body, HTTP_REQUEST_BODY_BYTES);
248 1 : if (body instanceof URLSearchParams) return recordText(body.toString(), HTTP_REQUEST_BODY_BYTES);
249 1 : if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) return recordBytes(body, contentType, HTTP_REQUEST_BODY_BYTES);
250 0 : if (body instanceof Blob) return { kind: 'binary', byteLength: body.size };
251 0 : return { kind: 'opaque' };
252 1 : }
253 :
254 : /// The content type that fetch sends for a body without content type header
255 1 : function implicitContentType(body: BodyInit): string | null {
256 1 : if (typeof body == 'string') return 'text/plain;charset=UTF-8';
257 1 : if (body instanceof URLSearchParams) return 'application/x-www-form-urlencoded;charset=UTF-8';
258 1 : if (body instanceof Blob && body.type != '') return body.type;
259 1 : return null;
260 1 : }
261 :
262 : /// Is the value of a header masked in the viewer?
263 1 : export function isSensitiveHeader(name: string): boolean {
264 1 : return SENSITIVE_HEADERS.has(name.toLowerCase());
265 1 : }
266 :
267 : /// Mask the values of sensitive headers
268 1 : export function maskHeaders(headers: [string, string][]): [string, string][] {
269 1 : return headers.map(([name, value]) => [name, isSensitiveHeader(name) ? '••••••••' : value]);
270 1 : }
271 :
272 : /// Quote a word for POSIX shells
273 1 : function shellQuote(word: string): string {
274 1 : return `'${word.replace(/'/g, `'\\''`)}'`;
275 1 : }
276 :
277 : /// Format a request as a curl command. The command includes the values of sensitive headers.
278 1 : export function formatCurlCommand(entry: HttpRequestEntry): string {
279 1 : const notes: string[] = [];
280 1 : const args: string[] = [`curl ${shellQuote(entry.url)}`];
281 1 : const body = entry.requestBody;
282 : // curl sends a POST when it has data
283 1 : if (entry.method != 'GET' && !(entry.method == 'POST' && body?.kind == 'text')) {
284 1 : args.push(`-X ${entry.method}`);
285 : }
286 1 : for (const [name, value] of entry.requestHeaders) {
287 1 : args.push(`-H ${shellQuote(`${name}: ${value}`)}`);
288 : }
289 1 : switch (body?.kind) {
290 1 : case 'text':
291 1 : if (body.truncated) notes.push(`# The request body is cut to ${body.text.length} of ${body.byteLength} bytes`);
292 1 : args.push(`--data-raw ${shellQuote(body.text)}`);
293 1 : break;
294 1 : case 'binary':
295 1 : notes.push(`# The binary request body of ${body.byteLength} bytes is left out`);
296 1 : break;
297 0 : case 'opaque':
298 0 : notes.push('# The request body is left out, the log could not read it');
299 : break;
300 1 : }
301 1 : return [...notes, args.join(' \\\n ')].join('\n');
302 1 : }
|
