fix: add FetchProxy
This will let us intercept fetch requests (until now we're only proxying
XMLHttpRequest), in order to fix the issues we're experiencing with some
features.
Bug: twpowertools:229
Change-Id: I277473c05479ca39bb6183a51855382124890bde
diff --git a/src/xhrInterceptor/fetchProxy/FetchBody.ts b/src/xhrInterceptor/fetchProxy/FetchBody.ts
new file mode 100644
index 0000000..a178009
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchBody.ts
@@ -0,0 +1,12 @@
+export default class FetchBody {
+ constructor(private body: RequestInit['body'] | undefined) {}
+
+ async getJSONRequestBody() {
+ if (!this.body) return undefined;
+
+ // Using Response is a hack, but it works and converts a possibly long code
+ // into a one-liner :D
+ // Source of inspiration: https://stackoverflow.com/a/72718732
+ return await new Response(this.body).json();
+ }
+}
diff --git a/src/xhrInterceptor/fetchProxy/FetchHeaders.test.ts b/src/xhrInterceptor/fetchProxy/FetchHeaders.test.ts
new file mode 100644
index 0000000..c075a2d
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchHeaders.test.ts
@@ -0,0 +1,88 @@
+/**
+ * @jest-environment ./src/xhrInterceptor/fetchProxy/__environments__/fetchEnvironment.ts
+ */
+
+import { describe, expect, it } from '@jest/globals';
+import FetchHeaders from './FetchHeaders';
+
+describe('FetchHeaders', () => {
+ const dummyHeaderName = 'X-Foo';
+ const dummyHeaderValue = 'bar';
+ const dummyHeadersPerFormat: { description: string; value: HeadersInit }[] = [
+ {
+ description: 'a Headers instance',
+ value: new Headers({ [dummyHeaderName]: dummyHeaderValue }),
+ },
+ {
+ description: 'an object',
+ value: { [dummyHeaderName]: dummyHeaderValue },
+ },
+ {
+ description: 'an array',
+ value: [[dummyHeaderName, dummyHeaderValue]],
+ },
+ ];
+
+ describe.each(dummyHeadersPerFormat)(
+ 'when the headers are presented as $description',
+ ({ value }) => {
+ describe('hasValue', () => {
+ it('should return true when the header with the provided value is present', () => {
+ const sut = new FetchHeaders(value);
+ const result = sut.hasValue(dummyHeaderName, dummyHeaderValue);
+
+ expect(result).toBe(true);
+ });
+
+ it("should return true when a header with the provided value is present even if the provided name doesn't have the same case", () => {
+ const sut = new FetchHeaders(value);
+ const result = sut.hasValue('x-FoO', dummyHeaderValue);
+
+ expect(result).toBe(true);
+ });
+
+ it('should return false when a header with the provided value is not present', () => {
+ const sut = new FetchHeaders(value);
+ const result = sut.hasValue('X-NonExistent', dummyHeaderValue);
+
+ expect(result).toBe(false);
+ });
+ });
+
+ describe('removeHeader', () => {
+ it('should remove the header if it is present', () => {
+ const sut = new FetchHeaders(value);
+ sut.removeHeader(dummyHeaderName);
+
+ if (value instanceof Headers) {
+ expect(value.has(dummyHeaderName)).toBe(false);
+ } else if (Array.isArray(value)) {
+ expect(value).not.toContain(
+ expect.arrayContaining([dummyHeaderName]),
+ );
+ } else {
+ expect(value).not.toHaveProperty(dummyHeaderName);
+ }
+ });
+ });
+ },
+ );
+
+ describe('when the headers are presented as undefined', () => {
+ describe('hasValue', () => {
+ it('should return false', () => {
+ const sut = new FetchHeaders(undefined);
+ const result = sut.hasValue(dummyHeaderName, dummyHeaderValue);
+
+ expect(result).toBe(false);
+ });
+ });
+
+ describe('removeHeader', () => {
+ it('should not throw', () => {
+ const sut = new FetchHeaders(undefined);
+ expect(() => sut.removeHeader(dummyHeaderName)).not.toThrow();
+ });
+ });
+ });
+});
diff --git a/src/xhrInterceptor/fetchProxy/FetchHeaders.ts b/src/xhrInterceptor/fetchProxy/FetchHeaders.ts
new file mode 100644
index 0000000..4968d01
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchHeaders.ts
@@ -0,0 +1,47 @@
+export default class FetchHeaders {
+ constructor(private headers: HeadersInit | undefined) {}
+
+ hasValue(name: string, value: string) {
+ if (!this.headers) {
+ return false;
+ } else if (this.headers instanceof Headers) {
+ return this.headers.get(name) == value;
+ } else {
+ const headersArray = Array.isArray(this.headers)
+ ? this.headers
+ : Object.entries(this.headers);
+ return headersArray.some(
+ ([candidateHeaderName, candidateValue]) =>
+ this.isEqualCaseInsensitive(candidateHeaderName, name) &&
+ candidateValue === value,
+ );
+ }
+ }
+
+ removeHeader(name: string) {
+ if (!this.headers) {
+ return;
+ } else if (this.headers instanceof Headers) {
+ this.headers.delete(name);
+ } else if (Array.isArray(this.headers)) {
+ const index = this.headers.findIndex(([candidateHeaderName]) =>
+ this.isEqualCaseInsensitive(candidateHeaderName, name),
+ );
+ if (index !== -1) delete this.headers[index];
+ } else {
+ const headerNames = Object.keys(this.headers);
+ const headerName = headerNames.find((candidateName) =>
+ this.isEqualCaseInsensitive(candidateName, name),
+ );
+ if (headerName) delete this.headers[headerName];
+ }
+ }
+
+ private isEqualCaseInsensitive(a: string, b: string) {
+ return (
+ a.localeCompare(b, undefined, {
+ sensitivity: 'accent',
+ }) == 0
+ );
+ }
+}
diff --git a/src/xhrInterceptor/fetchProxy/FetchInput.test.ts b/src/xhrInterceptor/fetchProxy/FetchInput.test.ts
new file mode 100644
index 0000000..3c20b5b
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchInput.test.ts
@@ -0,0 +1,39 @@
+/**
+ * @jest-environment ./src/xhrInterceptor/fetchProxy/__environments__/fetchEnvironment.ts
+ */
+
+import { describe, expect, it } from '@jest/globals';
+import FetchInput from './FetchInput';
+
+describe('FetchInput', () => {
+ describe('getUrl', () => {
+ const urlString = 'https://example.avm99963.com/';
+
+ it('should return input when input is already a string', () => {
+ const dummyInput = urlString;
+
+ const sut = new FetchInput(dummyInput);
+ const result = sut.getUrl();
+
+ expect(result).toBe(urlString);
+ });
+
+ it('should return stringified url when input is a URL instance', () => {
+ const dummyInput = new URL(urlString);
+
+ const sut = new FetchInput(dummyInput);
+ const result = sut.getUrl();
+
+ expect(result).toBe(urlString);
+ });
+
+ it('should return url string when input is a Request instance', () => {
+ const dummyInput = new Request(urlString);
+
+ const sut = new FetchInput(dummyInput);
+ const result = sut.getUrl();
+
+ expect(result).toBe(urlString);
+ });
+ });
+});
diff --git a/src/xhrInterceptor/fetchProxy/FetchInput.ts b/src/xhrInterceptor/fetchProxy/FetchInput.ts
new file mode 100644
index 0000000..75be8b8
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchInput.ts
@@ -0,0 +1,16 @@
+export default class FetchInput {
+ constructor(private input: RequestInfo | URL) {}
+
+ /**
+ * Returns the fetch input URL as a string.
+ */
+ getUrl() {
+ if (typeof this.input === 'string') {
+ return this.input;
+ } else if (this.input instanceof URL) {
+ return this.input.toString();
+ } else {
+ return this.input.url;
+ }
+ }
+}
diff --git a/src/xhrInterceptor/fetchProxy/FetchProxy.test.ts b/src/xhrInterceptor/fetchProxy/FetchProxy.test.ts
new file mode 100644
index 0000000..e686535
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchProxy.test.ts
@@ -0,0 +1,210 @@
+/**
+ * @jest-environment ./src/xhrInterceptor/fetchProxy/__environments__/fetchEnvironment.ts
+ */
+
+import { beforeEach, describe, expect, it, jest } from '@jest/globals';
+import FetchProxy from './FetchProxy';
+import MessageIdTracker from '../MessageIdTracker';
+import { ResponseModifierPort } from '../ResponseModifier.port';
+import {
+ InterceptorHandlerMock,
+ matchInterceptorsMock,
+ triggerEventMock,
+} from '../interceptors/__mocks__/InterceptorHandler.mock';
+import {
+ Interceptor,
+ InterceptorFilter,
+} from '../interceptors/InterceptorHandler.port';
+
+jest.mock('../interceptors/InterceptorHandler.adapter');
+
+const interceptMock = jest.fn<ResponseModifierPort['intercept']>();
+class MockResponseModifier implements ResponseModifierPort {
+ async intercept(
+ ...args: Parameters<ResponseModifierPort['intercept']>
+ ): ReturnType<ResponseModifierPort['intercept']> {
+ return interceptMock(...args);
+ }
+}
+
+const fetchMock = jest.fn<typeof window.fetch>();
+
+const consoleErrorMock = jest.spyOn(global.console, 'error');
+
+const dummyResponse = new Response('{}', { status: 200 });
+beforeEach(() => {
+ jest.resetAllMocks();
+
+ window.fetch = fetchMock;
+
+ // Sensible defaults which can be overriden in each test.
+ fetchMock.mockResolvedValue(dummyResponse);
+ interceptMock.mockResolvedValue({ wasModified: false });
+ matchInterceptorsMock.mockReturnValue([]);
+ consoleErrorMock.mockImplementation(() => {});
+});
+
+describe('FetchProxy', () => {
+ describe('when calling fetch after enabling interception', () => {
+ const dummyUrl: string = 'https://dummy.avm99963.com/';
+ const dummyInit: RequestInit = {
+ body: '{"1":"request"}',
+ };
+ const dummyInitProtoJs = {
+ body: '["dummy"]',
+ headers: {
+ 'Content-Type': 'application/json+protobuf',
+ },
+ };
+
+ let fetchProxy: FetchProxy;
+ beforeEach(() => {
+ fetchProxy = new FetchProxy(
+ new MockResponseModifier(),
+ new InterceptorHandlerMock(),
+ new MessageIdTracker(),
+ );
+ fetchProxy.enableInterception();
+ });
+
+ it('should call originalFetch with the original arguments', async () => {
+ await window.fetch(dummyUrl, dummyInit);
+
+ expect(fetchMock).toBeCalledTimes(1);
+ expect(fetchMock).toBeCalledWith(dummyUrl, dummyInit);
+ });
+
+ it('should remove the X-Client header before passing the request to fetch', async () => {
+ await window.fetch(dummyUrl, {
+ ...dummyInit,
+ headers: {
+ 'X-Client': 'twpt',
+ },
+ });
+
+ expect(fetchMock).toBeCalledTimes(1);
+ expect(fetchMock.mock.calls[0][1].headers).toBeDefined();
+ expect(fetchMock.mock.calls[0][1].headers).not.toHaveProperty('X-Client');
+ });
+
+ describe.each(['request', 'response'])(
+ 'regarding %s interceptors',
+ (interceptorFilter: InterceptorFilter) => {
+ describe('when no interceptors match', () => {
+ it(`should not send a ${interceptorFilter} interceptor event`, async () => {
+ matchInterceptorsMock.mockReturnValue([]);
+
+ await window.fetch(dummyUrl, dummyInit);
+
+ expect(triggerEventMock).toHaveBeenCalledTimes(0);
+ });
+ });
+
+ describe('when an interceptor matches', () => {
+ const dummyInterceptor: Interceptor = {
+ eventName: 'dummy_event',
+ urlRegex: /.*/,
+ intercepts: interceptorFilter,
+ };
+ const dummyResponse = { 42: 'response' };
+ beforeEach(() => {
+ matchInterceptorsMock.mockImplementation(
+ (filter: InterceptorFilter) =>
+ filter === interceptorFilter ? [dummyInterceptor] : [],
+ );
+ fetchMock.mockResolvedValue(
+ new Response(JSON.stringify(dummyResponse)),
+ );
+ });
+
+ it(`should send a ${interceptorFilter} interceptor event`, async () => {
+ await window.fetch(dummyUrl, dummyInit);
+
+ expect(triggerEventMock).toHaveBeenCalledTimes(1);
+ const expectedBody =
+ interceptorFilter === 'request'
+ ? { 1: 'request' }
+ : dummyResponse;
+ expect(triggerEventMock).toHaveBeenCalledWith(
+ dummyInterceptor.eventName,
+ expectedBody,
+ expect.anything(),
+ );
+ });
+
+ it(`should send a ${interceptorFilter} interceptor event with normalized protobuf when the request is application/json+protobuf`, async () => {
+ fetchMock.mockResolvedValue(new Response('["dummy"]'));
+
+ await window.fetch(dummyUrl, dummyInitProtoJs);
+
+ expect(triggerEventMock).toHaveBeenCalledTimes(1);
+ const eventBody = triggerEventMock.mock.calls[0][1];
+ expect(eventBody[1]).toBe('dummy');
+ });
+
+ it(`should not reject when triggering the event fails`, async () => {
+ triggerEventMock.mockImplementation(() => {
+ throw new Error('dummy error');
+ });
+
+ expect(await window.fetch(dummyUrl, dummyInit)).resolves;
+ });
+
+ // TODO: add test to ensure something is logged when the previous condition happens
+ });
+ },
+ );
+
+ describe('regarding response modifiers', () => {
+ const dummyModifiedResponse = { 99: 'modified' };
+ beforeEach(() => {
+ interceptMock.mockResolvedValue({
+ wasModified: true,
+ modifiedResponse: dummyModifiedResponse,
+ });
+ });
+
+ it('should pass the intercepted response to ResponseModifier', async () => {
+ const dummyResponse = { 1: 'request' };
+ fetchMock.mockResolvedValue(
+ new Response(JSON.stringify(dummyResponse)),
+ );
+
+ await window.fetch(dummyUrl, dummyInit);
+
+ expect(interceptMock).toHaveBeenCalledTimes(1);
+ expect(interceptMock).toHaveBeenCalledWith({
+ url: dummyUrl,
+ originalResponse: dummyResponse,
+ });
+ });
+
+ it('should return the modified response when ResponseModifier modifies it', async () => {
+ const response = await window.fetch(dummyUrl, dummyInit);
+ const result = await response.json();
+
+ expect(result).toEqual(dummyModifiedResponse);
+ });
+
+ it('should not reject when ResponseModifier throws an error', async () => {
+ interceptMock.mockImplementation(() => {
+ throw new Error('dummy error');
+ });
+
+ expect(await window.fetch(dummyUrl, dummyInit)).resolves;
+ });
+ });
+
+ it('should not reject when a body is not passed', async () => {
+ matchInterceptorsMock.mockImplementation((filter: InterceptorFilter) => [
+ {
+ eventName: 'dummy_event',
+ urlRegex: /.*/,
+ intercepts: filter,
+ },
+ ]);
+
+ expect(await window.fetch(dummyUrl)).resolves;
+ });
+ });
+});
diff --git a/src/xhrInterceptor/fetchProxy/FetchProxy.ts b/src/xhrInterceptor/fetchProxy/FetchProxy.ts
new file mode 100644
index 0000000..71031c5
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchProxy.ts
@@ -0,0 +1,40 @@
+import { InterceptorHandlerPort } from '../interceptors/InterceptorHandler.port';
+import MessageIdTracker from '../MessageIdTracker';
+import { ResponseModifierPort } from '../ResponseModifier.port';
+import FetchProxyCallHandler from './FetchProxyCallHandler';
+
+/**
+ * Class which lets us override window.fetch to proxy the requests through our
+ * internal interceptors to read/modify requests/responses.
+ */
+export default class FetchProxy {
+ private originalFetch: typeof window.fetch;
+ private isInterceptEnabled = false;
+
+ constructor(
+ private responseModifier: ResponseModifierPort,
+ private interceptorHandler: InterceptorHandlerPort,
+ private messageIdTracker: MessageIdTracker,
+ ) {}
+
+ enableInterception() {
+ if (this.isInterceptEnabled) return;
+
+ this.isInterceptEnabled = true;
+
+ this.originalFetch = window.fetch;
+ this.overrideFetch();
+ }
+
+ private overrideFetch() {
+ window.fetch = async (...args) => {
+ const fetchProxyCallhandler = new FetchProxyCallHandler(
+ this.responseModifier,
+ this.interceptorHandler,
+ this.messageIdTracker,
+ this.originalFetch,
+ );
+ return await fetchProxyCallhandler.proxiedFetch(...args);
+ };
+ }
+}
diff --git a/src/xhrInterceptor/fetchProxy/FetchProxyCallHandler.ts b/src/xhrInterceptor/fetchProxy/FetchProxyCallHandler.ts
new file mode 100644
index 0000000..bba616b
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/FetchProxyCallHandler.ts
@@ -0,0 +1,179 @@
+import { XClientHeader, XClientValue } from '../../common/api';
+import {
+ correctArrayKeys,
+ inverseCorrectArrayKeys,
+} from '../../common/protojs';
+import { InterceptorHandlerPort } from '../interceptors/InterceptorHandler.port';
+import MessageIdTracker from '../MessageIdTracker';
+import { ResponseModifierPort } from '../ResponseModifier.port';
+import FetchBody from './FetchBody';
+import FetchHeaders from './FetchHeaders';
+import FetchInput from './FetchInput';
+
+export default class FetchProxyCallHandler {
+ private fetchHeaders: FetchHeaders;
+ private fetchBody: FetchBody;
+ private fetchInput: FetchInput;
+
+ private messageId: number;
+ private url: string;
+ private isArrayProto: boolean;
+
+ constructor(
+ private responseModifier: ResponseModifierPort,
+ private interceptorHandler: InterceptorHandlerPort,
+ private messageIdTracker: MessageIdTracker,
+ private originalFetch: typeof window.fetch,
+ ) {}
+
+ async proxiedFetch(
+ input: RequestInfo | URL,
+ init?: RequestInit,
+ ): Promise<Response> {
+ this.fetchHeaders = new FetchHeaders(init?.headers);
+ this.fetchBody = new FetchBody(init?.body);
+ this.fetchInput = new FetchInput(input);
+
+ const shouldIgnore = this.fetchHeaders.hasValue(
+ XClientHeader,
+ XClientValue,
+ );
+
+ // Remove the header after being read to preserve user privacy.
+ //
+ // If you're a Googler/TW team member reading this, and would like us to
+ // send this header to the server (e.g. for analytics purposes), please
+ // feel free to contact us (the community) at twpowertools-discuss [at]
+ // googlegroups.com!
+ this.fetchHeaders.removeHeader(XClientHeader);
+
+ if (shouldIgnore) {
+ return await this.originalFetch.apply(global, [input, init]);
+ }
+
+ this.messageId = this.messageIdTracker.getNewId();
+ this.url = this.fetchInput.getUrl();
+ this.isArrayProto = this.fetchHeaders.hasValue(
+ 'Content-Type',
+ 'application/json+protobuf',
+ );
+
+ await this.attemptToSendRequestInterceptorEvent();
+
+ const originalResponse: Response = await this.originalFetch.apply(global, [
+ input,
+ init,
+ ]);
+
+ const response = await this.attemptToModifyResponse(originalResponse);
+
+ await this.attemptToSendResponseInterceptorEvent(response);
+
+ return response;
+ }
+
+ private async attemptToSendRequestInterceptorEvent() {
+ try {
+ await this.sendRequestInterceptorEvent();
+ } catch (e) {
+ console.error(
+ `[FetchProxy] An error ocurred sending a request interceptor event for ${this.url}:`,
+ e,
+ );
+ }
+ }
+
+ private async sendRequestInterceptorEvent() {
+ const interceptors = this.interceptorHandler.matchInterceptors(
+ 'request',
+ this.url,
+ );
+ if (interceptors.length === 0) return;
+
+ const rawBody = await this.fetchBody.getJSONRequestBody();
+ if (!rawBody) return;
+
+ const body = this.isArrayProto ? correctArrayKeys(rawBody) : rawBody;
+
+ for (const interceptor of interceptors) {
+ this.interceptorHandler.triggerEvent(
+ interceptor.eventName,
+ body,
+ this.messageId,
+ );
+ }
+ }
+
+ private async attemptToModifyResponse(
+ originalResponse: Response,
+ ): Promise<Response> {
+ try {
+ return await this.modifyResponse(originalResponse);
+ } catch (e) {
+ console.error(
+ `[Fetch Proxy] Couldn\'t modify the response for ${this.url}`,
+ e,
+ );
+ return originalResponse;
+ }
+ }
+
+ private async modifyResponse(originalResponse: Response) {
+ const response = originalResponse.clone();
+ let json = await response.json();
+
+ if (this.isArrayProto) {
+ correctArrayKeys(json);
+ }
+
+ const result = await this.responseModifier.intercept({
+ originalResponse: json,
+ url: this.url,
+ });
+ if (result.wasModified) {
+ json = result.modifiedResponse;
+ }
+
+ if (this.isArrayProto) {
+ inverseCorrectArrayKeys(json);
+ }
+
+ return new Response(JSON.stringify(json), {
+ status: response.status,
+ statusText: response.statusText,
+ headers: response.headers,
+ });
+ }
+
+ private async attemptToSendResponseInterceptorEvent(response: Response) {
+ try {
+ await this.sendResponseInterceptorEvent(response);
+ } catch (e) {
+ console.error(
+ `[FetchProxy] An error ocurred sending a response interceptor event for ${this.url}:`,
+ e,
+ );
+ }
+ }
+
+ private async sendResponseInterceptorEvent(response: Response) {
+ const interceptors = this.interceptorHandler.matchInterceptors(
+ 'response',
+ this.url,
+ );
+ if (interceptors.length === 0) return;
+
+ const rawBody = await response.clone().json();
+ if (!rawBody) return;
+
+ const body = this.isArrayProto ? correctArrayKeys(rawBody) : rawBody;
+
+ for (const interceptor of interceptors) {
+ this.interceptorHandler.triggerEvent(
+ interceptor.eventName,
+ body,
+ this.messageId,
+ );
+ }
+ }
+}
diff --git a/src/xhrInterceptor/fetchProxy/__environments__/fetchEnvironment.ts b/src/xhrInterceptor/fetchProxy/__environments__/fetchEnvironment.ts
new file mode 100644
index 0000000..0e70bf7
--- /dev/null
+++ b/src/xhrInterceptor/fetchProxy/__environments__/fetchEnvironment.ts
@@ -0,0 +1,14 @@
+import JSDOMEnvironment from 'jest-environment-jsdom';
+
+// https://github.com/facebook/jest/blob/v29.4.3/website/versioned_docs/version-29.4/Configuration.md#testenvironment-string
+export default class FixJSDOMEnvironment extends JSDOMEnvironment {
+ constructor(...args: ConstructorParameters<typeof JSDOMEnvironment>) {
+ super(...args);
+
+ // TODO: Remove this (and the environment) once https://github.com/jsdom/jsdom/issues/1724 is closed.
+ this.global.fetch = fetch;
+ this.global.Headers = Headers;
+ this.global.Request = Request;
+ this.global.Response = Response;
+ }
+}