feat(trace viewer): Extending existing NetworkTab view (#5009)

feat(trace viewer): Extending existing NetworkTab view

Currently the network tab contains a limited amount of information on the resources that were loaded in the browser. This change proposes extending the details displayed for each resource, to include:

- HTTP method,
- Full url,
- Easily visible response content type,
- Request headers,
- Request & response bodies.

Such level of information could help quickly understand what happened in the application, when it was communicating with backend services. This can help debug tests quicker to figure out why they are failing.

This implementation still needs some clean up & tests improvement, but I wanted to propose such changes and gather your feedback before going too far.

src/cli/traceViewer/snapshotRouter.ts

@@ -169,7 +169,7 @@ export class SnapshotRouter {
   }
 
   private async _readResource(event: trace.NetworkResourceTraceEvent, overrideSha1: string | undefined) {
-    const body = await this._readSha1(overrideSha1 || event.sha1);
+    const body = await this._readSha1(overrideSha1 || event.responseSha1);
     if (!body)
       return;
     return {

src/cli/traceViewer/traceViewer.ts

@@ -92,6 +92,12 @@ class TraceViewer {
     await uiPage.exposeBinding('readFile', async (_, path: string) => {
       return fs.readFileSync(path).toString();
     });
+    await uiPage.exposeBinding('readResource', async (_, sha1: string) => {
+      if (!this._document)
+        return;
+
+      return fs.readFileSync(path.join(this._document.resourcesDir, sha1)).toString('base64');
+    });
     await uiPage.exposeBinding('renderSnapshot', async (_, arg: { action: ActionTraceEvent, snapshot: { name: string, snapshotId?: string } }) => {
       const { action, snapshot } = arg;
       if (!this._document)

src/cli/traceViewer/web/index.tsx

@@ -25,6 +25,7 @@ declare global {
   interface Window {
     getTraceModel(): Promise<TraceModel>;
     readFile(filePath: string): Promise<string>;
+    readResource(sha1: string): Promise<string>;
     renderSnapshot(arg: { action: trace.ActionTraceEvent, snapshot: { name: string, snapshotId?: string } }): void;
   }
 }

src/cli/traceViewer/web/styles.tsx

@@ -26,6 +26,7 @@ export const GlobalStyles = () => <style>{`
   --green: #4CAF50;
   --purple: #9C27B0;
   --yellow: #FFC107;
+  --white: #FFFFFF;
   --blue: #2196F3;
   --transparent-blue: #2196F355;
   --orange: #d24726;
@@ -40,6 +41,7 @@ export const GlobalStyles = () => <style>{`
   --settings: #E7E7E7;
   --sidebar-width: 250px;
   --light-pink: #ff69b460;
+  --network-content-bg: #dcdcdb;
   --box-shadow: rgba(0, 0, 0, 0.133) 0px 1.6px 3.6px 0px, rgba(0, 0, 0, 0.11) 0px 0.3px 0.9px 0px;
 }
 

src/cli/traceViewer/web/ui/helpers.tsx

@@ -57,9 +57,10 @@ export function useMeasure<T extends Element>() {
 export const Expandable: React.FunctionComponent<{
   title: JSX.Element,
   body: JSX.Element,
+  setExpanded: Function,
+  expanded: Boolean,
   style?: React.CSSProperties,
-}> = ({ title, body, style }) => {
-  const [expanded, setExpanded] = React.useState(true);
+}> = ({ title, body, setExpanded, expanded, style }) => {
   return <div style={{ ...style, display: 'flex', flexDirection: 'column' }}>
     <div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', whiteSpace: 'nowrap' }}>
       <div

src/cli/traceViewer/web/ui/networkResourceDetails.css

@@ -0,0 +1,112 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+.network-request {
+  box-shadow: var(--box-shadow);
+  white-space: nowrap;
+  display: flex;
+  align-items: center;
+  padding: 0 10px;
+  margin-bottom: 10px;
+  background: #fdfcfc;
+  width: 100%;
+  border: 3px solid transparent;
+  flex: none;
+  outline: none;
+}
+
+.network-request.selected,
+.network-request:hover {
+  border-color: var(--inactive-focus-ring);
+}
+
+.network-request.selected:focus {
+  border-color: var(--orange);
+}
+
+.network-request-title {
+  height: 36px;
+  display: flex;
+  align-items: center;
+  flex: 1;
+}
+
+.network-request-title-status {
+  font-weight: bold;
+  height: 100%;
+  display: flex;
+  align-items: center;
+  padding: 0px 5px;
+  margin-right: 5px;
+}
+
+.network-request-title-status.status-success {
+  background-color: var(--green);
+}
+
+.network-request-title-status.status-failure {
+  background-color: var(--red);
+  color: var(--white);
+}
+
+.network-request-title-status.status-neutral {
+  background-color: var(--white);
+}
+
+.network-request-title-method {
+  font-weight: bold;
+}
+
+.network-request-title-url {
+  overflow: hidden;
+  text-overflow: ellipsis;
+  flex: 1;
+}
+
+.network-request-title-content-type {
+  font-weight: bold;
+}
+
+.network-request-details {
+  font-family: var(--monospace-font);
+  width: 100%;
+}
+
+.network-request-details-url {
+  white-space: normal;
+  word-wrap: break-word;
+}
+
+.network-request-headers {
+  white-space: pre;
+  overflow: hidden;
+}
+
+.network-request-body {
+  white-space: pre;
+  overflow: scroll;
+  background-color: var(--network-content-bg);
+  border: black 1px solid;
+  max-height: 500px;
+}
+
+.network-request-response-body {
+  white-space: pre;
+  overflow: scroll;
+  background-color: var(--network-content-bg);
+  border: black 1px solid;
+  max-height: 500px;
+}

src/cli/traceViewer/web/ui/networkResourceDetails.tsx

@@ -0,0 +1,116 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './networkResourceDetails.css';
+import * as React from 'react';
+import { Expandable } from './helpers';
+import { NetworkResourceTraceEvent } from '../../../../trace/traceTypes';
+
+
+export const NetworkResourceDetails: React.FunctionComponent<{
+  resource: NetworkResourceTraceEvent,
+  index: number,
+  selected: boolean,
+  setSelected: React.Dispatch<React.SetStateAction<number>>,
+}> = ({ resource, index, selected, setSelected }) => {
+  const [expanded, setExpanded] = React.useState(false);
+  const [requestBody, setRequestBody] = React.useState<string | null>(null);
+  const [responseBody, setResponseBody] = React.useState<string | null>(null);
+
+  React.useEffect(() => {
+    setExpanded(false);
+    setSelected(-1);
+  }, [resource, setSelected]);
+
+  React.useEffect(() => {
+    const readResources = async  () => {
+      if (resource.requestSha1 !== 'none') {
+        const requestResource = await window.readResource(resource.requestSha1);
+        setRequestBody(requestResource);
+      }
+
+      if (resource.responseSha1 !== 'none') {
+        const responseResource = await window.readResource(resource.responseSha1);
+        setResponseBody(responseResource);
+      }
+    };
+
+    readResources();
+  }, [expanded, resource.responseSha1, resource.requestSha1]);
+
+  function formatBody(body: string | null, contentType: string): string {
+    if (body === null)
+      return 'Loading...';
+
+    const bodyStr = atob(body);
+
+    if (bodyStr === '')
+      return '<Empty>';
+
+    if (contentType.includes('application/json')) {
+      try {
+        return JSON.stringify(JSON.parse(bodyStr), null, 2);
+      } catch (err) {
+        return bodyStr;
+      }
+    }
+
+    if (contentType.includes('application/x-www-form-urlencoded'))
+      return decodeURIComponent(bodyStr);
+
+    return bodyStr;
+  }
+
+  function formatStatus(status: number): string {
+    if (status >= 200 && status < 400)
+      return 'status-success';
+
+    if (status >= 400)
+      return 'status-failure';
+
+    return 'status-neutral';
+  }
+
+  const requestContentTypeHeader = resource.requestHeaders.find(q => q.name === 'Content-Type');
+  const requestContentType = requestContentTypeHeader ? requestContentTypeHeader.value : '';
+
+  return <div
+    className={'network-request ' + (selected ? 'selected' : '')} onClick={() => setSelected(index)}>
+    <Expandable expanded={expanded} setExpanded={setExpanded} style={{ width: '100%' }} title={
+      <div className='network-request-title'>
+        <div className={'network-request-title-status ' + formatStatus(resource.status)}>{resource.status}</div>
+        <div className='network-request-title-method'>{resource.method}: &nbsp;</div>
+        <div className='network-request-title-url'>{resource.url}</div>
+        <div className='network-request-title-content-type'>{resource.contentType}</div>
+      </div>
+    } body={
+      <div className='network-request-details'>
+        <h4>URL</h4>
+        <div className='network-request-details-url'>{resource.url}</div>
+        <h4>Request Headers</h4>
+        <div className='network-request-headers'>{resource.requestHeaders.map(pair => `${pair.name}: ${pair.value}`).join('\n')}</div>
+        <h4>Response Headers</h4>
+        <div className='network-request-headers'>{resource.responseHeaders.map(pair => `${pair.name}: ${pair.value}`).join('\n')}</div>
+        {resource.requestSha1 !== 'none' ? <h4>Request Body</h4> : ''}
+        {resource.requestSha1 !== 'none' ? <div className='network-request-body'>{formatBody(requestBody, requestContentType)}</div> : ''}
+        <h4>Response Body</h4>
+        {resource.responseSha1 === 'none' ? <div className='network-request-response-body'>Response body is not available for this request.</div> : ''}
+        {responseBody !== null && resource.contentType.includes('image') ? <img src={`data:${resource.contentType};base64,${responseBody}`} /> : ''}
+        {responseBody !== null && !resource.contentType.includes('image') ? <div className='network-request-response-body'>{formatBody(responseBody, resource.contentType)}</div> : ''}
+      </div>
+    }/>
+  </div>;
+};

src/cli/traceViewer/web/ui/networkTab.css

@@ -24,45 +24,3 @@
 .network-tab:focus {
   outline: none;
 }
-
-.network-request {
-  box-shadow: var(--box-shadow);
-  white-space: nowrap;
-  display: flex;
-  align-items: center;
-  padding: 0 10px;
-  margin-bottom: 10px;
-  background: #fdfcfc;
-  width: 100%;
-  border: 3px solid transparent;
-  flex: none;
-  outline: none;
-}
-
-.network-request-title {
-  height: 36px;
-  display: flex;
-  align-items: center;
-  overflow: hidden;
-  text-overflow: ellipsis;
-}
-
-.network-request-details {
-  font-family: var(--monospace-font);
-  white-space: pre;
-  overflow: hidden;
-}
-
-.network-request-title > div {
-  overflow: hidden;
-  text-overflow: ellipsis;
-}
-
-.network-request.selected,
-.network-request:hover {
-  border-color: var(--inactive-focus-ring);
-}
-
-.network-request.selected:focus {
-  border-color: var(--orange);
-}

src/cli/traceViewer/web/ui/networkTab.tsx

@@ -17,23 +17,18 @@
 import { ActionEntry } from '../../traceModel';
 import './networkTab.css';
 import * as React from 'react';
-import { Expandable } from './helpers';
+import { NetworkResourceDetails } from './networkResourceDetails';
 
 export const NetworkTab: React.FunctionComponent<{
   actionEntry: ActionEntry | undefined,
 }> = ({ actionEntry }) => {
   const [selected, setSelected] = React.useState(0);
+
   return <div className='network-tab'>{
     (actionEntry ? actionEntry.resources : []).map((resource, index) => {
-      return <div key={index}
-        className={'network-request ' + (index === selected ? 'selected' : '')}
-        onClick={() => setSelected(index)}>
-        <Expandable style={{ width: '100%' }} title={
-          <div className='network-request-title'><div>{resource.url}</div></div>
-        } body={
-          <div className='network-request-details'>{resource.responseHeaders.map(pair => `${pair.name}: ${pair.value}`).join('\n')}</div>
-        }/>
-      </div>;
+      return <NetworkResourceDetails resource={resource} key={index} index={index} selected={selected === index} setSelected={setSelected} />;
     })
   }</div>;
 };
+
+

src/trace/snapshotter.ts

@@ -30,7 +30,11 @@ export type SnapshotterResource = {
   url: string,
   contentType: string,
   responseHeaders: { name: string, value: string }[],
-  sha1: string,
+  requestHeaders: { name: string, value: string }[],
+  method: string,
+  status: number,
+  requestSha1: string,
+  responseSha1: string,
 };
 
 export type SnapshotterBlob = {
@@ -132,18 +136,29 @@ export class Snapshotter {
         contentType = value;
     }
 
+    const method = original.method();
+    const status = response.status();
+    const requestBody = original.postDataBuffer();
+    const requestSha1 = requestBody ? calculateSha1(requestBody) : 'none';
+    const requestHeaders = original.headers();
     const body = await response.body().catch(e => debugLogger.log('error', e));
-    const sha1 = body ? calculateSha1(body) : 'none';
+    const responseSha1 = body ? calculateSha1(body) : 'none';
     const resource: SnapshotterResource = {
       pageId: this._delegate.pageId(page),
       frameId: response.frame()._id,
       url,
       contentType,
       responseHeaders: response.headers(),
-      sha1,
+      requestHeaders,
+      method,
+      status,
+      requestSha1,
+      responseSha1,
     };
     this._delegate.onResource(resource);
+    if (requestBody)
+      this._delegate.onBlob({ sha1: requestSha1, buffer: requestBody });
     if (body)
-      this._delegate.onBlob({ sha1, buffer: body });
+      this._delegate.onBlob({ sha1: responseSha1, buffer: body });
   }
 }

src/trace/traceTypes.ts

@@ -39,7 +39,11 @@ export type NetworkResourceTraceEvent = {
   url: string,
   contentType: string,
   responseHeaders: { name: string, value: string }[],
-  sha1: string,
+  requestHeaders: { name: string, value: string }[],
+  method: string,
+  status: number,
+  requestSha1: string,
+  responseSha1: string,
 };
 
 export type PageCreatedTraceEvent = {

src/trace/tracer.ts

@@ -120,7 +120,11 @@ class ContextTracer implements SnapshotterDelegate, ActionListener {
       url: resource.url,
       contentType: resource.contentType,
       responseHeaders: resource.responseHeaders,
-      sha1: resource.sha1,
+      requestHeaders: resource.requestHeaders,
+      method: resource.method,
+      status: resource.status,
+      requestSha1: resource.requestSha1,
+      responseSha1: resource.responseSha1,
     };
     this._appendTraceEvent(event);
   }

test/assets/trace-resources.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Tracer XHR Network Resource example</title>
+  </head>
+  <body>
+    <div id="response-status"></div>
+    <div id="response-body"></div>
+    <script>
+      async function performXHR() {
+        const response = await window.fetch('./file.json', {method: 'POST', body: JSON.stringify({prop: 'value'})});
+        document.querySelector('#response-status').innerText = response.status;
+        const responseText = await response.text();
+        document.querySelector('#response-body').innerText = responseText;
+      }
+    </script>
+    <a onclick="javascipt:performXHR();">Download</a>
+  </body>
+</html>

test/trace.spec.ts

@@ -19,7 +19,7 @@ import type * as trace from '../src/trace/traceTypes';
 import * as path from 'path';
 import * as fs from 'fs';
 
-it('should record trace', test => test.fixme(), async ({browser, testInfo, server}) => {
+it('should record trace', async ({browser, testInfo, server}) => {
   const traceDir = testInfo.outputPath('trace');
   const context = await browser.newContext({ _traceDir: traceDir } as any);
   const page = await context.newPage();
@@ -46,6 +46,16 @@ it('should record trace', test => test.fixme(), async ({browser, testInfo, serve
   expect(gotoEvent.pageId).toBe(pageId);
   expect(gotoEvent.value).toBe(url);
 
+  const resourceEvent = traceEvents.find(event => event.type === 'resource' && event.url.endsWith('/frames/style.css')) as trace.NetworkResourceTraceEvent;
+  expect(resourceEvent).toBeTruthy();
+  expect(resourceEvent.contextId).toBe(contextId);
+  expect(resourceEvent.pageId).toBe(pageId);
+  expect(resourceEvent.method).toBe('GET');
+  expect(resourceEvent.status).toBe(200);
+  expect(resourceEvent.requestHeaders).toBeTruthy();
+  expect(resourceEvent.requestHeaders.length).toBeGreaterThan(0);
+  expect(resourceEvent.requestSha1).toBe('none');
+
   const clickEvent = traceEvents.find(event => event.type === 'action' && event.action === 'click') as trace.ActionTraceEvent;
   expect(clickEvent).toBeTruthy();
   expect(clickEvent.snapshots.length).toBe(2);
@@ -54,3 +64,47 @@ it('should record trace', test => test.fixme(), async ({browser, testInfo, serve
 
   expect(fs.existsSync(path.join(traceDir, 'resources', snapshotEvent.sha1))).toBe(true);
 });
+
+it('should record trace with POST', async ({browser, testInfo, server}) => {
+  const traceDir = testInfo.outputPath('trace');
+  const context = await browser.newContext({ _traceDir: traceDir } as any);
+  const page = await context.newPage();
+  const url = server.PREFIX + '/trace-resources.html';
+  await page.goto(url);
+  await page.click('text=Download');
+  await page.waitForSelector(`#response-status:text("404")`);
+  await context.close();
+
+  const tracePath = path.join(traceDir, fs.readdirSync(traceDir).find(n => n.endsWith('.trace')));
+  const traceFileContent = await fs.promises.readFile(tracePath, 'utf8');
+  const traceEvents = traceFileContent.split('\n').filter(line => !!line).map(line => JSON.parse(line)) as trace.TraceEvent[];
+
+  const contextEvent = traceEvents.find(event => event.type === 'context-created') as trace.ContextCreatedTraceEvent;
+  expect(contextEvent).toBeTruthy();
+  const contextId = contextEvent.contextId;
+
+  const pageEvent = traceEvents.find(event => event.type === 'page-created') as trace.PageCreatedTraceEvent;
+  expect(pageEvent).toBeTruthy();
+  expect(pageEvent.contextId).toBe(contextId);
+  const pageId = pageEvent.pageId;
+
+  const gotoEvent = traceEvents.find(event => event.type === 'action' && event.action === 'goto') as trace.ActionTraceEvent;
+  expect(gotoEvent).toBeTruthy();
+  expect(gotoEvent.contextId).toBe(contextId);
+  expect(gotoEvent.pageId).toBe(pageId);
+  expect(gotoEvent.value).toBe(url);
+
+  const resourceEvent = traceEvents.find(event => event.type === 'resource' && event.url.endsWith('/file.json')) as trace.NetworkResourceTraceEvent;
+  expect(resourceEvent).toBeTruthy();
+  expect(resourceEvent.contextId).toBe(contextId);
+  expect(resourceEvent.pageId).toBe(pageId);
+  expect(resourceEvent.method).toBe('POST');
+  expect(resourceEvent.status).toBe(404);
+  expect(resourceEvent.requestHeaders).toBeTruthy();
+  expect(resourceEvent.requestHeaders.length).toBeGreaterThan(0);
+  expect(resourceEvent.requestSha1).toBeTruthy();
+  expect(resourceEvent.responseSha1).toBeTruthy();
+
+  expect(fs.existsSync(path.join(traceDir, 'resources', resourceEvent.requestSha1))).toBe(true);
+  expect(fs.existsSync(path.join(traceDir, 'resources', resourceEvent.responseSha1))).toBe(true);
+});