playwright/utils/doclint/check_public_api/MDBuilder.js

/**
 * Copyright 2017 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

const Documentation = require('./Documentation');
const commonmark = require('commonmark');

class MDOutline {
  /**
   * @param {!Page} page
   * @param {string} text
   * @return {!MDOutline}
   */
  static async create(page, text) {
    // Render markdown as HTML.
    const reader = new commonmark.Parser();
    const parsed = reader.parse(text);
    const writer = new commonmark.HtmlRenderer();
    const html = writer.render(parsed);

    const logConsole = msg => console.log(msg.text());
    page.on('console', logConsole);
    // Extract headings.
    await page.setContent(html);
    const {classes, errors} = await page.evaluate(() => {
      const classes = [];
      const errors = [];
      const headers = document.body.querySelectorAll('h3');
      for (let i = 0; i < headers.length; i++) {
        const fragment = extractSiblingsIntoFragment(headers[i], headers[i + 1]);
        classes.push(parseClass(fragment));
      }
      return {classes, errors};

      /**
       * @param {HTMLLIElement} element
       * @param {boolean} defaultRequired
       */
      function parseProperty(element, defaultRequired) {
        const clone = element.cloneNode(true);
        const ul = clone.querySelector(':scope > ul');
        const str = parseComment(extractSiblingsIntoFragment(clone.firstChild, ul));
        const name = str.substring(0, str.indexOf('<')).replace(/\`/g, '').trim();
        const type = findType(str);
        const properties = [];
        let comment = str.substring(str.indexOf('<') + type.length + 2).trim();
        const hasNonEnumProperties = type.split('|').some(part => {
          const basicTypes = new Set(['string', 'number', 'boolean']);
          const arrayTypes = new Set([...basicTypes].map(type => `Array<${type}>`));
          return !basicTypes.has(part) && !arrayTypes.has(part) && !(part.startsWith('"') && part.endsWith('"'));
        });
        if (hasNonEnumProperties) {
          for (const childElement of element.querySelectorAll(':scope > ul > li')) {
            const text = childElement.textContent;
            if (text.startsWith(`"`) || text.startsWith(`'`))
              continue;
            const property = parseProperty(childElement, true);
            property.required = defaultRequired;
            if (property.comment.toLowerCase().includes('defaults to '))
              property.required = false;
            if (property.comment.startsWith('Optional '))
              property.required = false;
            if (property.comment.toLowerCase().includes('if applicable.'))
              property.required = false;
            if (property.comment.toLowerCase().includes('if available.'))
              property.required = false;
            if (property.comment.includes('**required**'))
              property.required = true;
            properties.push(property);
          }
        } else if (ul) {
          comment += '\n' + parseComment(ul).split('\n').map(l => ` - ${l}`).join('\n');
        }
        return {
          name,
          type,
          comment,
          properties
        };
      }

      /**
       * @param {string} str
       * @return {string}
       */
      function findType(str) {
        const start = str.indexOf('<') + 1;
        let count = 1;
        for (let i = start; i < str.length; i++) {
          if (str[i] === '<') count++;
          if (str[i] === '>') count--;
          if (!count)
            return str.substring(start, i);
        }
        return 'unknown';
      }

      /**
       * @param {DocumentFragment} content
       */
      function parseClass(content) {
        const members = [];
        const commentWalker = document.createTreeWalker(content, NodeFilter.SHOW_COMMENT | NodeFilter.SHOW_ELEMENT, {
          acceptNode(node) {
            if (node instanceof HTMLElement && node.tagName === 'H4')
              return NodeFilter.FILTER_ACCEPT;
            if (!(node instanceof Comment))
              return NodeFilter.FILTER_REJECT;
            if (node.data.trim().startsWith('GEN:toc'))
              return NodeFilter.FILTER_ACCEPT;
            return NodeFilter.FILTER_REJECT;
          }
        });
        const commentEnd = commentWalker.nextNode();
        const headers = content.querySelectorAll('h4');
        const name = content.firstChild.textContent;
        let extendsName = null;
        let commentStart = content.firstChild.nextSibling;
        const extendsElement = content.querySelector('ul');
        if (extendsElement && extendsElement.textContent.trim().startsWith('extends:')) {
          commentStart = extendsElement.nextSibling;
          extendsName = extendsElement.querySelector('a').textContent;
        }
        const comment = parseComment(extractSiblingsIntoFragment(commentStart, commentEnd));
        for (let i = 0; i < headers.length; i++) {
          const fragment = extractSiblingsIntoFragment(headers[i], headers[i + 1]);
          members.push(parseMember(fragment));
        }
        return {
          name,
          comment,
          extendsName,
          members
        };
      }

      /**
       * @param {Node} content
       */
      function parseComment(content) {
        for (const code of content.querySelectorAll('pre > code'))
          code.replaceWith('```' + code.className.substring('language-'.length) + '\n' + code.textContent + '```');
        for (const code of content.querySelectorAll('code'))
          code.replaceWith('`' + code.textContent + '`');
        for (const strong of content.querySelectorAll('strong'))
          strong.replaceWith('**' + parseComment(strong) + '**');
        return content.textContent.trim();
      }

      /**
       * @param {DocumentFragment} content
       */
      function parseMember(content) {
        const name = content.firstChild.textContent;
        const args = [];
        let returnType = null;

        const paramRegex = /^\w+\.[\w$]+\((.*)\)$/;
        const matches = paramRegex.exec(name) || ['', ''];
        const parameters = matches[1];
        const optionalStartIndex = parameters.indexOf('[');
        const optinalParamsStr = optionalStartIndex !== -1 ? parameters.substring(optionalStartIndex).replace(/[\[\]]/g, '') : '';
        const optionalparams = new Set(optinalParamsStr.split(',').filter(x => x).map(x => x.trim()));
        const ul = content.querySelector('ul');
        for (const element of content.querySelectorAll('h4 + ul > li')) {
          if (element.matches('li') && element.textContent.trim().startsWith('<')) {
            returnType = parseProperty(element, false);
          } else if (element.matches('li') && element.firstChild.matches && element.firstChild.matches('code')) {
            const property = parseProperty(element, false);
            property.required = !optionalparams.has(property.name) && !property.name.startsWith('...');
            args.push(property);
          } else if (element.matches('li') && element.firstChild.nodeType === Element.TEXT_NODE && element.firstChild.textContent.toLowerCase().startsWith('return')) {
            returnType = parseProperty(element, true);
            const expectedText = 'returns: ';
            let actualText = element.firstChild.textContent;
            let angleIndex = actualText.indexOf('<');
            let spaceIndex = actualText.indexOf(' ');
            angleIndex = angleIndex === -1 ? actualText.length : angleIndex;
            spaceIndex = spaceIndex === -1 ? actualText.length : spaceIndex + 1;
            actualText = actualText.substring(0, Math.min(angleIndex, spaceIndex));
            if (actualText !== expectedText)
              errors.push(`${name} has mistyped 'return' type declaration: expected exactly '${expectedText}', found '${actualText}'.`);
          }
        }
        const comment = parseComment(extractSiblingsIntoFragment(ul ? ul.nextSibling : content.querySelector('h4').nextSibling));
        return {
          name,
          args,
          returnType,
          comment
        };
      }

      /**
       * @param {!Node} fromInclusive
       * @param {!Node} toExclusive
       * @return {!DocumentFragment}
       */
      function extractSiblingsIntoFragment(fromInclusive, toExclusive) {
        const fragment = document.createDocumentFragment();
        let node = fromInclusive;
        while (node && node !== toExclusive) {
          const next = node.nextSibling;
          fragment.appendChild(node);
          node = next;
        }
        return fragment;
      }
    });
    page.off('console', logConsole);
    return new MDOutline(classes, errors);
  }

  constructor(classes, errors) {
    this.classes = [];
    this.errors = errors;
    const classHeading = /^class: (\w+)$/;
    const constructorRegex = /^new (\w+)\((.*)\)$/;
    const methodRegex = /^(\w+)\.([\w$]+)\((.*)\)$/;
    const propertyRegex = /^(\w+)\.(\w+)$/;
    const eventRegex = /^event: '(\w+)'$/;
    let currentClassName = null;
    let currentClassMembers = [];
    let currentClassComment = '';
    let currentClassExtends = null;
    for (const cls of classes) {
      const match = cls.name.match(classHeading);
      if (!match)
        continue;
      currentClassName = match[1];
      currentClassComment = cls.comment;
      currentClassExtends = cls.extendsName;
      for (const member of cls.members) {
        if (constructorRegex.test(member.name)) {
          const match = member.name.match(constructorRegex);
          handleMethod.call(this, member, match[1], 'constructor', match[2]);
        } else if (methodRegex.test(member.name)) {
          const match = member.name.match(methodRegex);
          handleMethod.call(this, member, match[1], match[2], match[3]);
        } else if (propertyRegex.test(member.name)) {
          const match = member.name.match(propertyRegex);
          handleProperty.call(this, member, match[1], match[2]);
        } else if (eventRegex.test(member.name)) {
          const match = member.name.match(eventRegex);
          handleEvent.call(this, member, match[1]);
        }
      }
      flushClassIfNeeded.call(this);
    }

    function handleMethod(member, className, methodName, parameters) {
      if (!currentClassName || !className || !methodName || className.toLowerCase() !== currentClassName.toLowerCase()) {
        this.errors.push(`Failed to process header as method: ${member.name}`);
        return;
      }
      parameters = parameters.trim().replace(/[\[\]]/g, '');
      if (parameters !== member.args.map(arg => arg.name).join(', '))
        this.errors.push(`Heading arguments for "${member.name}" do not match described ones, i.e. "${parameters}" != "${member.args.map(a => a.name).join(', ')}"`);
      const args = member.args.map(createPropertyFromJSON);
      let returnType = null;
      let returnComment = '';
      if (member.returnType) {
        const returnProperty = createPropertyFromJSON(member.returnType);
        returnType = returnProperty.type;
        returnComment = returnProperty.comment;
      }
      const method = Documentation.Member.createMethod(methodName, args, returnType, returnComment, member.comment);
      currentClassMembers.push(method);
    }

    function createPropertyFromJSON(payload) {
      const type = new Documentation.Type(payload.type, payload.properties.map(createPropertyFromJSON));
      const required = payload.required;
      return Documentation.Member.createProperty(payload.name, type, payload.comment, required);
    }

    function handleProperty(member, className, propertyName) {
      if (!currentClassName || !className || !propertyName || className.toLowerCase() !== currentClassName.toLowerCase()) {
        this.errors.push(`Failed to process header as property: ${member.name}`);
        return;
      }
      const type = member.returnType ? member.returnType.type : null;
      const properties = member.returnType ? member.returnType.properties : [];
      currentClassMembers.push(createPropertyFromJSON({type, name: propertyName, properties, comment: member.comment}));
    }

    function handleEvent(member, eventName) {
      if (!currentClassName || !eventName) {
        this.errors.push(`Failed to process header as event: ${member.name}`);
        return;
      }
      currentClassMembers.push(Documentation.Member.createEvent(eventName, member.returnType && createPropertyFromJSON(member.returnType).type, member.comment));
    }

    function flushClassIfNeeded() {
      if (currentClassName === null)
        return;
      this.classes.push(new Documentation.Class(currentClassName, currentClassMembers, currentClassExtends, currentClassComment));
      currentClassName = null;
      currentClassMembers = [];
    }
  }
}

/**
 * @param {!Page} page
 * @param {!Array<!Source>} sources
 * @return {!Promise<{documentation: !Documentation, errors: !Array<string>}>}
 */
module.exports = async function(page, sources) {
  const classes = [];
  const errors = [];
  for (const source of sources) {
    const outline = await MDOutline.create(page, source.text());
    classes.push(...outline.classes);
    errors.push(...outline.errors);
  }
  const documentation = new Documentation(classes);


  // Push base class documentation to derived classes.
  for (const [name, clazz] of documentation.classes.entries()) {
    clazz.validateOrder(errors, clazz);

    if (!clazz.extends || clazz.extends === 'EventEmitter' || clazz.extends === 'Error')
      continue;
    const superClass = documentation.classes.get(clazz.extends);
    if (!superClass) {
      errors.push(`Undefined superclass: ${superClass} in ${name}`);
      continue;
    }
    for (const memberName of clazz.members.keys()) {
      if (superClass.members.has(memberName))
        errors.push(`Member documentation overrides base: ${name}.${memberName} over ${clazz.extends}.${memberName}`);
    }

    clazz.membersArray = [...clazz.membersArray, ...superClass.membersArray];
    clazz.index();
  }
  return { documentation, errors };
};