test build on sphinx-markdown-builder

docs-website/sphinx/Makefile

@@ -26,10 +26,10 @@ $(VENV_SENTINEL): requirements.txt
 
 # Not using Python's http.server because it enables caching headers.
 serve:
-	serve -p 3001 _build/html/
+	serve -p 3001 _build/markdown/
 
-md: html
+md:
-	$(VENV_DIR)/bin/python3 convert_sphinx_to_docusaurus.py
+	@$(SPHINXBUILD) -M markdown "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) && $(VENV_DIR)/bin/python3 convert_sphinx_to_docusaurus.py
 
 # Route other targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).

docs-website/sphinx/apidocs/sdk.rst

@@ -0,0 +1,9 @@
+DataHub SDK
+=======
+
+The DataHub SDK is a Python library for interacting with the DataHub platform.
+
+.. automodule:: datahub.sdk.search_client
+
+.. automodule:: datahub.sdk.lineage_client
+

docs-website/sphinx/conf.py

@@ -26,8 +26,10 @@ extensions = [
     "sphinx_autodoc_typehints",
     # This enables us to autogenerate docs for our CLI.
     "sphinx_click",
+    "sphinx_markdown_builder",
 ]
 
+markdown_anchor_sections = True
 napoleon_use_param = True
 
 # Move type hint info to function description instead of signature

docs-website/sphinx/convert_sphinx_to_docusaurus.py

@@ -1,78 +1,39 @@
 import pathlib
-import json
-from bs4 import BeautifulSoup
-
 
 SPHINX_ROOT_DIR = pathlib.Path(".")
-SPHINX_BUILD_DIR = SPHINX_ROOT_DIR / pathlib.Path("_build/html/apidocs")
+SPHINX_BUILD_DIR = SPHINX_ROOT_DIR / pathlib.Path("_build/markdown/apidocs")
 DOCS_OUTPUT_DIR = pathlib.Path("../docs/python-sdk")
 
 
-def html_to_mdx(html: str) -> str:
-    # Because the HTML uses `class` and has `{}` in it, it isn't valid
-    # MDX. As such, we use React's dangerouslySetInnerHTML.
-    return f"""
-
-<div dangerouslySetInnerHTML={{{{__html: {json.dumps(html)}}}}}></div>
-
-"""
-
-
-def bs4_to_mdx(soup: BeautifulSoup) -> str:
-    # TODO: Eventually we should do something smarter here to
-    # generate something that's closer to real Markdown. This would
-    # be helpful, for example, for enabling Docusaurus to generate
-    # a table of contents for the page.
-    return html_to_mdx(str(soup))
-
-
-def convert_html_to_md(html_file: pathlib.Path) -> str:
-    html = html_file.read_text()
-    soup = BeautifulSoup(html, "html.parser")
-
-    body = soup.find("main").find("div", {"class": "bd-article-container"})
-    article = body.find("article")
-
-    # Remove all the "permalink to this heading" links.
-    for link in article.find_all("a", {"class": "headerlink"}):
-        link.decompose()
-
-    # Remove the trailing " – " from arguments that are missing
-    # a description.
-    for item in article.select("dl.field-list dd p"):
-        # Note - that's U+2013, not a normal hyphen.
-        if str(item).endswith(" – </p>"):
-            parent = item.parent
-            # print("orig item", item)
-            new_item = BeautifulSoup(str(item)[:-7] + "</p>", "html.parser")
-            # print("new-item", str(new_item))
-            parent.p.replace_with(new_item)
-            # print("item post replace", parent)
-
-    # Extract title from the h1.
-    title_element = article.find("h1")
-    title = title_element.text
-    title_element.decompose()
-
-    # TODO - generate nicer slugs for these pages
-    md_meta = f"""---
-title: {title}
----\n\n"""
-
-    return md_meta + bs4_to_mdx(article)
-
-
 def main():
     DOCS_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
 
-    for doc in SPHINX_BUILD_DIR.glob("**/*.html"):
+    for doc in SPHINX_BUILD_DIR.glob("**/*.md"):
-        md = convert_html_to_md(doc)
+        outfile = DOCS_OUTPUT_DIR / doc.relative_to(SPHINX_BUILD_DIR)
-
-        outfile = DOCS_OUTPUT_DIR / doc.relative_to(SPHINX_BUILD_DIR).with_suffix(".md")
         outfile.parent.mkdir(parents=True, exist_ok=True)
-        outfile.write_text(md)
 
-        print(f"Generated {outfile}")
+        with open(doc, "r") as f:
+            content = f.read()
+
+        # Replace dangerous characters
+        replacements = [
+            ("<function ", "<\\function "),
+            ("<id>", "<\\id>"),
+            ("<type>", "<\\type>"),
+            ("<id1>", "<\\id1>"),
+            ("<id2>", "<\\id2>"),
+            ("MDXContent.isMDXComponent = true", ""),
+        ]
+        for old, new in replacements:
+            content = content.replace(old, new)
+
+        # Wrap the entire content with div (top and bottom)
+        final_content = f"<div className=\"python-sdk\">\n\n{content.strip()}\n\n</div>\n"
+
+        with open(outfile, "w") as f:
+            f.write(final_content)
+
+        print(f"✅ Generated {outfile}")
 
 if __name__ == "__main__":
     main()

docs-website/sphinx/requirements.txt

@@ -5,6 +5,7 @@ sphinx-click==4.4.0
 sphinx_autodoc_typehints==1.22
 pydata-sphinx-theme==0.13.1
 snowballstemmer>=2.2,<3 # Fixes https://github.com/sphinx-doc/sphinx/issues/13533
+sphinx-markdown-builder==0.6.8
 
 # Because of https://github.com/pydata/pydata-sphinx-theme/issues/108
 accessible-pygments

docs-website/src/styles/sphinx.scss

@@ -1,124 +1,85 @@
-// Styles for Sphinx Python SDK generated docs
+.python-sdk {
-$borderRadius: 5px;
+  font-size: 16px;
+  line-height: 1.6;
+  color: var(--ifm-font-color-base);
 
-dl.py {
+  h1, h2, h3, h4 {
-  margin-bottom: calc(var(--ifm-spacing-vertical) * 2);
+    font-weight: 600;
-  font-size: 14px;
+    margin-top: 2rem;
-  border: 1px solid var(--ifm-hr-border-color);
+    margin-bottom: 1rem;
-  border-radius: $borderRadius;
+  }
 
-  code {
+  h3, h4 {
-    border: none;
+    border-bottom: 1px solid var(--ifm-hr-border-color);
-    background: none;
+    padding-bottom: 0.25rem;
+  }
+
+  // Signature block
+  h3:has(code), h4:has(code) {
+    background-color: var(--ifm-code-background);
+    border-radius: 6px;
+    padding: 0.75rem 1rem 0.75rem 2rem;
+    font-family: var(--ifm-font-family-monospace);
+    font-size: 0.95rem;
+    font-weight: 500;
+    color: var(--ifm-font-color-base);
   }
 
   p {
-    margin-bottom: 0;
+    margin-bottom: 1rem;
   }
 
-  dl {
+  code {
-    margin-bottom: var(--ifm-spacing-vertical);
+    background-color: var(--ifm-code-background);
+    padding: 0.2em 0.4em;
+    border-radius: 4px;
+    font-size: 0.9em;
   }
 
-  // The parameter name:
+  ul {
-  em.sig-param > span:first-child {
+    margin-bottom: 1.5rem;
-    font-weight: bold;
-  }
 
-  > dd:not(:empty) {
+    &:has(li strong) {
-    padding-bottom: var(--ifm-spacing-vertical);
+      background-color: var(--ifm-background-surface-color);
-  }
+      border-radius: 8px;
-
+      padding: 1rem 1rem 1rem 2rem;
-  dt.sig {
+      box-shadow: 0 1px 3px rgba(0,0,0,0.06);
-    box-sizing: border-box;
-    font-size: 0.9rem;
-    padding: var(--ifm-spacing-vertical);
-    border-radius: $borderRadius;
-    font-family: var(--ifm-font-family-monospace);
-    background-color: var(--ifm-background-surface-color);
-  }
-
-  > dd {
-    &:not(:empty) {
-      padding-top: calc(var(--ifm-spacing-vertical) / 2);
-      margin-top: 0;
-      margin-left: var(--ifm-spacing-horizontal);
-      margin-right: var(--ifm-spacing-horizontal);
-    }
-  }
-
-  // e.g. `class`, or `def`
-  em.property {
-    color: var(--ifm-font-color-base);
-    font-weight: bold;
-  }
-
-  // e.g. `MyClass`
-  span.sig-name {
-    color: #2774b3;
-    font-weight: bold;
-  }
-
-  // e.g classmethod
-  em.property {
-    color: #66b327;
-  }
-
-  em.sig-param {
-    span.default_value {
-      color: #66b327;
-    }
-  }
-
-  span.sig-return {
-    span.sig-return-typehint {
-      color: var(--ifm-font-color-base);
-
-      pre {
-        color: var(--ifm-font-color-base);
-      }
-    }
-  }
-
-  dl.field-list {
-    padding-top: calc(var(--ifm-spacing-vertical) / 2);
-    display: grid;
-    grid-template-columns: fit-content(30%) auto;
-    &:not(:first-child) {
-      border-top: 1px solid var(--ifm-hr-border-color);
     }
 
-    dt {
+    li {
-      margin-right: 0.5em;
+      margin-bottom: 0.75rem;
-    }
 
-    dd {
+      strong {
-      font-family: var(--ifm-font-family-monospace);
+        display: inline-block;
-    }
+        color: #2774b3;
-
+        font-weight: 600;
-    dt,
+        min-width: 90px;
-    dd {
-      margin-left: 0;
-      padding-left: 0;
-
-      &:not(:first-of-type) {
-        border-top: 1px solid var(--ifm-hr-border-color);
-        padding-top: var(--ifm-spacing-vertical);
-      }
-      &:not(:last-of-type) {
-        padding-bottom: var(--ifm-spacing-vertical);
       }
 
-      ul {
+      code {
-        list-style-type: none;
+        background: transparent;
-        padding-left: 0;
+        color: #66b327;
-        li {
+        padding: 0;
-          p {
-            margin: 0;
-            padding: 0;
-          }
-        }
       }
     }
+
+    // nested ul for params or return type formatting
+    ul {
+      margin-top: 0.5rem;
+      margin-bottom: 0.5rem;
+      padding-left: 1rem;
+      border-left: 2px dotted var(--ifm-color-emphasis-300);
+    }
+  }
+
+  // Return type section
+  li:has(strong:contains("Return type")) {
+    font-style: italic;
+    color: var(--ifm-font-color-secondary);
+  }
+
+  // Parameter type callouts like (Optional[str])
+  em > code {
+    color: #c678dd;
   }
 }