feat(dbt): show source and compiled code in the UI (#10028)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java

@@ -238,6 +238,7 @@ public class DatasetMapper implements ModelMapper<EntityResponse, Dataset> {
     graphqlProperties.setMaterialized(properties.isMaterialized());
     graphqlProperties.setLanguage(properties.getViewLanguage());
     graphqlProperties.setLogic(properties.getViewLogic());
+    graphqlProperties.setFormattedLogic(properties.getFormattedViewLogic());
     dataset.setViewProperties(graphqlProperties);
   }
 

datahub-graphql-core/src/main/resources/entity.graphql

@@ -3186,6 +3186,12 @@ type ViewProperties {
   """
   logic: String!
 
+  """
+  A formatted version of the logic associated with the view.
+  For dbt, this contains the compiled SQL.
+  """
+  formattedLogic: String
+
   """
   The language in which the view logic is written, for example SQL
   """

datahub-web-react/src/app/entity/shared/tabs/Dataset/View/ViewDefinitionTab.tsx

@@ -1,11 +1,12 @@
-import { Typography } from 'antd';
-import React from 'react';
+import { Radio, Typography } from 'antd';
+import React, { useState } from 'react';
 import styled from 'styled-components';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated';
 import { ANTD_GRAY } from '../../../constants';
 import { useBaseEntity } from '../../../EntityContext';
 import { InfoItem } from '../../../components/styled/InfoItem';
+import { DBT_URN } from '../../../../../ingest/source/builder/constants';
 
 const InfoSection = styled.div`
     border-bottom: 1px solid ${ANTD_GRAY[4.5]};
@@ -23,9 +24,14 @@ const InfoItemContent = styled.div`
     padding-top: 8px;
 `;
 
+const FormattingSelector = styled.div`
+    margin-top: 10px;
+`;
+
 const QueryText = styled(Typography.Paragraph)`
-    margin-top: 20px;
+    margin-top: 15px;
     background-color: ${ANTD_GRAY[2]};
+    border-radius: 5px;
 `;
 
 // NOTE: Yes, using `!important` is a shame. However, the SyntaxHighlighter is applying styles directly
@@ -38,9 +44,16 @@ const NestedSyntax = styled(SyntaxHighlighter)`
 export default function ViewDefinitionTab() {
     const baseEntity = useBaseEntity<GetDatasetQuery>();
     const logic = baseEntity?.dataset?.viewProperties?.logic || 'UNKNOWN';
+    const formattedLogic = baseEntity?.dataset?.viewProperties?.formattedLogic;
     const materialized = (baseEntity?.dataset?.viewProperties?.materialized && true) || false;
     const language = baseEntity?.dataset?.viewProperties?.language || 'UNKNOWN';
 
+    const isDbt = baseEntity?.dataset?.platform?.urn === DBT_URN;
+    const formatOptions = isDbt ? ['Source', 'Compiled'] : ['Raw', 'Formatted'];
+
+    const canShowFormatted = !!formattedLogic;
+    const [showFormatted, setShowFormatted] = useState(false);
+
     return (
         <>
             <InfoSection>
@@ -56,8 +69,21 @@ export default function ViewDefinitionTab() {
             </InfoSection>
             <InfoSection>
                 <Typography.Title level={5}>Logic</Typography.Title>
+                {canShowFormatted && (
+                    <FormattingSelector>
+                        <Radio.Group
+                            options={[
+                                { label: formatOptions[0], value: false },
+                                { label: formatOptions[1], value: true },
+                            ]}
+                            onChange={(e) => setShowFormatted(e.target.value)}
+                            value={showFormatted}
+                            optionType="button"
+                        />
+                    </FormattingSelector>
+                )}
                 <QueryText>
-                    <NestedSyntax language="sql">{logic}</NestedSyntax>
+                    <NestedSyntax language="sql">{showFormatted ? formattedLogic : logic}</NestedSyntax>
                 </QueryText>
             </InfoSection>
         </>

datahub-web-react/src/graphql/dataset.graphql

@@ -230,6 +230,7 @@ fragment viewProperties on Dataset {
     viewProperties {
         materialized
         logic
+        formattedLogic
         language
     }
 }

docs/how/updating-datahub.md

@@ -22,6 +22,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 - #9934 and #10075 - Stateful ingestion is now enabled by default if a `pipeline_name` is set and either a datahub-rest sink or `datahub_api` is specified. It will still be disabled by default when any other sink type is used or if there is no pipeline name set.
 - #10002 - The `DataHubGraph` client no longer makes a request to the backend during initialization. If you want to preserve the old behavior, call `graph.test_connection()` after constructing the client.
+- #10026 - The dbt `use_compiled_code` option has been removed, because we now support capturing both source and compiled dbt SQL. This can be configured using `include_compiled_code`, which will be default enabled in 0.13.1.
 - #10055 - Assertion entities generated by dbt are now associated with the dbt dataset entity, and not the entity in the data warehouse.
 
 ### Potential Downtime