Add dbt cloud URL as an input parameter (#9911)

* Add dbt cloud URL as an input parameter

* support dbt cloud url from UI

* Flag dbtCloudUrl as required

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>

ingestion/src/metadata/utils/dbt_config.py

@@ -157,7 +157,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
         expiry = 0
         auth_token = config.dbtCloudAuthToken.get_secret_value(), expiry
         client_config = ClientConfig(
-            base_url="https://cloud.getdbt.com",
+            base_url=config.dbtCloudUrl,
             api_version="api/v2",
             auth_token=lambda: auth_token,
             auth_header="Authorization",

openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json

@@ -2,27 +2,34 @@
   "$id": "https://open-metadata.org/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "DBT Cloud Config",
-  "description": "DBT Cloud configuration.",
+  "description": "dbt Cloud configuration.",
   "javaType": "org.openmetadata.schema.metadataIngestion.dbtconfig.DbtCloudConfig",
   "type": "object",
   "properties": {
     "dbtCloudAuthToken": {
-      "title": "DBT Cloud Authentication Token",
-      "description": "DBT cloud account authentication token",
+      "title": "dbt Cloud Authentication Token",
+      "description": "dbt cloud account authentication token",
       "type": "string",
       "format": "password"
     },
     "dbtCloudAccountId": {
-      "title": "DBT Cloud Account Id",
-      "description": "DBT cloud account Id",
+      "title": "dbt Cloud Account Id",
+      "description": "dbt cloud account Id",
       "type": "string"
     },
     "dbtCloudProjectId": {
-      "title": "DBT Cloud Project Id",
-      "description": "In case of multiple projects in a DBT cloud account, specify the project's id from which you want to extract the DBT run artifacts",
+      "title": "dbt Cloud Project Id",
+      "description": "In case of multiple projects in a dbt cloud account, specify the project's id from which you want to extract the dbt run artifacts",
       "type": "string"
+    },
+    "dbtCloudUrl": {
+      "title": "dbt Cloud URL",
+      "description": "URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or https://emea.dbt.com/",
+      "type": "string",
+      "format": "uri",
+      "default": "https://cloud.getdbt.com"
     }
   },
   "additionalProperties": false,
-  "required": ["dbtCloudAuthToken", "dbtCloudAccountId"]
+  "required": ["dbtCloudAuthToken", "dbtCloudAccountId", "dbtCloudUrl"]
 }

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.test.tsx

@@ -22,11 +22,13 @@ const mockAuthTokenChange = jest.fn();
 const mockUpdateDescriptions = jest.fn();
 const mockDbtCloudProjectId = jest.fn();
 const mockUpdateDBTClassification = jest.fn();
+const mockUpdateDBTCloudUrl = jest.fn();
 
 const mockProps = {
   dbtCloudAccountId: '',
   dbtCloudAuthToken: '',
   dbtUpdateDescriptions: false,
+  dbtCloudUrl: 'https://cloud.getdbt.com/',
   okText: 'Next',
   cancelText: 'Back',
   onCancel: mockCancel,
@@ -35,6 +37,7 @@ const mockProps = {
   handleCloudAuthTokenChange: mockAuthTokenChange,
   handleUpdateDescriptions: mockUpdateDescriptions,
   handleDbtCloudProjectId: mockDbtCloudProjectId,
+  handleDbtCloudUrl: mockUpdateDBTCloudUrl,
   handleUpdateDBTClassification: mockUpdateDBTClassification,
 };
 

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.tsx

@@ -34,6 +34,7 @@ interface Props extends DBTFormCommonProps, DbtConfigCloud {
   handleUpdateDescriptions: (value: boolean) => void;
   handleDbtCloudProjectId: (value: string) => void;
   handleUpdateDBTClassification: (value: string) => void;
+  handleDbtCloudUrl: (value: string) => void;
 }
 
 export const DBTCloudConfig: FunctionComponent<Props> = ({
@@ -41,6 +42,7 @@ export const DBTCloudConfig: FunctionComponent<Props> = ({
   dbtCloudAuthToken = '',
   dbtCloudProjectId,
   dbtUpdateDescriptions = false,
+  dbtCloudUrl = 'https://cloud.getdbt.com/',
   okText,
   cancelText,
   onCancel,
@@ -50,6 +52,7 @@ export const DBTCloudConfig: FunctionComponent<Props> = ({
   handleUpdateDescriptions,
   handleDbtCloudProjectId,
   dbtClassificationName,
+  handleDbtCloudUrl,
   handleUpdateDBTClassification,
 }: Props) => {
   const [errors, setErrors] = useState<ErrorDbtCloud>();
@@ -68,6 +71,7 @@ export const DBTCloudConfig: FunctionComponent<Props> = ({
       dbtUpdateDescriptions,
       dbtCloudProjectId,
       dbtClassificationName,
+      dbtCloudUrl,
     };
     if (validate(submitData)) {
       onSubmit(submitData);
@@ -136,6 +140,25 @@ export const DBTCloudConfig: FunctionComponent<Props> = ({
           onChange={(e) => handleDbtCloudProjectId(e.target.value)}
         />
       </Field>
+
+      <Field>
+        <label className="tw-block tw-form-label tw-mb-1" htmlFor="dbtCloudUrl">
+          {requiredField('dbt Cloud URL')}
+        </label>
+        <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
+          URL to connect to your dbt cloud instance. E.g.,
+          https://cloud.getdbt.com or https://emea.dbt.com/
+        </p>
+        <input
+          className="tw-form-inputs tw-form-inputs-padding"
+          data-testid="dbtCloudUrl"
+          id="dbtCloudUrl"
+          name="dbtCloudUrl"
+          type="text"
+          value={dbtCloudUrl}
+          onChange={(e) => handleDbtCloudUrl(e.target.value)}
+        />
+      </Field>
       {getSeparator('')}
 
       <DBTCommonFields

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigForm.interface.ts

@@ -44,6 +44,7 @@ export type DbtConfigCloud = Pick<
   | 'dbtUpdateDescriptions'
   | 'dbtCloudProjectId'
   | 'dbtClassificationName'
+  | 'dbtCloudUrl'
 >;
 
 export type DbtConfigLocal = Pick<

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx

@@ -89,6 +89,7 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
         dbtCloudAccountId={dbtConfig?.dbtCloudAccountId}
         dbtCloudAuthToken={dbtConfig?.dbtCloudAuthToken}
         dbtCloudProjectId={dbtConfig?.dbtCloudProjectId}
+        dbtCloudUrl={dbtConfig.dbtCloudUrl}
         dbtUpdateDescriptions={dbtConfig?.dbtUpdateDescriptions}
         handleCloudAccountIdChange={(val) => {
           updateDbtConfig('dbtCloudAccountId', val);
@@ -99,6 +100,7 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
         handleDbtCloudProjectId={(val) => {
           updateDbtConfig('dbtCloudProjectId', val);
         }}
+        handleDbtCloudUrl={(val) => updateDbtConfig('dbtCloudUrl', val)}
         handleUpdateDBTClassification={(val) => {
           updateDbtConfig('dbtClassificationName', val);
         }}