F1-63: Add support for MotherDuck into SDK and gooddata-dbt

Author: jaceksan

gooddata-dbt/gooddata_dbt/dbt/profiles.py

@@ -9,11 +9,14 @@
 import yaml
 from gooddata_sdk import (
     BasicCredentials,
+    CatalogDataSourceMotherDuck,
     CatalogDataSourcePostgres,
     CatalogDataSourceSnowflake,
     CatalogDataSourceVertica,
+    MotherDuckAttributes,
     PostgresAttributes,
     SnowflakeAttributes,
+    TokenCredentialsFromEnvVar,
     VerticaAttributes,
 )
 
@@ -112,13 +115,50 @@ def to_gooddata(self, data_source_id: str, schema_name: str) -> CatalogDataSourc
         )
 
 
-DbtOutput = Union[DbtOutputPostgreSQL, DbtOutputSnowflake, DbtOutputVertica]
+@attrs.define(auto_attribs=True, kw_only=True)
+class DbtOutputMotherDuck(Base):
+    name: str
+    title: str
+    path: str
+    schema: str
+    database: str = ""
+
+    def __attrs_post_init__(self) -> None:
+        self.database = self.validate_connection_props()
+
+    def validate_connection_props(self) -> str:
+        if not self.path.startswith("md:"):
+            raise ValueError(f"Path {self.path} is not a valid MotherDuck path.")
+        _, db_name = self.path.split(":", 1)
+        if db_name:
+            return db_name
+        elif self.database:
+            return self.database
+        else:
+            raise ValueError(f"Database name is neither specified in {self.path=} nor in {self.database=}")
+
+    def to_gooddata(self, data_source_id: str, schema_name: str) -> CatalogDataSourceMotherDuck:
+        return CatalogDataSourceMotherDuck(
+            id=data_source_id,
+            name=self.title,
+            db_specific_attributes=MotherDuckAttributes(
+                db_name=quote_plus(self.database),
+            ),
+            # Schema name is collected from dbt manifest from relevant tables
+            schema=schema_name,
+            credentials=TokenCredentialsFromEnvVar(
+                env_var_name="MOTHERDUCK_TOKEN",
+            ),
+        )
+
+
+DbtOutput = Union[DbtOutputPostgreSQL, DbtOutputSnowflake, DbtOutputVertica, DbtOutputMotherDuck]
 
 
 @attrs.define(auto_attribs=True, kw_only=True)
 class DbtProfile(Base):
     name: str
-    outputs: List[Union[DbtOutputPostgreSQL, DbtOutputSnowflake, DbtOutputVertica]]
+    outputs: List[DbtOutput]
 
 
 class DbtProfiles:
@@ -135,14 +175,17 @@ def __init__(self, args: argparse.Namespace) -> None:
 
     @staticmethod
     def inject_env_vars(output_def: Dict) -> None:
-        env_re = re.compile(r"env_var\('([^']+)'(,\s*'([^']+)')?\)")
+        env_re = re.compile(r"\{\{ env_var\('([^']+)'(,\s*'([^']+)')?\) \}\}")
         for output_key, output_value in output_def.items():
             if (env_match := env_re.search(str(output_value))) is not None:
                 default_value = None
                 if len(env_match.groups()) == 3:
                     default_value = env_match.group(3)
                 final_value = os.getenv(env_match.group(1)) or default_value
-                output_def[output_key] = final_value
+                if final_value is None:
+                    output_def[output_key] = ""
+                else:
+                    output_def[output_key] = env_re.sub(final_value, str(output_value))
             # else do nothing, real value seems to be stored in dbt profile
 
     @staticmethod
@@ -154,6 +197,8 @@ def to_data_class(output: str, output_def: Dict) -> DbtOutput:
             return DbtOutputSnowflake.from_dict({"name": output, **output_def})
         elif db_type == "vertica":
             return DbtOutputVertica.from_dict({"name": output, **output_def})
+        elif db_type == "duckdb":
+            return DbtOutputMotherDuck.from_dict({"name": output, **output_def})
         else:
             raise Exception(f"Unsupported database type {output=} {db_type=}")
 

gooddata-dbt/gooddata_dbt/gooddata/api_wrapper.py

@@ -4,6 +4,7 @@
 from typing import Any, List, Union
 
 from gooddata_sdk import (
+    CatalogDataSourceMotherDuck,
     CatalogDataSourcePostgres,
     CatalogDataSourceSnowflake,
     CatalogDataSourceVertica,
@@ -20,7 +21,12 @@
 
 from gooddata_dbt.gooddata.config import GoodDataConfigLocalizationTo, GoodDataConfigProduct
 
-DataSource = Union[CatalogDataSourcePostgres, CatalogDataSourceSnowflake, CatalogDataSourceVertica]
+DataSource = Union[
+    CatalogDataSourcePostgres,
+    CatalogDataSourceSnowflake,
+    CatalogDataSourceVertica,
+    CatalogDataSourceMotherDuck,
+]
 
 
 class GoodDataApiWrapper:

gooddata-sdk/gooddata_sdk/__init__.py

@@ -32,6 +32,7 @@
     CatalogDataSourceDatabricks,
     CatalogDataSourceGreenplum,
     CatalogDataSourceMariaDb,
+    CatalogDataSourceMotherDuck,
     CatalogDataSourceMsSql,
     CatalogDataSourceMySql,
     CatalogDataSourcePostgres,
@@ -41,6 +42,7 @@
     DatabricksAttributes,
     GreenplumAttributes,
     MariaDbAttributes,
+    MotherDuckAttributes,
     MsSqlAttributes,
     MySqlAttributes,
     PostgresAttributes,
@@ -51,7 +53,12 @@
 from gooddata_sdk.catalog.data_source.service import CatalogDataSourceService
 from gooddata_sdk.catalog.data_source.validation.data_source import DataSourceValidator
 from gooddata_sdk.catalog.depends_on import CatalogDependsOn, CatalogDependsOnDateFilter
-from gooddata_sdk.catalog.entity import AttrCatalogEntity, BasicCredentials, TokenCredentialsFromFile
+from gooddata_sdk.catalog.entity import (
+    AttrCatalogEntity,
+    BasicCredentials,
+    TokenCredentialsFromEnvVar,
+    TokenCredentialsFromFile,
+)
 from gooddata_sdk.catalog.export.request import (
     ExportCustomLabel,
     ExportCustomMetric,

gooddata-sdk/gooddata_sdk/catalog/data_source/entity_model/data_source.py

@@ -279,3 +279,14 @@ class CatalogDataSourceMySql(CatalogDataSource):
 class CatalogDataSourceMariaDb(CatalogDataSourceMySql):
     type: str = "MARIADB"
     db_vendor: str = "mariadb"
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class MotherDuckAttributes(DatabaseAttributes):
+    db_name: str
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class CatalogDataSourceMotherDuck(CatalogDataSource):
+    _URL_TMPL: ClassVar[str] = "jdbc:duckdb:md:{db_name}"
+    type: str = "MOTHERDUCK"

gooddata-sdk/gooddata_sdk/catalog/entity.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import base64
+import os
 from pathlib import Path
 from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar
 
@@ -181,6 +182,34 @@ def token_from_file(file_path: Path) -> str:
             return base64.b64encode(fp.read()).decode("utf-8")
 
 
+@attr.s(auto_attribs=True, kw_only=True)
+class TokenCredentialsFromEnvVar(Credentials):
+    env_var_name: str
+    token: str = attr.field(init=False, repr=lambda value: "***")
+
+    def __attrs_post_init__(self) -> None:
+        self.token = self.token_from_env_var(self.env_var_name)
+
+    def to_api_args(self) -> dict[str, Any]:
+        return {self.TOKEN_KEY: self.token}
+
+    @classmethod
+    def is_part_of_api(cls, entity: dict[str, Any]) -> bool:
+        return cls.USER_KEY not in entity
+
+    @classmethod
+    def from_api(cls, entity: dict[str, Any]) -> TokenCredentialsFromEnvVar:
+        # Credentials are not returned for security reasons
+        raise NotImplementedError
+
+    @staticmethod
+    def token_from_env_var(env_var_name: str) -> str:
+        token = os.getenv(env_var_name)
+        if token is None or token == "":
+            raise ValueError(f"Environment variable {env_var_name} is not set")
+        return base64.b64encode(token.encode("utf-8")).decode("utf-8")
+
+
 @attr.s(auto_attribs=True, kw_only=True)
 class BasicCredentials(Credentials):
     username: str