postgres-sql-query-updated (#7421)

ingestion/src/metadata/ingestion/source/database/postgres_lineage.py

@@ -28,8 +28,8 @@ class PostgresLineageSource(PostgresQueryParserSource, LineageSource):
 
     filters = """
                 AND (
-                    s.query ILIKE 'create table %% as select %%'
+                    s.query ILIKE '%%create table %% as select %%'
-                    OR s.query ILIKE 'insert %%'
+                    OR s.query ILIKE '%%insert %%'
                 )
             """
 

ingestion/src/metadata/ingestion/source/database/postgres_query_parser.py

@@ -14,7 +14,7 @@ Postgres Query parser module
 import csv
 import traceback
 from abc import ABC
-from datetime import datetime, timedelta
+from datetime import datetime
 from typing import Iterable, Optional
 
 from sqlalchemy.engine.base import Engine
@@ -64,13 +64,11 @@ class PostgresQueryParserSource(QueryParserSource, ABC):
             )
         return cls(config, metadata_config)
 
-    def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str:
+    def get_sql_statement(self) -> str:
         """
         returns sql statement to fetch query logs
         """
         return self.sql_stmt.format(
-            start_time=start_time,
-            end_time=end_time,
             result_limit=self.config.sourceConfig.config.resultLimit,
             filters=self.filters,
         )
@@ -135,20 +133,9 @@ class PostgresQueryParserSource(QueryParserSource, ABC):
             logger.debug(traceback.format_exc())
 
     def process_table_query(self) -> Optional[Iterable[TableQuery]]:
-        daydiff = self.end - self.start
-        for i in range(daydiff.days):
-            logger.info(
-                f"Scanning query logs for {(self.start + timedelta(days=i)).date()} - "
-                f"{(self.start + timedelta(days=i+1)).date()}"
-            )
         try:
             with get_connection(self.connection).connect() as conn:
-                    rows = conn.execute(
+                rows = conn.execute(self.get_sql_statement())
-                        self.get_sql_statement(
-                            start_time=self.start + timedelta(days=i),
-                            end_time=self.start + timedelta(days=i + 2),
-                        )
-                    )
                 queries = []
                 for row in rows:
                     row = dict(row)
@@ -161,18 +148,11 @@ class PostgresQueryParserSource(QueryParserSource, ABC):
                             schema_name=row.get("schema_name"),
                         ):
                             continue
-
-                            end_time = row["start_time"] + timedelta(
-                                milliseconds=row["total_exec_time"]
-                            )
-                            date_time = end_time.strftime("%m/%d/%Y, %H:%M:%S")
                         queries.append(
                             TableQuery(
                                 query=row["query_text"],
                                 userName=row["usename"],
-                                    startTime=str(row["start_time"]),
+                                analysisDate=datetime.now(),
-                                    endTime=date_time,
-                                    analysisDate=row["start_time"],
                                 aborted=self.get_aborted_status(row),
                                 databaseName=self.get_database_name(row),
                                 serviceName=self.config.serviceName,
@@ -191,9 +171,4 @@ class PostgresQueryParserSource(QueryParserSource, ABC):
         """
         Method to get database name
         """
-        key = "datname"
+        return data.get("database_name")
-        if self.config.sourceConfig.config.queryLogFilePath:
-            key = "database_name"
-        if not data[key] and self.connection.database:
-            return self.connection.database
-        return data[key]

ingestion/src/metadata/ingestion/source/database/postgres_usage.py

@@ -26,9 +26,4 @@ class PostgresUsageSource(PostgresQueryParserSource, UsageSource):
     """
 
     sql_stmt = POSTGRES_SQL_STATEMENT
-    filters = """
+    filters = ""
-                AND (
-                     NOT s.query ILIKE 'create table %% as select %%'
-                    OR   NOT s.query ILIKE  'insert %%'
-                )
-            """

ingestion/src/metadata/utils/sql_queries.py

@@ -468,22 +468,16 @@ POSTGRES_SQL_STATEMENT = textwrap.dedent(
     """
       SELECT
         u.usename,
-        d.datname,
+        d.datname database_name,
         s.query query_text,
-        a.query_start start_time,
+        s.total_exec_time
-        s.total_exec_time,
-        s.mean_exec_time,
-        s.calls
       FROM
         pg_stat_statements s
         JOIN pg_catalog.pg_database d ON s.dbid = d.oid
         JOIN pg_catalog.pg_user u ON s.userid = u.usesysid
-        JOIN pg_catalog.pg_stat_activity a ON d.datname = a.datname
       WHERE
-        a.query_start >= '{start_time}' AND
+        s.query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND
-        a.state_change <  current_timestamp
+        s.query NOT LIKE '/* {{"app": "dbt", %%}} */%%'
-        AND s.query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
-        AND s.query NOT LIKE '/* {{"app": "dbt", %%}} */%%'
         {filters}
       LIMIT {result_limit}
     """

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json

@@ -65,6 +65,12 @@
       "title": "Supports Metadata Extraction",
       "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
     },
+    "supportsUsageExtraction": {
+      "$ref": "../connectionBasicType.json#/definitions/supportsUsageExtraction"
+    },
+    "supportsLineageExtraction": {
+      "$ref": "../connectionBasicType.json#/definitions/supportsLineageExtraction"
+    },
     "supportsProfiler": {
       "title": "Supports Profiler",
       "$ref": "../connectionBasicType.json#/definitions/supportsProfiler"