Added delete API for Raw Cost Analysis Report Rows (#13435)

* Added delete API

* review comments

* fixed checkstyle

* fixed naming

* checkstyle

---------

Co-authored-by: Ayush Shah <ayush@getcollate.io>

ingestion/src/metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py

@@ -60,6 +60,16 @@ class RawCostAnalysisReportDataProcessor(DataProcessor):
 
     _data_processor_type = ReportDataType.RawCostAnalysisReportData.value
 
+    def __init__(self, metadata: OpenMetadata):
+        super().__init__(metadata)
+        self.pre_hook = self._pre_hook_fn
+
+    def _pre_hook_fn(self):
+        """
+        Method to delete the previous rows of the RawCostAnalysisReportData type report
+        """
+        self.metadata.delete_report_data(ReportDataType.RawCostAnalysisReportData)
+
     def yield_refined_data(self) -> Iterable[ReportData]:
         """yield refined data"""
         for _, value in self._refined_data.items():

ingestion/src/metadata/data_insight/source/metadata.py

@@ -114,7 +114,7 @@ class DataInsightSource(Source):
         for report_data_type in ReportDataType:
             logger.info(f"Processing data for report type {report_data_type}")
             try:
-                self.metadata.delete_report_data(report_data_type, self.date)
+                self.metadata.delete_report_data_at_date(report_data_type, self.date)
                 producer = producer_factory.create(
                     report_data_type.value, self.metadata
                 )

ingestion/src/metadata/ingestion/ometa/mixins/data_insight_mixin.py

@@ -175,7 +175,9 @@ class DataInsightMixin:
         event_type_value = event_type.value
         self.client.delete(f"/analytics/web/events/{event_type_value}/{tmsp}/collect")
 
-    def delete_report_data(self, report_data_type: ReportDataType, date: str) -> None:
+    def delete_report_data_at_date(
+        self, report_data_type: ReportDataType, date: str
+    ) -> None:
         """Delete report data at a specific date for a specific report data type
 
         Args:
@@ -185,3 +187,11 @@ class DataInsightMixin:
         self.client.delete(
             f"/analytics/dataInsights/data/{report_data_type.value}/{date}"
         )
+
+    def delete_report_data(self, report_data_type: ReportDataType) -> None:
+        """Delete report data for a specific report data type
+
+        Args:
+            report_data_type (ReportDataType): report date type to delete
+        """
+        self.client.delete(f"/analytics/dataInsights/data/{report_data_type.value}")

ingestion/src/metadata/ingestion/source/database/greenplum/metadata.py

@@ -116,14 +116,14 @@ class GreenplumSource(CommonDbSourceService):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata_config: OpenMetadataConnection):
+    def create(cls, config_dict, metadata: OpenMetadataConnection):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: GreenplumConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, GreenplumConnection):
             raise InvalidSourceException(
                 f"Expected GreenplumConnection, but got {connection}"
             )
-        return cls(config, metadata_config)
+        return cls(config, metadata)
 
     def query_table_names_and_types(
         self, schema_name: str

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java

@@ -3308,6 +3308,9 @@ public interface CollectionDAO {
             "DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType and DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) = DATE(:date)",
         connectionType = POSTGRES)
     void deleteReportDataTypeAtDate(@BindFQN("reportDataType") String reportDataType, @Bind("date") String date);
+
+    @SqlUpdate("DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType")
+    void deletePreviousReportData(@BindFQN("reportDataType") String reportDataType);
   }
 
   interface ProfilerDataTimeSeriesDAO extends EntityTimeSeriesDAO {

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ReportDataRepository.java

@@ -39,10 +39,22 @@ public class ReportDataRepository extends EntityTimeSeriesRepository<ReportData>
     cleanUpIndex(reportDataType, date);
   }
 
+  public void deleteReportData(ReportDataType reportDataType) {
+    ((CollectionDAO.ReportDataTimeSeriesDAO) timeSeriesDao).deletePreviousReportData(reportDataType.value());
+    cleanUpPreviousIndex(reportDataType);
+  }
+
   private void cleanUpIndex(ReportDataType reportDataType, String date) {
     HashMap<String, Object> params = new HashMap<>();
     params.put("date_", date);
     String scriptTxt = "doc['timestamp'].value.toLocalDate() == LocalDate.parse(params.date_);";
     searchRepository.deleteByScript(reportDataType.toString(), scriptTxt, params);
   }
+
+  private void cleanUpPreviousIndex(ReportDataType reportDataType) {
+    HashMap<String, Object> params = new HashMap<>();
+    params.put("reportDataType_", reportDataType.value());
+    String scriptTxt = "doc['reportDataType'].value ==  params.reportDataType_";
+    searchRepository.deleteByScript(reportDataType.toString(), scriptTxt, params);
+  }
 }

openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/ReportDataResource.java

@@ -143,4 +143,31 @@ public class ReportDataResource extends EntityTimeSeriesResource<ReportData, Rep
     repository.deleteReportDataAtDate(reportDataType, date);
     return Response.ok().build();
   }
+
+  @DELETE
+  @Path("/{reportDataType}")
+  @Operation(
+      operationId = "deletePreviousReportData",
+      summary = "Delete all the previous report data for a given report data type",
+      description = "Delete all the previous report data for a given report data type.",
+      responses = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "Successfully deleted previous report data.",
+            content = @Content(mediaType = "application/json", schema = @Schema(implementation = ReportData.class)))
+      })
+  public Response deletePreviousReportData(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "report data type", schema = @Schema(implementation = ReportDataType.class))
+          @NonNull
+          @PathParam("reportDataType")
+          ReportDataType reportDataType)
+      throws IOException {
+    OperationContext operationContext = new OperationContext(Entity.DATA_INSIGHT_CHART, MetadataOperation.DELETE);
+    ResourceContextInterface resourceContext = ReportDataContext.builder().build();
+    authorizer.authorize(securityContext, operationContext, resourceContext);
+    repository.deleteReportData(reportDataType);
+    return Response.ok().build();
+  }
 }

openmetadata-service/src/main/resources/elasticsearch/indexMapping.json

@@ -152,7 +152,7 @@
   "RawCostAnalysisReportData": {
     "indexName": "raw_cost_analysis_report_data_index",
     "indexMappingFile": "/elasticsearch/raw_cost_analysis_report_data_index.json",
-    "alias": "rawCostAnalysisReportData",
+    "alias": "RawCostAnalysisReportData",
     "parentAliases": []
   },
   "AggregatedCostAnalysisReportData": {